projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: obey kptr_restrict in traces
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db 30#include <asm/lowcore.h>
fdf03650 31#include <asm/etr.h>
b0c632db 32#include <asm/pgtable.h>
f5daba1d 33#include <asm/nmi.h>
a0616cde 34#include <asm/switch_to.h>
6d3da241 35#include <asm/isc.h>
1526bf9c 36#include <asm/sclp.h>
8f2abe6a 37#include "kvm-s390.h"
b0c632db
HC		 38#include "gaccess.h"
39
ea2cdd27
DH		 40#define KMSG_COMPONENT "kvm-s390"
41#undef pr_fmt
42#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43
5786fffa
CH		 44#define CREATE_TRACE_POINTS
45#include "trace.h"
ade38c31 46#include "trace-s390.h"
5786fffa 47
41408c28 48#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 49#define LOCAL_IRQS 32
50#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 52
b0c632db
HC		 53#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54
55struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 57 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
ce2e4f0b 67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 90 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 110 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 111 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 116 { NULL }
117};
118
9d8d5786
MM		 119/* upper facilities limit for kvm */
120unsigned long kvm_s390_fac_list_mask[] = {
a3ed8dae 121 0xffe6fffbfcfdfc40UL,
53df84f8 122 0x005e800000000000UL,
9d8d5786 123};
b0c632db 124
9d8d5786 125unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 126{
9d8d5786
MM		 127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 129}
130
9d8d5786 131static struct gmap_notifier gmap_notifier;
78f26131 132debug_info_t *kvm_s390_dbf;
9d8d5786 133
b0c632db 134/* Section: not file related */
13a34e06 135int kvm_arch_hardware_enable(void)
b0c632db
HC		 136{
137 /* every s390 is virtualization enabled ;-) */
10474ae8 138 return 0;
b0c632db
HC		 139}
140
2c70fe44
CB		 141static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142
fdf03650
FZ		 143/*
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
148 */
149static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 void *v)
151{
152 struct kvm *kvm;
153 struct kvm_vcpu *vcpu;
154 int i;
155 unsigned long long *delta = v;
156
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
161 }
162 }
163 return NOTIFY_OK;
164}
165
166static struct notifier_block kvm_clock_notifier = {
167 .notifier_call = kvm_clock_sync,
168};
169
b0c632db
HC		 170int kvm_arch_hardware_setup(void)
171{
2c70fe44
CB		 172 gmap_notifier.notifier_call = kvm_gmap_notifier;
173 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 174 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
175 &kvm_clock_notifier);
b0c632db
HC		 176 return 0;
177}
178
179void kvm_arch_hardware_unsetup(void)
180{
2c70fe44 181 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 182 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
183 &kvm_clock_notifier);
b0c632db
HC		 184}
185
b0c632db
HC		 186int kvm_arch_init(void *opaque)
187{
78f26131
CB		 188 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
189 if (!kvm_s390_dbf)
190 return -ENOMEM;
191
192 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
193 debug_unregister(kvm_s390_dbf);
194 return -ENOMEM;
195 }
196
84877d93
CH		 197 /* Register floating interrupt controller interface. */
198 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 199}
200
78f26131
CB		 201void kvm_arch_exit(void)
202{
203 debug_unregister(kvm_s390_dbf);
204}
205
b0c632db
HC		 206/* Section: device related */
207long kvm_arch_dev_ioctl(struct file *filp,
208 unsigned int ioctl, unsigned long arg)
209{
210 if (ioctl == KVM_S390_ENABLE_SIE)
211 return s390_enable_sie();
212 return -EINVAL;
213}
214
784aa3d7 215int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 216{
d7b0b5eb
CO		 217 int r;
218
2bd0ac4e 219 switch (ext) {
d7b0b5eb 220 case KVM_CAP_S390_PSW:
b6cf8788 221 case KVM_CAP_S390_GMAP:
52e16b18 222 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 223#ifdef CONFIG_KVM_S390_UCONTROL
224 case KVM_CAP_S390_UCONTROL:
225#endif
3c038e6b 226 case KVM_CAP_ASYNC_PF:
60b413c9 227 case KVM_CAP_SYNC_REGS:
14eebd91 228 case KVM_CAP_ONE_REG:
d6712df9 229 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 230 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 231 case KVM_CAP_IOEVENTFD:
c05c4186 232 case KVM_CAP_DEVICE_CTRL:
d938dc55 233 case KVM_CAP_ENABLE_CAP_VM:
78599d90 234 case KVM_CAP_S390_IRQCHIP:
f2061656 235 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 236 case KVM_CAP_MP_STATE:
47b43c52 237 case KVM_CAP_S390_INJECT_IRQ:
2444b352 238 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 239 case KVM_CAP_S390_USER_STSI:
30ee2a98 240 case KVM_CAP_S390_SKEYS:
816c7667 241 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 242 r = 1;
243 break;
41408c28
TH		 244 case KVM_CAP_S390_MEM_OP:
245 r = MEM_OP_MAX_SIZE;
246 break;
e726b1bd
CB		 247 case KVM_CAP_NR_VCPUS:
248 case KVM_CAP_MAX_VCPUS:
fe0edcb7
ED		 249 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
250 : KVM_S390_BSCA_CPU_SLOTS;
e726b1bd 251 break;
e1e2e605
NW		 252 case KVM_CAP_NR_MEMSLOTS:
253 r = KVM_USER_MEM_SLOTS;
254 break;
1526bf9c 255 case KVM_CAP_S390_COW:
abf09bed 256 r = MACHINE_HAS_ESOP;
1526bf9c 257 break;
68c55750
EF		 258 case KVM_CAP_S390_VECTOR_REGISTERS:
259 r = MACHINE_HAS_VX;
260 break;
2bd0ac4e 261 default:
d7b0b5eb 262 r = 0;
2bd0ac4e 263 }
d7b0b5eb 264 return r;
b0c632db
HC		 265}
266
15f36ebd
JH		 267static void kvm_s390_sync_dirty_log(struct kvm *kvm,
268 struct kvm_memory_slot *memslot)
269{
270 gfn_t cur_gfn, last_gfn;
271 unsigned long address;
272 struct gmap *gmap = kvm->arch.gmap;
273
274 down_read(&gmap->mm->mmap_sem);
275 /* Loop over all guest pages */
276 last_gfn = memslot->base_gfn + memslot->npages;
277 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
278 address = gfn_to_hva_memslot(memslot, cur_gfn);
279
280 if (gmap_test_and_clear_dirty(address, gmap))
281 mark_page_dirty(kvm, cur_gfn);
282 }
283 up_read(&gmap->mm->mmap_sem);
284}
285
b0c632db 286/* Section: vm related */
a6e2f683
ED		 287static void sca_del_vcpu(struct kvm_vcpu *vcpu);
288
b0c632db
HC		 289/*
290 * Get (and clear) the dirty memory log for a memory slot.
291 */
292int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
293 struct kvm_dirty_log *log)
294{
15f36ebd
JH		 295 int r;
296 unsigned long n;
9f6b8029 297 struct kvm_memslots *slots;
15f36ebd
JH		 298 struct kvm_memory_slot *memslot;
299 int is_dirty = 0;
300
301 mutex_lock(&kvm->slots_lock);
302
303 r = -EINVAL;
304 if (log->slot >= KVM_USER_MEM_SLOTS)
305 goto out;
306
9f6b8029
PB		 307 slots = kvm_memslots(kvm);
308 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 309 r = -ENOENT;
310 if (!memslot->dirty_bitmap)
311 goto out;
312
313 kvm_s390_sync_dirty_log(kvm, memslot);
314 r = kvm_get_dirty_log(kvm, log, &is_dirty);
315 if (r)
316 goto out;
317
318 /* Clear the dirty log */
319 if (is_dirty) {
320 n = kvm_dirty_bitmap_bytes(memslot);
321 memset(memslot->dirty_bitmap, 0, n);
322 }
323 r = 0;
324out:
325 mutex_unlock(&kvm->slots_lock);
326 return r;
b0c632db
HC		 327}
328
d938dc55
CH		 329static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
330{
331 int r;
332
333 if (cap->flags)
334 return -EINVAL;
335
336 switch (cap->cap) {
84223598 337 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 338 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 339 kvm->arch.use_irqchip = 1;
340 r = 0;
341 break;
2444b352 342 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 343 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 344 kvm->arch.user_sigp = 1;
345 r = 0;
346 break;
68c55750 347 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 348 mutex_lock(&kvm->lock);
349 if (atomic_read(&kvm->online_vcpus)) {
350 r = -EBUSY;
351 } else if (MACHINE_HAS_VX) {
18280d8b
MM		 352 set_kvm_facility(kvm->arch.model.fac->mask, 129);
353 set_kvm_facility(kvm->arch.model.fac->list, 129);
354 r = 0;
355 } else
356 r = -EINVAL;
5967c17b 357 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 358 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
359 r ? "(not available)" : "(success)");
68c55750 360 break;
e44fc8c9 361 case KVM_CAP_S390_USER_STSI:
c92ea7b9 362 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 363 kvm->arch.user_stsi = 1;
364 r = 0;
365 break;
d938dc55
CH		 366 default:
367 r = -EINVAL;
368 break;
369 }
370 return r;
371}
372
8c0a7ce6
DD		 373static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
374{
375 int ret;
376
377 switch (attr->attr) {
378 case KVM_S390_VM_MEM_LIMIT_SIZE:
379 ret = 0;
c92ea7b9
CB		 380 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
381 kvm->arch.gmap->asce_end);
8c0a7ce6
DD		 382 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
383 ret = -EFAULT;
384 break;
385 default:
386 ret = -ENXIO;
387 break;
388 }
389 return ret;
390}
391
392static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 393{
394 int ret;
395 unsigned int idx;
396 switch (attr->attr) {
397 case KVM_S390_VM_MEM_ENABLE_CMMA:
e6db1d61
DD		 398 /* enable CMMA only for z10 and later (EDAT_1) */
399 ret = -EINVAL;
400 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
401 break;
402
4f718eab 403 ret = -EBUSY;
c92ea7b9 404 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 405 mutex_lock(&kvm->lock);
406 if (atomic_read(&kvm->online_vcpus) == 0) {
407 kvm->arch.use_cmma = 1;
408 ret = 0;
409 }
410 mutex_unlock(&kvm->lock);
411 break;
412 case KVM_S390_VM_MEM_CLR_CMMA:
c3489155
DD		 413 ret = -EINVAL;
414 if (!kvm->arch.use_cmma)
415 break;
416
c92ea7b9 417 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 418 mutex_lock(&kvm->lock);
419 idx = srcu_read_lock(&kvm->srcu);
a13cff31 420 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 421 srcu_read_unlock(&kvm->srcu, idx);
422 mutex_unlock(&kvm->lock);
423 ret = 0;
424 break;
8c0a7ce6
DD		 425 case KVM_S390_VM_MEM_LIMIT_SIZE: {
426 unsigned long new_limit;
427
428 if (kvm_is_ucontrol(kvm))
429 return -EINVAL;
430
431 if (get_user(new_limit, (u64 __user *)attr->addr))
432 return -EFAULT;
433
434 if (new_limit > kvm->arch.gmap->asce_end)
435 return -E2BIG;
436
437 ret = -EBUSY;
438 mutex_lock(&kvm->lock);
439 if (atomic_read(&kvm->online_vcpus) == 0) {
440 /* gmap_alloc will round the limit up */
441 struct gmap *new = gmap_alloc(current->mm, new_limit);
442
443 if (!new) {
444 ret = -ENOMEM;
445 } else {
446 gmap_free(kvm->arch.gmap);
447 new->private = kvm;
448 kvm->arch.gmap = new;
449 ret = 0;
450 }
451 }
452 mutex_unlock(&kvm->lock);
c92ea7b9 453 VM_EVENT(kvm, 3, "SET: max guest memory: %lu bytes", new_limit);
8c0a7ce6
DD		 454 break;
455 }
4f718eab
DD		 456 default:
457 ret = -ENXIO;
458 break;
459 }
460 return ret;
461}
462
a374e892
TK		 463static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
464
465static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
466{
467 struct kvm_vcpu *vcpu;
468 int i;
469
9d8d5786 470 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 471 return -EINVAL;
472
473 mutex_lock(&kvm->lock);
474 switch (attr->attr) {
475 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
476 get_random_bytes(
477 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
478 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
479 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 480 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 481 break;
482 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
483 get_random_bytes(
484 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
485 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
486 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 487 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 488 break;
489 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
490 kvm->arch.crypto.aes_kw = 0;
491 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
492 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 493 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 494 break;
495 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
496 kvm->arch.crypto.dea_kw = 0;
497 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
498 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 499 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 500 break;
501 default:
502 mutex_unlock(&kvm->lock);
503 return -ENXIO;
504 }
505
506 kvm_for_each_vcpu(i, vcpu, kvm) {
507 kvm_s390_vcpu_crypto_setup(vcpu);
508 exit_sie(vcpu);
509 }
510 mutex_unlock(&kvm->lock);
511 return 0;
512}
513
72f25020
JH		 514static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
515{
516 u8 gtod_high;
517
518 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
519 sizeof(gtod_high)))
520 return -EFAULT;
521
522 if (gtod_high != 0)
523 return -EINVAL;
58c383c6 524 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 525
526 return 0;
527}
528
529static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
530{
5a3d883a 531 u64 gtod;
72f25020
JH		 532
533 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
534 return -EFAULT;
535
25ed1675 536 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 537 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 538 return 0;
539}
540
541static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
542{
543 int ret;
544
545 if (attr->flags)
546 return -EINVAL;
547
548 switch (attr->attr) {
549 case KVM_S390_VM_TOD_HIGH:
550 ret = kvm_s390_set_tod_high(kvm, attr);
551 break;
552 case KVM_S390_VM_TOD_LOW:
553 ret = kvm_s390_set_tod_low(kvm, attr);
554 break;
555 default:
556 ret = -ENXIO;
557 break;
558 }
559 return ret;
560}
561
562static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
563{
564 u8 gtod_high = 0;
565
566 if (copy_to_user((void __user *)attr->addr, &gtod_high,
567 sizeof(gtod_high)))
568 return -EFAULT;
58c383c6 569 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 570
571 return 0;
572}
573
574static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
575{
5a3d883a 576 u64 gtod;
72f25020 577
60417fcc 578 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 579 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
580 return -EFAULT;
58c383c6 581 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 582
583 return 0;
584}
585
586static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
587{
588 int ret;
589
590 if (attr->flags)
591 return -EINVAL;
592
593 switch (attr->attr) {
594 case KVM_S390_VM_TOD_HIGH:
595 ret = kvm_s390_get_tod_high(kvm, attr);
596 break;
597 case KVM_S390_VM_TOD_LOW:
598 ret = kvm_s390_get_tod_low(kvm, attr);
599 break;
600 default:
601 ret = -ENXIO;
602 break;
603 }
604 return ret;
605}
606
658b6eda
MM		 607static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
608{
609 struct kvm_s390_vm_cpu_processor *proc;
610 int ret = 0;
611
612 mutex_lock(&kvm->lock);
613 if (atomic_read(&kvm->online_vcpus)) {
614 ret = -EBUSY;
615 goto out;
616 }
617 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
618 if (!proc) {
619 ret = -ENOMEM;
620 goto out;
621 }
622 if (!copy_from_user(proc, (void __user *)attr->addr,
623 sizeof(*proc))) {
624 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
625 sizeof(struct cpuid));
626 kvm->arch.model.ibc = proc->ibc;
981467c9 627 memcpy(kvm->arch.model.fac->list, proc->fac_list,
658b6eda
MM		 628 S390_ARCH_FAC_LIST_SIZE_BYTE);
629 } else
630 ret = -EFAULT;
631 kfree(proc);
632out:
633 mutex_unlock(&kvm->lock);
634 return ret;
635}
636
637static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
638{
639 int ret = -ENXIO;
640
641 switch (attr->attr) {
642 case KVM_S390_VM_CPU_PROCESSOR:
643 ret = kvm_s390_set_processor(kvm, attr);
644 break;
645 }
646 return ret;
647}
648
649static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
650{
651 struct kvm_s390_vm_cpu_processor *proc;
652 int ret = 0;
653
654 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
655 if (!proc) {
656 ret = -ENOMEM;
657 goto out;
658 }
659 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
660 proc->ibc = kvm->arch.model.ibc;
981467c9 661 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 662 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
663 ret = -EFAULT;
664 kfree(proc);
665out:
666 return ret;
667}
668
669static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
670{
671 struct kvm_s390_vm_cpu_machine *mach;
672 int ret = 0;
673
674 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
675 if (!mach) {
676 ret = -ENOMEM;
677 goto out;
678 }
679 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 680 mach->ibc = sclp.ibc;
981467c9
MM		 681 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
682 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 683 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 684 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 685 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
686 ret = -EFAULT;
687 kfree(mach);
688out:
689 return ret;
690}
691
692static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
693{
694 int ret = -ENXIO;
695
696 switch (attr->attr) {
697 case KVM_S390_VM_CPU_PROCESSOR:
698 ret = kvm_s390_get_processor(kvm, attr);
699 break;
700 case KVM_S390_VM_CPU_MACHINE:
701 ret = kvm_s390_get_machine(kvm, attr);
702 break;
703 }
704 return ret;
705}
706
f2061656
DD		 707static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
708{
709 int ret;
710
711 switch (attr->group) {
4f718eab 712 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 713 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 714 break;
72f25020
JH		 715 case KVM_S390_VM_TOD:
716 ret = kvm_s390_set_tod(kvm, attr);
717 break;
658b6eda
MM		 718 case KVM_S390_VM_CPU_MODEL:
719 ret = kvm_s390_set_cpu_model(kvm, attr);
720 break;
a374e892
TK		 721 case KVM_S390_VM_CRYPTO:
722 ret = kvm_s390_vm_set_crypto(kvm, attr);
723 break;
f2061656
DD		 724 default:
725 ret = -ENXIO;
726 break;
727 }
728
729 return ret;
730}
731
732static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
733{
8c0a7ce6
DD		 734 int ret;
735
736 switch (attr->group) {
737 case KVM_S390_VM_MEM_CTRL:
738 ret = kvm_s390_get_mem_control(kvm, attr);
739 break;
72f25020
JH		 740 case KVM_S390_VM_TOD:
741 ret = kvm_s390_get_tod(kvm, attr);
742 break;
658b6eda
MM		 743 case KVM_S390_VM_CPU_MODEL:
744 ret = kvm_s390_get_cpu_model(kvm, attr);
745 break;
8c0a7ce6
DD		 746 default:
747 ret = -ENXIO;
748 break;
749 }
750
751 return ret;
f2061656
DD		 752}
753
754static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
755{
756 int ret;
757
758 switch (attr->group) {
4f718eab
DD		 759 case KVM_S390_VM_MEM_CTRL:
760 switch (attr->attr) {
761 case KVM_S390_VM_MEM_ENABLE_CMMA:
762 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 763 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 764 ret = 0;
765 break;
766 default:
767 ret = -ENXIO;
768 break;
769 }
770 break;
72f25020
JH		 771 case KVM_S390_VM_TOD:
772 switch (attr->attr) {
773 case KVM_S390_VM_TOD_LOW:
774 case KVM_S390_VM_TOD_HIGH:
775 ret = 0;
776 break;
777 default:
778 ret = -ENXIO;
779 break;
780 }
781 break;
658b6eda
MM		 782 case KVM_S390_VM_CPU_MODEL:
783 switch (attr->attr) {
784 case KVM_S390_VM_CPU_PROCESSOR:
785 case KVM_S390_VM_CPU_MACHINE:
786 ret = 0;
787 break;
788 default:
789 ret = -ENXIO;
790 break;
791 }
792 break;
a374e892
TK		 793 case KVM_S390_VM_CRYPTO:
794 switch (attr->attr) {
795 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
796 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
797 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
798 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
799 ret = 0;
800 break;
801 default:
802 ret = -ENXIO;
803 break;
804 }
805 break;
f2061656
DD		 806 default:
807 ret = -ENXIO;
808 break;
809 }
810
811 return ret;
812}
813
30ee2a98
JH		 814static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
815{
816 uint8_t *keys;
817 uint64_t hva;
818 unsigned long curkey;
819 int i, r = 0;
820
821 if (args->flags != 0)
822 return -EINVAL;
823
824 /* Is this guest using storage keys? */
825 if (!mm_use_skey(current->mm))
826 return KVM_S390_GET_SKEYS_NONE;
827
828 /* Enforce sane limit on memory allocation */
829 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
830 return -EINVAL;
831
832 keys = kmalloc_array(args->count, sizeof(uint8_t),
833 GFP_KERNEL | __GFP_NOWARN);
834 if (!keys)
835 keys = vmalloc(sizeof(uint8_t) * args->count);
836 if (!keys)
837 return -ENOMEM;
838
839 for (i = 0; i < args->count; i++) {
840 hva = gfn_to_hva(kvm, args->start_gfn + i);
841 if (kvm_is_error_hva(hva)) {
842 r = -EFAULT;
843 goto out;
844 }
845
846 curkey = get_guest_storage_key(current->mm, hva);
847 if (IS_ERR_VALUE(curkey)) {
848 r = curkey;
849 goto out;
850 }
851 keys[i] = curkey;
852 }
853
854 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
855 sizeof(uint8_t) * args->count);
856 if (r)
857 r = -EFAULT;
858out:
859 kvfree(keys);
860 return r;
861}
862
863static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
864{
865 uint8_t *keys;
866 uint64_t hva;
867 int i, r = 0;
868
869 if (args->flags != 0)
870 return -EINVAL;
871
872 /* Enforce sane limit on memory allocation */
873 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
874 return -EINVAL;
875
876 keys = kmalloc_array(args->count, sizeof(uint8_t),
877 GFP_KERNEL | __GFP_NOWARN);
878 if (!keys)
879 keys = vmalloc(sizeof(uint8_t) * args->count);
880 if (!keys)
881 return -ENOMEM;
882
883 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
884 sizeof(uint8_t) * args->count);
885 if (r) {
886 r = -EFAULT;
887 goto out;
888 }
889
890 /* Enable storage key handling for the guest */
14d4a425
DD		 891 r = s390_enable_skey();
892 if (r)
893 goto out;
30ee2a98
JH		 894
895 for (i = 0; i < args->count; i++) {
896 hva = gfn_to_hva(kvm, args->start_gfn + i);
897 if (kvm_is_error_hva(hva)) {
898 r = -EFAULT;
899 goto out;
900 }
901
902 /* Lowest order bit is reserved */
903 if (keys[i] & 0x01) {
904 r = -EINVAL;
905 goto out;
906 }
907
908 r = set_guest_storage_key(current->mm, hva,
909 (unsigned long)keys[i], 0);
910 if (r)
911 goto out;
912 }
913out:
914 kvfree(keys);
915 return r;
916}
917
b0c632db
HC		 918long kvm_arch_vm_ioctl(struct file *filp,
919 unsigned int ioctl, unsigned long arg)
920{
921 struct kvm *kvm = filp->private_data;
922 void __user *argp = (void __user *)arg;
f2061656 923 struct kvm_device_attr attr;
b0c632db
HC		 924 int r;
925
926 switch (ioctl) {
ba5c1e9b
CO		 927 case KVM_S390_INTERRUPT: {
928 struct kvm_s390_interrupt s390int;
929
930 r = -EFAULT;
931 if (copy_from_user(&s390int, argp, sizeof(s390int)))
932 break;
933 r = kvm_s390_inject_vm(kvm, &s390int);
934 break;
935 }
d938dc55
CH		 936 case KVM_ENABLE_CAP: {
937 struct kvm_enable_cap cap;
938 r = -EFAULT;
939 if (copy_from_user(&cap, argp, sizeof(cap)))
940 break;
941 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
942 break;
943 }
84223598
CH		 944 case KVM_CREATE_IRQCHIP: {
945 struct kvm_irq_routing_entry routing;
946
947 r = -EINVAL;
948 if (kvm->arch.use_irqchip) {
949 /* Set up dummy routing. */
950 memset(&routing, 0, sizeof(routing));
152b2839 951 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 952 }
953 break;
954 }
f2061656
DD		 955 case KVM_SET_DEVICE_ATTR: {
956 r = -EFAULT;
957 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
958 break;
959 r = kvm_s390_vm_set_attr(kvm, &attr);
960 break;
961 }
962 case KVM_GET_DEVICE_ATTR: {
963 r = -EFAULT;
964 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
965 break;
966 r = kvm_s390_vm_get_attr(kvm, &attr);
967 break;
968 }
969 case KVM_HAS_DEVICE_ATTR: {
970 r = -EFAULT;
971 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
972 break;
973 r = kvm_s390_vm_has_attr(kvm, &attr);
974 break;
975 }
30ee2a98
JH		 976 case KVM_S390_GET_SKEYS: {
977 struct kvm_s390_skeys args;
978
979 r = -EFAULT;
980 if (copy_from_user(&args, argp,
981 sizeof(struct kvm_s390_skeys)))
982 break;
983 r = kvm_s390_get_skeys(kvm, &args);
984 break;
985 }
986 case KVM_S390_SET_SKEYS: {
987 struct kvm_s390_skeys args;
988
989 r = -EFAULT;
990 if (copy_from_user(&args, argp,
991 sizeof(struct kvm_s390_skeys)))
992 break;
993 r = kvm_s390_set_skeys(kvm, &args);
994 break;
995 }
b0c632db 996 default:
367e1319 997 r = -ENOTTY;
b0c632db
HC		 998 }
999
1000 return r;
1001}
1002
45c9b47c
TK		 1003static int kvm_s390_query_ap_config(u8 *config)
1004{
1005 u32 fcn_code = 0x04000000UL;
86044c8c 1006 u32 cc = 0;
45c9b47c 1007
86044c8c 1008 memset(config, 0, 128);
45c9b47c
TK		 1009 asm volatile(
1010 "lgr 0,%1\n"
1011 "lgr 2,%2\n"
1012 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1013 "0: ipm %0\n"
45c9b47c 1014 "srl %0,28\n"
86044c8c
CB		 1015 "1:\n"
1016 EX_TABLE(0b, 1b)
1017 : "+r" (cc)
45c9b47c
TK		 1018 : "r" (fcn_code), "r" (config)
1019 : "cc", "0", "2", "memory"
1020 );
1021
1022 return cc;
1023}
1024
1025static int kvm_s390_apxa_installed(void)
1026{
1027 u8 config[128];
1028 int cc;
1029
a6aacc3f 1030 if (test_facility(12)) {
45c9b47c
TK		 1031 cc = kvm_s390_query_ap_config(config);
1032
1033 if (cc)
1034 pr_err("PQAP(QCI) failed with cc=%d", cc);
1035 else
1036 return config[0] & 0x40;
1037 }
1038
1039 return 0;
1040}
1041
1042static void kvm_s390_set_crycb_format(struct kvm *kvm)
1043{
1044 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1045
1046 if (kvm_s390_apxa_installed())
1047 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1048 else
1049 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1050}
1051
9d8d5786
MM		 1052static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1053{
1054 get_cpu_id(cpu_id);
1055 cpu_id->version = 0xff;
1056}
1057
5102ee87
TK		 1058static int kvm_s390_crypto_init(struct kvm *kvm)
1059{
9d8d5786 1060 if (!test_kvm_facility(kvm, 76))
5102ee87
TK		 1061 return 0;
1062
1063 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1064 GFP_KERNEL | GFP_DMA);
1065 if (!kvm->arch.crypto.crycb)
1066 return -ENOMEM;
1067
45c9b47c 1068 kvm_s390_set_crycb_format(kvm);
5102ee87 1069
ed6f76b4
TK		 1070 /* Enable AES/DEA protected key functions by default */
1071 kvm->arch.crypto.aes_kw = 1;
1072 kvm->arch.crypto.dea_kw = 1;
1073 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1074 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1075 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1076 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
a374e892 1077
5102ee87
TK		 1078 return 0;
1079}
1080
7d43bafc
ED		 1081static void sca_dispose(struct kvm *kvm)
1082{
1083 if (kvm->arch.use_esca)
5e044315 1084 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1085 else
1086 free_page((unsigned long)(kvm->arch.sca));
1087 kvm->arch.sca = NULL;
1088}
1089
e08b9637 1090int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1091{
9d8d5786 1092 int i, rc;
b0c632db 1093 char debug_name[16];
f6c137ff 1094 static unsigned long sca_offset;
b0c632db 1095
e08b9637
CO		 1096 rc = -EINVAL;
1097#ifdef CONFIG_KVM_S390_UCONTROL
1098 if (type & ~KVM_VM_S390_UCONTROL)
1099 goto out_err;
1100 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1101 goto out_err;
1102#else
1103 if (type)
1104 goto out_err;
1105#endif
1106
b0c632db
HC		 1107 rc = s390_enable_sie();
1108 if (rc)
d89f5eff 1109 goto out_err;
b0c632db 1110
b290411a
CO		 1111 rc = -ENOMEM;
1112
7d43bafc 1113 kvm->arch.use_esca = 0; /* start with basic SCA */
5e044315 1114 rwlock_init(&kvm->arch.sca_lock);
bc784cce 1115 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
b0c632db 1116 if (!kvm->arch.sca)
d89f5eff 1117 goto out_err;
f6c137ff 1118 spin_lock(&kvm_lock);
c5c2c393 1119 sca_offset += 16;
bc784cce 1120 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1121 sca_offset = 0;
bc784cce
ED		 1122 kvm->arch.sca = (struct bsca_block *)
1123 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1124 spin_unlock(&kvm_lock);
b0c632db
HC		 1125
1126 sprintf(debug_name, "kvm-%u", current->pid);
1127
1cb9cf72 1128 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1129 if (!kvm->arch.dbf)
40f5b735 1130 goto out_err;
b0c632db 1131
9d8d5786
MM		 1132 /*
1133 * The architectural maximum amount of facilities is 16 kbit. To store
1134 * this amount, 2 kbyte of memory is required. Thus we need a full
981467c9
MM		 1135 * page to hold the guest facility list (arch.model.fac->list) and the
1136 * facility mask (arch.model.fac->mask). Its address size has to be
9d8d5786
MM		 1137 * 31 bits and word aligned.
1138 */
1139 kvm->arch.model.fac =
981467c9 1140 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
9d8d5786 1141 if (!kvm->arch.model.fac)
40f5b735 1142 goto out_err;
9d8d5786 1143
fb5bf93f 1144 /* Populate the facility mask initially. */
981467c9 1145 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
94422ee8 1146 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1147 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1148 if (i < kvm_s390_fac_list_mask_size())
981467c9 1149 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1150 else
981467c9 1151 kvm->arch.model.fac->mask[i] = 0UL;
9d8d5786
MM		 1152 }
1153
981467c9
MM		 1154 /* Populate the facility list initially. */
1155 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1156 S390_ARCH_FAC_LIST_SIZE_BYTE);
1157
9d8d5786 1158 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
37c5f6c8 1159 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1160
5102ee87 1161 if (kvm_s390_crypto_init(kvm) < 0)
40f5b735 1162 goto out_err;
5102ee87 1163
ba5c1e9b 1164 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1165 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1166 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1167 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1168 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1169
b0c632db 1170 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1171 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1172
e08b9637
CO		 1173 if (type & KVM_VM_S390_UCONTROL) {
1174 kvm->arch.gmap = NULL;
1175 } else {
0349985a 1176 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637 1177 if (!kvm->arch.gmap)
40f5b735 1178 goto out_err;
2c70fe44 1179 kvm->arch.gmap->private = kvm;
24eb3a82 1180 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1181 }
fa6b7fe9
CH		 1182
1183 kvm->arch.css_support = 0;
84223598 1184 kvm->arch.use_irqchip = 0;
72f25020 1185 kvm->arch.epoch = 0;
fa6b7fe9 1186
8ad35755 1187 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1188 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1189
d89f5eff 1190 return 0;
40f5b735 1191out_err:
5102ee87 1192 kfree(kvm->arch.crypto.crycb);
9d8d5786 1193 free_page((unsigned long)kvm->arch.model.fac);
598841ca 1194 debug_unregister(kvm->arch.dbf);
7d43bafc 1195 sca_dispose(kvm);
78f26131 1196 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1197 return rc;
b0c632db
HC		 1198}
1199
d329c035
CB		 1200void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1201{
1202 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1203 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1204 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1205 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1206 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1207 sca_del_vcpu(vcpu);
abf4a71e 1208 smp_mb();
27e0393f
CO		 1209
1210 if (kvm_is_ucontrol(vcpu->kvm))
1211 gmap_free(vcpu->arch.gmap);
1212
e6db1d61 1213 if (vcpu->kvm->arch.use_cmma)
b31605c1 1214 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1215 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1216
6692cef3 1217 kvm_vcpu_uninit(vcpu);
b110feaf 1218 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1219}
1220
1221static void kvm_free_vcpus(struct kvm *kvm)
1222{
1223 unsigned int i;
988a2cae 1224 struct kvm_vcpu *vcpu;
d329c035 1225
988a2cae
GN		 1226 kvm_for_each_vcpu(i, vcpu, kvm)
1227 kvm_arch_vcpu_destroy(vcpu);
1228
1229 mutex_lock(&kvm->lock);
1230 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1231 kvm->vcpus[i] = NULL;
1232
1233 atomic_set(&kvm->online_vcpus, 0);
1234 mutex_unlock(&kvm->lock);
d329c035
CB		 1235}
1236
b0c632db
HC		 1237void kvm_arch_destroy_vm(struct kvm *kvm)
1238{
d329c035 1239 kvm_free_vcpus(kvm);
9d8d5786 1240 free_page((unsigned long)kvm->arch.model.fac);
7d43bafc 1241 sca_dispose(kvm);
d329c035 1242 debug_unregister(kvm->arch.dbf);
5102ee87 1243 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 1244 if (!kvm_is_ucontrol(kvm))
1245 gmap_free(kvm->arch.gmap);
841b91c5 1246 kvm_s390_destroy_adapters(kvm);
67335e63 1247 kvm_s390_clear_float_irqs(kvm);
8335713a 1248 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1249}
1250
1251/* Section: vcpu related */
dafd032a
DD		 1252static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1253{
1254 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1255 if (!vcpu->arch.gmap)
1256 return -ENOMEM;
1257 vcpu->arch.gmap->private = vcpu->kvm;
1258
1259 return 0;
1260}
1261
a6e2f683
ED		 1262static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1263{
5e044315 1264 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1265 if (vcpu->kvm->arch.use_esca) {
1266 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1267
7d43bafc 1268 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1269 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1270 } else {
1271 struct bsca_block *sca = vcpu->kvm->arch.sca;
1272
1273 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1274 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1275 }
5e044315 1276 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1277}
1278
eaa78f34 1279static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1280{
eaa78f34
DH		 1281 read_lock(&vcpu->kvm->arch.sca_lock);
1282 if (vcpu->kvm->arch.use_esca) {
1283 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1284
eaa78f34 1285 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1286 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1287 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1288 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1289 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1290 } else {
eaa78f34 1291 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1292
eaa78f34 1293 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1294 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1295 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1296 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1297 }
eaa78f34 1298 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1299}
1300
1301/* Basic SCA to Extended SCA data copy routines */
1302static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1303{
1304 d->sda = s->sda;
1305 d->sigp_ctrl.c = s->sigp_ctrl.c;
1306 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1307}
1308
1309static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1310{
1311 int i;
1312
1313 d->ipte_control = s->ipte_control;
1314 d->mcn[0] = s->mcn;
1315 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1316 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1317}
1318
1319static int sca_switch_to_extended(struct kvm *kvm)
1320{
1321 struct bsca_block *old_sca = kvm->arch.sca;
1322 struct esca_block *new_sca;
1323 struct kvm_vcpu *vcpu;
1324 unsigned int vcpu_idx;
1325 u32 scaol, scaoh;
1326
1327 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1328 if (!new_sca)
1329 return -ENOMEM;
1330
1331 scaoh = (u32)((u64)(new_sca) >> 32);
1332 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1333
1334 kvm_s390_vcpu_block_all(kvm);
1335 write_lock(&kvm->arch.sca_lock);
1336
1337 sca_copy_b_to_e(new_sca, old_sca);
1338
1339 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1340 vcpu->arch.sie_block->scaoh = scaoh;
1341 vcpu->arch.sie_block->scaol = scaol;
1342 vcpu->arch.sie_block->ecb2 |= 0x04U;
1343 }
1344 kvm->arch.sca = new_sca;
1345 kvm->arch.use_esca = 1;
1346
1347 write_unlock(&kvm->arch.sca_lock);
1348 kvm_s390_vcpu_unblock_all(kvm);
1349
1350 free_page((unsigned long)old_sca);
1351
8335713a
CB		 1352 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1353 old_sca, kvm->arch.sca);
5e044315 1354 return 0;
a6e2f683
ED		 1355}
1356
1357static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1358{
5e044315
ED		 1359 int rc;
1360
1361 if (id < KVM_S390_BSCA_CPU_SLOTS)
1362 return true;
1363 if (!sclp.has_esca)
1364 return false;
1365
1366 mutex_lock(&kvm->lock);
1367 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1368 mutex_unlock(&kvm->lock);
1369
1370 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1371}
1372
b0c632db
HC		 1373int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1374{
3c038e6b
DD		 1375 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1376 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1377 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1378 KVM_SYNC_GPRS |
9eed0735 1379 KVM_SYNC_ACRS |
b028ee3e
DH		 1380 KVM_SYNC_CRS |
1381 KVM_SYNC_ARCH0 |
1382 KVM_SYNC_PFAULT;
68c55750
EF		 1383 if (test_kvm_facility(vcpu->kvm, 129))
1384 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
dafd032a
DD		 1385
1386 if (kvm_is_ucontrol(vcpu->kvm))
1387 return __kvm_ucontrol_vcpu_init(vcpu);
1388
b0c632db
HC		 1389 return 0;
1390}
1391
9977e886
HB		 1392/*
1393 * Backs up the current FP/VX register save area on a particular
1394 * destination. Used to switch between different register save
1395 * areas.
1396 */
1397static inline void save_fpu_to(struct fpu *dst)
1398{
1399 dst->fpc = current->thread.fpu.fpc;
9977e886
HB		 1400 dst->regs = current->thread.fpu.regs;
1401}
1402
1403/*
1404 * Switches the FP/VX register save area from which to lazy
1405 * restore register contents.
1406 */
1407static inline void load_fpu_from(struct fpu *from)
1408{
1409 current->thread.fpu.fpc = from->fpc;
9977e886
HB		 1410 current->thread.fpu.regs = from->regs;
1411}
1412
b0c632db
HC		 1413void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1414{
9977e886 1415 /* Save host register state */
d0164ee2 1416 save_fpu_regs();
9977e886 1417 save_fpu_to(&vcpu->arch.host_fpregs);
96b2d7a8 1418
18280d8b 1419 if (test_kvm_facility(vcpu->kvm, 129)) {
9977e886 1420 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886
HB		 1421 /*
1422 * Use the register save area in the SIE-control block
1423 * for register restore and save in kvm_arch_vcpu_put()
1424 */
1425 current->thread.fpu.vxrs =
1426 (__vector128 *)&vcpu->run->s.regs.vrs;
9977e886
HB		 1427 } else
1428 load_fpu_from(&vcpu->arch.guest_fpregs);
1429
1430 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1431 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1432 current->thread.fpu.fpc = 0;
1433
1434 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1435 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1436 gmap_enable(vcpu->arch.gmap);
805de8f4 1437 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 1438}
1439
1440void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1441{
805de8f4 1442 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1443 gmap_disable(vcpu->arch.gmap);
9977e886 1444
d0164ee2 1445 save_fpu_regs();
9977e886 1446
18280d8b 1447 if (test_kvm_facility(vcpu->kvm, 129))
9977e886
HB		 1448 /*
1449 * kvm_arch_vcpu_load() set up the register save area to
1450 * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1451 * are already saved. Only the floating-point control must be
1452 * copied.
1453 */
1454 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
68c55750 1455 else
9977e886
HB		 1456 save_fpu_to(&vcpu->arch.guest_fpregs);
1457 load_fpu_from(&vcpu->arch.host_fpregs);
1458
1459 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1460 restore_access_regs(vcpu->arch.host_acrs);
1461}
1462
1463static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1464{
1465 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1466 vcpu->arch.sie_block->gpsw.mask = 0UL;
1467 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1468 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 1469 vcpu->arch.sie_block->cputm = 0UL;
1470 vcpu->arch.sie_block->ckc = 0UL;
1471 vcpu->arch.sie_block->todpr = 0;
1472 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1473 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1474 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1475 vcpu->arch.guest_fpregs.fpc = 0;
1476 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1477 vcpu->arch.sie_block->gbea = 1;
672550fb 1478 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1479 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1480 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1481 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1482 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1483 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1484}
1485
31928aa5 1486void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1487{
72f25020 1488 mutex_lock(&vcpu->kvm->lock);
fdf03650 1489 preempt_disable();
72f25020 1490 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1491 preempt_enable();
72f25020 1492 mutex_unlock(&vcpu->kvm->lock);
25508824 1493 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1494 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1495 sca_add_vcpu(vcpu);
25508824
DH		 1496 }
1497
42897d86
MT		 1498}
1499
5102ee87
TK		 1500static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1501{
9d8d5786 1502 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1503 return;
1504
a374e892
TK		 1505 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1506
1507 if (vcpu->kvm->arch.crypto.aes_kw)
1508 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1509 if (vcpu->kvm->arch.crypto.dea_kw)
1510 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1511
5102ee87
TK		 1512 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1513}
1514
b31605c1
DD		 1515void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1516{
1517 free_page(vcpu->arch.sie_block->cbrlo);
1518 vcpu->arch.sie_block->cbrlo = 0;
1519}
1520
1521int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1522{
1523 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1524 if (!vcpu->arch.sie_block->cbrlo)
1525 return -ENOMEM;
1526
1527 vcpu->arch.sie_block->ecb2 |= 0x80;
1528 vcpu->arch.sie_block->ecb2 &= ~0x08;
1529 return 0;
1530}
1531
91520f1a
MM		 1532static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1533{
1534 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1535
1536 vcpu->arch.cpu_id = model->cpu_id;
1537 vcpu->arch.sie_block->ibc = model->ibc;
1538 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1539}
1540
b0c632db
HC		 1541int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1542{
b31605c1 1543 int rc = 0;
b31288fa 1544
9e6dabef
CH		 1545 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1546 CPUSTAT_SM |
a4a4f191
GH		 1547 CPUSTAT_STOPPED);
1548
53df84f8 1549 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1550 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1551 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1552 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1553
91520f1a
MM		 1554 kvm_s390_vcpu_setup_model(vcpu);
1555
fc34531d 1556 vcpu->arch.sie_block->ecb = 6;
9d8d5786 1557 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1558 vcpu->arch.sie_block->ecb |= 0x10;
1559
69d0d3a3 1560 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 1561 vcpu->arch.sie_block->eca = 0xC1002000U;
37c5f6c8 1562 if (sclp.has_siif)
217a4406 1563 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1564 if (sclp.has_sigpif)
ea5f4969 1565 vcpu->arch.sie_block->eca |= 0x10000000U;
18280d8b 1566 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1567 vcpu->arch.sie_block->eca |= 0x00020000;
1568 vcpu->arch.sie_block->ecd |= 0x20000000;
1569 }
492d8642 1570 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1571
e6db1d61 1572 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1573 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1574 if (rc)
1575 return rc;
b31288fa 1576 }
0ac96caf 1577 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1578 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1579
5102ee87
TK		 1580 kvm_s390_vcpu_crypto_setup(vcpu);
1581
b31605c1 1582 return rc;
b0c632db
HC		 1583}
1584
1585struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1586 unsigned int id)
1587{
4d47555a 1588 struct kvm_vcpu *vcpu;
7feb6bb8 1589 struct sie_page *sie_page;
4d47555a
CO		 1590 int rc = -EINVAL;
1591
4215825e 1592 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1593 goto out;
1594
1595 rc = -ENOMEM;
b0c632db 1596
b110feaf 1597 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1598 if (!vcpu)
4d47555a 1599 goto out;
b0c632db 1600
7feb6bb8
MM		 1601 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1602 if (!sie_page)
b0c632db
HC		 1603 goto out_free_cpu;
1604
7feb6bb8
MM		 1605 vcpu->arch.sie_block = &sie_page->sie_block;
1606 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1607
b0c632db 1608 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1609 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1610 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1611 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1612 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 1613
9977e886
HB		 1614 /*
1615 * Allocate a save area for floating-point registers. If the vector
1616 * extension is available, register contents are saved in the SIE
1617 * control block. The allocated save area is still required in
1618 * particular places, for example, in kvm_s390_vcpu_store_status().
1619 */
1620 vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1621 GFP_KERNEL);
2f8a43d4 1622 if (!vcpu->arch.guest_fpregs.fprs)
9977e886 1623 goto out_free_sie_block;
9977e886 1624
b0c632db
HC		 1625 rc = kvm_vcpu_init(vcpu, kvm, id);
1626 if (rc)
7b06bf2f 1627 goto out_free_sie_block;
8335713a 1628 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1629 vcpu->arch.sie_block);
ade38c31 1630 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1631
b0c632db 1632 return vcpu;
7b06bf2f
WY		 1633out_free_sie_block:
1634 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1635out_free_cpu:
b110feaf 1636 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1637out:
b0c632db
HC		 1638 return ERR_PTR(rc);
1639}
1640
b0c632db
HC		 1641int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1642{
9a022067 1643 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1644}
1645
27406cd5 1646void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1647{
805de8f4 1648 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1649 exit_sie(vcpu);
49b99e1e
CB		 1650}
1651
27406cd5 1652void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1653{
805de8f4 1654 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1655}
1656
8e236546
CB		 1657static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1658{
805de8f4 1659 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1660 exit_sie(vcpu);
8e236546
CB		 1661}
1662
1663static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1664{
9bf9fde2 1665 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1666}
1667
49b99e1e
CB		 1668/*
1669 * Kick a guest cpu out of SIE and wait until SIE is not running.
1670 * If the CPU is not running (e.g. waiting as idle) the function will
1671 * return immediately. */
1672void exit_sie(struct kvm_vcpu *vcpu)
1673{
805de8f4 1674 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1675 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1676 cpu_relax();
1677}
1678
8e236546
CB		 1679/* Kick a guest cpu out of SIE to process a request synchronously */
1680void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1681{
8e236546
CB		 1682 kvm_make_request(req, vcpu);
1683 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1684}
1685
2c70fe44
CB		 1686static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1687{
1688 int i;
1689 struct kvm *kvm = gmap->private;
1690 struct kvm_vcpu *vcpu;
1691
1692 kvm_for_each_vcpu(i, vcpu, kvm) {
1693 /* match against both prefix pages */
fda902cb 1694 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1695 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1696 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1697 }
1698 }
1699}
1700
b6d33834
CD		 1701int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1702{
1703 /* kvm common code refers to this, but never calls it */
1704 BUG();
1705 return 0;
1706}
1707
14eebd91
CO		 1708static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1709 struct kvm_one_reg *reg)
1710{
1711 int r = -EINVAL;
1712
1713 switch (reg->id) {
29b7c71b
CO		 1714 case KVM_REG_S390_TODPR:
1715 r = put_user(vcpu->arch.sie_block->todpr,
1716 (u32 __user *)reg->addr);
1717 break;
1718 case KVM_REG_S390_EPOCHDIFF:
1719 r = put_user(vcpu->arch.sie_block->epoch,
1720 (u64 __user *)reg->addr);
1721 break;
46a6dd1c
J		 1722 case KVM_REG_S390_CPU_TIMER:
1723 r = put_user(vcpu->arch.sie_block->cputm,
1724 (u64 __user *)reg->addr);
1725 break;
1726 case KVM_REG_S390_CLOCK_COMP:
1727 r = put_user(vcpu->arch.sie_block->ckc,
1728 (u64 __user *)reg->addr);
1729 break;
536336c2
DD		 1730 case KVM_REG_S390_PFTOKEN:
1731 r = put_user(vcpu->arch.pfault_token,
1732 (u64 __user *)reg->addr);
1733 break;
1734 case KVM_REG_S390_PFCOMPARE:
1735 r = put_user(vcpu->arch.pfault_compare,
1736 (u64 __user *)reg->addr);
1737 break;
1738 case KVM_REG_S390_PFSELECT:
1739 r = put_user(vcpu->arch.pfault_select,
1740 (u64 __user *)reg->addr);
1741 break;
672550fb
CB		 1742 case KVM_REG_S390_PP:
1743 r = put_user(vcpu->arch.sie_block->pp,
1744 (u64 __user *)reg->addr);
1745 break;
afa45ff5
CB		 1746 case KVM_REG_S390_GBEA:
1747 r = put_user(vcpu->arch.sie_block->gbea,
1748 (u64 __user *)reg->addr);
1749 break;
14eebd91
CO		 1750 default:
1751 break;
1752 }
1753
1754 return r;
1755}
1756
1757static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1758 struct kvm_one_reg *reg)
1759{
1760 int r = -EINVAL;
1761
1762 switch (reg->id) {
29b7c71b
CO		 1763 case KVM_REG_S390_TODPR:
1764 r = get_user(vcpu->arch.sie_block->todpr,
1765 (u32 __user *)reg->addr);
1766 break;
1767 case KVM_REG_S390_EPOCHDIFF:
1768 r = get_user(vcpu->arch.sie_block->epoch,
1769 (u64 __user *)reg->addr);
1770 break;
46a6dd1c
J		 1771 case KVM_REG_S390_CPU_TIMER:
1772 r = get_user(vcpu->arch.sie_block->cputm,
1773 (u64 __user *)reg->addr);
1774 break;
1775 case KVM_REG_S390_CLOCK_COMP:
1776 r = get_user(vcpu->arch.sie_block->ckc,
1777 (u64 __user *)reg->addr);
1778 break;
536336c2
DD		 1779 case KVM_REG_S390_PFTOKEN:
1780 r = get_user(vcpu->arch.pfault_token,
1781 (u64 __user *)reg->addr);
9fbd8082
DH		 1782 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1783 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1784 break;
1785 case KVM_REG_S390_PFCOMPARE:
1786 r = get_user(vcpu->arch.pfault_compare,
1787 (u64 __user *)reg->addr);
1788 break;
1789 case KVM_REG_S390_PFSELECT:
1790 r = get_user(vcpu->arch.pfault_select,
1791 (u64 __user *)reg->addr);
1792 break;
672550fb
CB		 1793 case KVM_REG_S390_PP:
1794 r = get_user(vcpu->arch.sie_block->pp,
1795 (u64 __user *)reg->addr);
1796 break;
afa45ff5
CB		 1797 case KVM_REG_S390_GBEA:
1798 r = get_user(vcpu->arch.sie_block->gbea,
1799 (u64 __user *)reg->addr);
1800 break;
14eebd91
CO		 1801 default:
1802 break;
1803 }
1804
1805 return r;
1806}
b6d33834 1807
b0c632db
HC		 1808static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1809{
b0c632db 1810 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1811 return 0;
1812}
1813
1814int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1815{
5a32c1af 1816 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1817 return 0;
1818}
1819
1820int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1821{
5a32c1af 1822 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1823 return 0;
1824}
1825
1826int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1827 struct kvm_sregs *sregs)
1828{
59674c1a 1829 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1830 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1831 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1832 return 0;
1833}
1834
1835int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1836 struct kvm_sregs *sregs)
1837{
59674c1a 1838 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1839 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1840 return 0;
1841}
1842
1843int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1844{
4725c860
MS		 1845 if (test_fp_ctl(fpu->fpc))
1846 return -EINVAL;
9977e886 1847 memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860 1848 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
d0164ee2 1849 save_fpu_regs();
9977e886 1850 load_fpu_from(&vcpu->arch.guest_fpregs);
b0c632db
HC		 1851 return 0;
1852}
1853
1854int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1855{
9977e886 1856 memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
b0c632db 1857 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1858 return 0;
1859}
1860
1861static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1862{
1863 int rc = 0;
1864
7a42fdc2 1865 if (!is_vcpu_stopped(vcpu))
b0c632db 1866 rc = -EBUSY;
d7b0b5eb
CO		 1867 else {
1868 vcpu->run->psw_mask = psw.mask;
1869 vcpu->run->psw_addr = psw.addr;
1870 }
b0c632db
HC		 1871 return rc;
1872}
1873
1874int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1875 struct kvm_translation *tr)
1876{
1877 return -EINVAL; /* not implemented yet */
1878}
1879
27291e21
DH		 1880#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1881 KVM_GUESTDBG_USE_HW_BP | \
1882 KVM_GUESTDBG_ENABLE)
1883
d0bfb940
JK		 1884int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1885 struct kvm_guest_debug *dbg)
b0c632db 1886{
27291e21
DH		 1887 int rc = 0;
1888
1889 vcpu->guest_debug = 0;
1890 kvm_s390_clear_bp_data(vcpu);
1891
2de3bfc2 1892 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1893 return -EINVAL;
1894
1895 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1896 vcpu->guest_debug = dbg->control;
1897 /* enforce guest PER */
805de8f4 1898 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1899
1900 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1901 rc = kvm_s390_import_bp_data(vcpu, dbg);
1902 } else {
805de8f4 1903 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1904 vcpu->arch.guestdbg.last_bp = 0;
1905 }
1906
1907 if (rc) {
1908 vcpu->guest_debug = 0;
1909 kvm_s390_clear_bp_data(vcpu);
805de8f4 1910 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 1911 }
1912
1913 return rc;
b0c632db
HC		 1914}
1915
62d9f0db
MT		 1916int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1917 struct kvm_mp_state *mp_state)
1918{
6352e4d2
DH		 1919 /* CHECK_STOP and LOAD are not supported yet */
1920 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1921 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1922}
1923
1924int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1925 struct kvm_mp_state *mp_state)
1926{
6352e4d2
DH		 1927 int rc = 0;
1928
1929 /* user space knows about this interface - let it control the state */
1930 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1931
1932 switch (mp_state->mp_state) {
1933 case KVM_MP_STATE_STOPPED:
1934 kvm_s390_vcpu_stop(vcpu);
1935 break;
1936 case KVM_MP_STATE_OPERATING:
1937 kvm_s390_vcpu_start(vcpu);
1938 break;
1939 case KVM_MP_STATE_LOAD:
1940 case KVM_MP_STATE_CHECK_STOP:
1941 /* fall through - CHECK_STOP and LOAD are not supported yet */
1942 default:
1943 rc = -ENXIO;
1944 }
1945
1946 return rc;
62d9f0db
MT		 1947}
1948
8ad35755
DH		 1949static bool ibs_enabled(struct kvm_vcpu *vcpu)
1950{
1951 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1952}
1953
2c70fe44
CB		 1954static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1955{
8ad35755 1956retry:
8e236546 1957 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 1958 if (!vcpu->requests)
1959 return 0;
2c70fe44
CB		 1960 /*
1961 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1962 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1963 * This ensures that the ipte instruction for this request has
1964 * already finished. We might race against a second unmapper that
1965 * wants to set the blocking bit. Lets just retry the request loop.
1966 */
8ad35755 1967 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1968 int rc;
1969 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1970 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1971 PAGE_SIZE * 2);
1972 if (rc)
1973 return rc;
8ad35755 1974 goto retry;
2c70fe44 1975 }
8ad35755 1976
d3d692c8
DH		 1977 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1978 vcpu->arch.sie_block->ihcpu = 0xffff;
1979 goto retry;
1980 }
1981
8ad35755
DH		 1982 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1983 if (!ibs_enabled(vcpu)) {
1984 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 1985 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 1986 &vcpu->arch.sie_block->cpuflags);
1987 }
1988 goto retry;
2c70fe44 1989 }
8ad35755
DH		 1990
1991 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1992 if (ibs_enabled(vcpu)) {
1993 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 1994 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 1995 &vcpu->arch.sie_block->cpuflags);
1996 }
1997 goto retry;
1998 }
1999
0759d068
DH		 2000 /* nothing to do, just clear the request */
2001 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2002
2c70fe44
CB		 2003 return 0;
2004}
2005
25ed1675
DH		 2006void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2007{
2008 struct kvm_vcpu *vcpu;
2009 int i;
2010
2011 mutex_lock(&kvm->lock);
2012 preempt_disable();
2013 kvm->arch.epoch = tod - get_tod_clock();
2014 kvm_s390_vcpu_block_all(kvm);
2015 kvm_for_each_vcpu(i, vcpu, kvm)
2016 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2017 kvm_s390_vcpu_unblock_all(kvm);
2018 preempt_enable();
2019 mutex_unlock(&kvm->lock);
2020}
2021
fa576c58
TH		 2022/**
2023 * kvm_arch_fault_in_page - fault-in guest page if necessary
2024 * @vcpu: The corresponding virtual cpu
2025 * @gpa: Guest physical address
2026 * @writable: Whether the page should be writable or not
2027 *
2028 * Make sure that a guest page has been faulted-in on the host.
2029 *
2030 * Return: Zero on success, negative error code otherwise.
2031 */
2032long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2033{
527e30b4
MS		 2034 return gmap_fault(vcpu->arch.gmap, gpa,
2035 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2036}
2037
3c038e6b
DD		 2038static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2039 unsigned long token)
2040{
2041 struct kvm_s390_interrupt inti;
383d0b05 2042 struct kvm_s390_irq irq;
3c038e6b
DD		 2043
2044 if (start_token) {
383d0b05
JF		 2045 irq.u.ext.ext_params2 = token;
2046 irq.type = KVM_S390_INT_PFAULT_INIT;
2047 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2048 } else {
2049 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2050 inti.parm64 = token;
3c038e6b
DD		 2051 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2052 }
2053}
2054
2055void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2056 struct kvm_async_pf *work)
2057{
2058 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2059 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2060}
2061
2062void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2063 struct kvm_async_pf *work)
2064{
2065 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2066 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2067}
2068
2069void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2070 struct kvm_async_pf *work)
2071{
2072 /* s390 will always inject the page directly */
2073}
2074
2075bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2076{
2077 /*
2078 * s390 will always inject the page directly,
2079 * but we still want check_async_completion to cleanup
2080 */
2081 return true;
2082}
2083
2084static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2085{
2086 hva_t hva;
2087 struct kvm_arch_async_pf arch;
2088 int rc;
2089
2090 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2091 return 0;
2092 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2093 vcpu->arch.pfault_compare)
2094 return 0;
2095 if (psw_extint_disabled(vcpu))
2096 return 0;
9a022067 2097 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2098 return 0;
2099 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2100 return 0;
2101 if (!vcpu->arch.gmap->pfault_enabled)
2102 return 0;
2103
81480cc1
HC		 2104 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2105 hva += current->thread.gmap_addr & ~PAGE_MASK;
2106 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2107 return 0;
2108
2109 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2110 return rc;
2111}
2112
3fb4c40f 2113static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2114{
3fb4c40f 2115 int rc, cpuflags;
e168bf8d 2116
3c038e6b
DD		 2117 /*
2118 * On s390 notifications for arriving pages will be delivered directly
2119 * to the guest but the house keeping for completed pfaults is
2120 * handled outside the worker.
2121 */
2122 kvm_check_async_pf_completion(vcpu);
2123
7ec7c8c7
CB		 2124 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2125 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2126
2127 if (need_resched())
2128 schedule();
2129
d3a73acb 2130 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2131 s390_handle_mcck();
2132
79395031
JF		 2133 if (!kvm_is_ucontrol(vcpu->kvm)) {
2134 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2135 if (rc)
2136 return rc;
2137 }
0ff31867 2138
2c70fe44
CB		 2139 rc = kvm_s390_handle_requests(vcpu);
2140 if (rc)
2141 return rc;
2142
27291e21
DH		 2143 if (guestdbg_enabled(vcpu)) {
2144 kvm_s390_backup_guest_per_regs(vcpu);
2145 kvm_s390_patch_guest_per_regs(vcpu);
2146 }
2147
b0c632db 2148 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2149 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2150 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2151 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2152
3fb4c40f
TH		 2153 return 0;
2154}
2155
492d8642
TH		 2156static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2157{
2158 psw_t *psw = &vcpu->arch.sie_block->gpsw;
2159 u8 opcode;
2160 int rc;
2161
2162 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2163 trace_kvm_s390_sie_fault(vcpu);
2164
2165 /*
2166 * We want to inject an addressing exception, which is defined as a
2167 * suppressing or terminating exception. However, since we came here
2168 * by a DAT access exception, the PSW still points to the faulting
2169 * instruction since DAT exceptions are nullifying. So we've got
2170 * to look up the current opcode to get the length of the instruction
2171 * to be able to forward the PSW.
2172 */
8ae04b8f 2173 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
492d8642
TH		 2174 if (rc)
2175 return kvm_s390_inject_prog_cond(vcpu, rc);
2176 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2177
2178 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2179}
2180
3fb4c40f
TH		 2181static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2182{
2b29a9fd
DD		 2183 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2184 vcpu->arch.sie_block->icptcode);
2185 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2186
27291e21
DH		 2187 if (guestdbg_enabled(vcpu))
2188 kvm_s390_restore_guest_per_regs(vcpu);
2189
7ec7c8c7
CB		 2190 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2191 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2192
2193 if (vcpu->arch.sie_block->icptcode > 0) {
2194 int rc = kvm_handle_sie_intercept(vcpu);
2195
2196 if (rc != -EOPNOTSUPP)
2197 return rc;
2198 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2199 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2200 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2201 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2202 return -EREMOTE;
2203 } else if (exit_reason != -EFAULT) {
2204 vcpu->stat.exit_null++;
2205 return 0;
210b1607
TH		 2206 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2207 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2208 vcpu->run->s390_ucontrol.trans_exc_code =
2209 current->thread.gmap_addr;
2210 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2211 return -EREMOTE;
24eb3a82 2212 } else if (current->thread.gmap_pfault) {
3c038e6b 2213 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2214 current->thread.gmap_pfault = 0;
71f116bf
DH		 2215 if (kvm_arch_setup_async_pf(vcpu))
2216 return 0;
2217 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2218 }
71f116bf 2219 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2220}
2221
2222static int __vcpu_run(struct kvm_vcpu *vcpu)
2223{
2224 int rc, exit_reason;
2225
800c1065
TH		 2226 /*
2227 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2228 * ning the guest), so that memslots (and other stuff) are protected
2229 */
2230 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2231
a76ccff6
TH		 2232 do {
2233 rc = vcpu_pre_run(vcpu);
2234 if (rc)
2235 break;
3fb4c40f 2236
800c1065 2237 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2238 /*
2239 * As PF_VCPU will be used in fault handler, between
2240 * guest_enter and guest_exit should be no uaccess.
2241 */
0097d12e
CB		 2242 local_irq_disable();
2243 __kvm_guest_enter();
2244 local_irq_enable();
a76ccff6
TH		 2245 exit_reason = sie64a(vcpu->arch.sie_block,
2246 vcpu->run->s.regs.gprs);
0097d12e
CB		 2247 local_irq_disable();
2248 __kvm_guest_exit();
2249 local_irq_enable();
800c1065 2250 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2251
2252 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2253 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2254
800c1065 2255 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2256 return rc;
b0c632db
HC		 2257}
2258
b028ee3e
DH		 2259static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2260{
2261 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2262 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2263 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2264 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2265 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2266 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2267 /* some control register changes require a tlb flush */
2268 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2269 }
2270 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2271 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2272 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2273 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2274 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2275 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2276 }
2277 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2278 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2279 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2280 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2281 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2282 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2283 }
2284 kvm_run->kvm_dirty_regs = 0;
2285}
2286
2287static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2288{
2289 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2290 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2291 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2292 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2293 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2294 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2295 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2296 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2297 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2298 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2299 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2300 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2301}
2302
b0c632db
HC		 2303int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2304{
8f2abe6a 2305 int rc;
b0c632db
HC		 2306 sigset_t sigsaved;
2307
27291e21
DH		 2308 if (guestdbg_exit_pending(vcpu)) {
2309 kvm_s390_prepare_debug_exit(vcpu);
2310 return 0;
2311 }
2312
b0c632db
HC		 2313 if (vcpu->sigset_active)
2314 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2315
6352e4d2
DH		 2316 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2317 kvm_s390_vcpu_start(vcpu);
2318 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2319 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2320 vcpu->vcpu_id);
2321 return -EINVAL;
2322 }
b0c632db 2323
b028ee3e 2324 sync_regs(vcpu, kvm_run);
d7b0b5eb 2325
dab4079d 2326 might_fault();
a76ccff6 2327 rc = __vcpu_run(vcpu);
9ace903d 2328
b1d16c49
CE		 2329 if (signal_pending(current) && !rc) {
2330 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2331 rc = -EINTR;
b1d16c49 2332 }
8f2abe6a 2333
27291e21
DH		 2334 if (guestdbg_exit_pending(vcpu) && !rc) {
2335 kvm_s390_prepare_debug_exit(vcpu);
2336 rc = 0;
2337 }
2338
8f2abe6a 2339 if (rc == -EREMOTE) {
71f116bf 2340 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2341 rc = 0;
2342 }
b0c632db 2343
b028ee3e 2344 store_regs(vcpu, kvm_run);
d7b0b5eb 2345
b0c632db
HC		 2346 if (vcpu->sigset_active)
2347 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2348
b0c632db 2349 vcpu->stat.exit_userspace++;
7e8e6ab4 2350 return rc;
b0c632db
HC		 2351}
2352
b0c632db
HC		 2353/*
2354 * store status at address
2355 * we use have two special cases:
2356 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2357 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2358 */
d0bce605 2359int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2360{
092670cd 2361 unsigned char archmode = 1;
fda902cb 2362 unsigned int px;
178bd789 2363 u64 clkcomp;
d0bce605 2364 int rc;
b0c632db 2365
d0bce605
HC		 2366 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2367 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2368 return -EFAULT;
d0bce605
HC		 2369 gpa = SAVE_AREA_BASE;
2370 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2371 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2372 return -EFAULT;
d0bce605
HC		 2373 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2374 }
2375 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2376 vcpu->arch.guest_fpregs.fprs, 128);
2377 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2378 vcpu->run->s.regs.gprs, 128);
2379 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2380 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 2381 px = kvm_s390_get_prefix(vcpu);
d0bce605 2382 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 2383 &px, 4);
d0bce605
HC		 2384 rc |= write_guest_abs(vcpu,
2385 gpa + offsetof(struct save_area, fp_ctrl_reg),
2386 &vcpu->arch.guest_fpregs.fpc, 4);
2387 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2388 &vcpu->arch.sie_block->todpr, 4);
2389 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2390 &vcpu->arch.sie_block->cputm, 8);
178bd789 2391 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 2392 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2393 &clkcomp, 8);
2394 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2395 &vcpu->run->s.regs.acrs, 64);
2396 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2397 &vcpu->arch.sie_block->gcr, 128);
2398 return rc ? -EFAULT : 0;
b0c632db
HC		 2399}
2400
e879892c
TH		 2401int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2402{
2403 /*
2404 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2405 * copying in vcpu load/put. Lets update our copies before we save
2406 * it into the save area
2407 */
d0164ee2 2408 save_fpu_regs();
9977e886
HB		 2409 if (test_kvm_facility(vcpu->kvm, 129)) {
2410 /*
2411 * If the vector extension is available, the vector registers
2412 * which overlaps with floating-point registers are saved in
2413 * the SIE-control block. Hence, extract the floating-point
2414 * registers and the FPC value and store them in the
2415 * guest_fpregs structure.
2416 */
9977e886
HB		 2417 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2418 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2419 current->thread.fpu.vxrs);
2420 } else
2421 save_fpu_to(&vcpu->arch.guest_fpregs);
e879892c
TH		 2422 save_access_regs(vcpu->run->s.regs.acrs);
2423
2424 return kvm_s390_store_status_unloaded(vcpu, addr);
2425}
2426
bc17de7c
EF		 2427/*
2428 * store additional status at address
2429 */
2430int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2431 unsigned long gpa)
2432{
2433 /* Only bits 0-53 are used for address formation */
2434 if (!(gpa & ~0x3ff))
2435 return 0;
2436
2437 return write_guest_abs(vcpu, gpa & ~0x3ff,
2438 (void *)&vcpu->run->s.regs.vrs, 512);
2439}
2440
2441int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2442{
2443 if (!test_kvm_facility(vcpu->kvm, 129))
2444 return 0;
2445
2446 /*
2447 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2448 * copying in vcpu load/put. We can simply call save_fpu_regs()
2449 * to save the current register state because we are in the
2450 * middle of a load/put cycle.
2451 *
2452 * Let's update our copies before we save it into the save area.
bc17de7c 2453 */
d0164ee2 2454 save_fpu_regs();
bc17de7c
EF		 2455
2456 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2457}
2458
8ad35755
DH		 2459static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2460{
2461 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2462 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2463}
2464
2465static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2466{
2467 unsigned int i;
2468 struct kvm_vcpu *vcpu;
2469
2470 kvm_for_each_vcpu(i, vcpu, kvm) {
2471 __disable_ibs_on_vcpu(vcpu);
2472 }
2473}
2474
2475static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2476{
2477 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2478 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2479}
2480
6852d7b6
DH		 2481void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2482{
8ad35755
DH		 2483 int i, online_vcpus, started_vcpus = 0;
2484
2485 if (!is_vcpu_stopped(vcpu))
2486 return;
2487
6852d7b6 2488 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2489 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2490 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2491 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2492
2493 for (i = 0; i < online_vcpus; i++) {
2494 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2495 started_vcpus++;
2496 }
2497
2498 if (started_vcpus == 0) {
2499 /* we're the only active VCPU -> speed it up */
2500 __enable_ibs_on_vcpu(vcpu);
2501 } else if (started_vcpus == 1) {
2502 /*
2503 * As we are starting a second VCPU, we have to disable
2504 * the IBS facility on all VCPUs to remove potentially
2505 * oustanding ENABLE requests.
2506 */
2507 __disable_ibs_on_all_vcpus(vcpu->kvm);
2508 }
2509
805de8f4 2510 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2511 /*
2512 * Another VCPU might have used IBS while we were offline.
2513 * Let's play safe and flush the VCPU at startup.
2514 */
d3d692c8 2515 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2516 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2517 return;
6852d7b6
DH		 2518}
2519
2520void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2521{
8ad35755
DH		 2522 int i, online_vcpus, started_vcpus = 0;
2523 struct kvm_vcpu *started_vcpu = NULL;
2524
2525 if (is_vcpu_stopped(vcpu))
2526 return;
2527
6852d7b6 2528 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2529 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2530 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2531 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2532
32f5ff63 2533 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2534 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2535
805de8f4 2536 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2537 __disable_ibs_on_vcpu(vcpu);
2538
2539 for (i = 0; i < online_vcpus; i++) {
2540 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2541 started_vcpus++;
2542 started_vcpu = vcpu->kvm->vcpus[i];
2543 }
2544 }
2545
2546 if (started_vcpus == 1) {
2547 /*
2548 * As we only have one VCPU left, we want to enable the
2549 * IBS facility for that VCPU to speed it up.
2550 */
2551 __enable_ibs_on_vcpu(started_vcpu);
2552 }
2553
433b9ee4 2554 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2555 return;
6852d7b6
DH		 2556}
2557
d6712df9
CH		 2558static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2559 struct kvm_enable_cap *cap)
2560{
2561 int r;
2562
2563 if (cap->flags)
2564 return -EINVAL;
2565
2566 switch (cap->cap) {
fa6b7fe9
CH		 2567 case KVM_CAP_S390_CSS_SUPPORT:
2568 if (!vcpu->kvm->arch.css_support) {
2569 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2570 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2571 trace_kvm_s390_enable_css(vcpu->kvm);
2572 }
2573 r = 0;
2574 break;
d6712df9
CH		 2575 default:
2576 r = -EINVAL;
2577 break;
2578 }
2579 return r;
2580}
2581
41408c28
TH		 2582static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2583 struct kvm_s390_mem_op *mop)
2584{
2585 void __user *uaddr = (void __user *)mop->buf;
2586 void *tmpbuf = NULL;
2587 int r, srcu_idx;
2588 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2589 | KVM_S390_MEMOP_F_CHECK_ONLY;
2590
2591 if (mop->flags & ~supported_flags)
2592 return -EINVAL;
2593
2594 if (mop->size > MEM_OP_MAX_SIZE)
2595 return -E2BIG;
2596
2597 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2598 tmpbuf = vmalloc(mop->size);
2599 if (!tmpbuf)
2600 return -ENOMEM;
2601 }
2602
2603 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2604
2605 switch (mop->op) {
2606 case KVM_S390_MEMOP_LOGICAL_READ:
2607 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2608 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2609 break;
2610 }
2611 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2612 if (r == 0) {
2613 if (copy_to_user(uaddr, tmpbuf, mop->size))
2614 r = -EFAULT;
2615 }
2616 break;
2617 case KVM_S390_MEMOP_LOGICAL_WRITE:
2618 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2619 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2620 break;
2621 }
2622 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2623 r = -EFAULT;
2624 break;
2625 }
2626 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2627 break;
2628 default:
2629 r = -EINVAL;
2630 }
2631
2632 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2633
2634 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2635 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2636
2637 vfree(tmpbuf);
2638 return r;
2639}
2640
b0c632db
HC		 2641long kvm_arch_vcpu_ioctl(struct file *filp,
2642 unsigned int ioctl, unsigned long arg)
2643{
2644 struct kvm_vcpu *vcpu = filp->private_data;
2645 void __user *argp = (void __user *)arg;
800c1065 2646 int idx;
bc923cc9 2647 long r;
b0c632db 2648
93736624 2649 switch (ioctl) {
47b43c52
JF		 2650 case KVM_S390_IRQ: {
2651 struct kvm_s390_irq s390irq;
2652
2653 r = -EFAULT;
2654 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2655 break;
2656 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2657 break;
2658 }
93736624 2659 case KVM_S390_INTERRUPT: {
ba5c1e9b 2660 struct kvm_s390_interrupt s390int;
383d0b05 2661 struct kvm_s390_irq s390irq;
ba5c1e9b 2662
93736624 2663 r = -EFAULT;
ba5c1e9b 2664 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2665 break;
383d0b05
JF		 2666 if (s390int_to_s390irq(&s390int, &s390irq))
2667 return -EINVAL;
2668 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2669 break;
ba5c1e9b 2670 }
b0c632db 2671 case KVM_S390_STORE_STATUS:
800c1065 2672 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2673 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2674 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2675 break;
b0c632db
HC		 2676 case KVM_S390_SET_INITIAL_PSW: {
2677 psw_t psw;
2678
bc923cc9 2679 r = -EFAULT;
b0c632db 2680 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2681 break;
2682 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2683 break;
b0c632db
HC		 2684 }
2685 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2686 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2687 break;
14eebd91
CO		 2688 case KVM_SET_ONE_REG:
2689 case KVM_GET_ONE_REG: {
2690 struct kvm_one_reg reg;
2691 r = -EFAULT;
2692 if (copy_from_user(&reg, argp, sizeof(reg)))
2693 break;
2694 if (ioctl == KVM_SET_ONE_REG)
2695 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2696 else
2697 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2698 break;
2699 }
27e0393f
CO		 2700#ifdef CONFIG_KVM_S390_UCONTROL
2701 case KVM_S390_UCAS_MAP: {
2702 struct kvm_s390_ucas_mapping ucasmap;
2703
2704 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2705 r = -EFAULT;
2706 break;
2707 }
2708
2709 if (!kvm_is_ucontrol(vcpu->kvm)) {
2710 r = -EINVAL;
2711 break;
2712 }
2713
2714 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2715 ucasmap.vcpu_addr, ucasmap.length);
2716 break;
2717 }
2718 case KVM_S390_UCAS_UNMAP: {
2719 struct kvm_s390_ucas_mapping ucasmap;
2720
2721 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2722 r = -EFAULT;
2723 break;
2724 }
2725
2726 if (!kvm_is_ucontrol(vcpu->kvm)) {
2727 r = -EINVAL;
2728 break;
2729 }
2730
2731 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2732 ucasmap.length);
2733 break;
2734 }
2735#endif
ccc7910f 2736 case KVM_S390_VCPU_FAULT: {
527e30b4 2737 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2738 break;
2739 }
d6712df9
CH		 2740 case KVM_ENABLE_CAP:
2741 {
2742 struct kvm_enable_cap cap;
2743 r = -EFAULT;
2744 if (copy_from_user(&cap, argp, sizeof(cap)))
2745 break;
2746 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2747 break;
2748 }
41408c28
TH		 2749 case KVM_S390_MEM_OP: {
2750 struct kvm_s390_mem_op mem_op;
2751
2752 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2753 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2754 else
2755 r = -EFAULT;
2756 break;
2757 }
816c7667
JF		 2758 case KVM_S390_SET_IRQ_STATE: {
2759 struct kvm_s390_irq_state irq_state;
2760
2761 r = -EFAULT;
2762 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2763 break;
2764 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2765 irq_state.len == 0 ||
2766 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2767 r = -EINVAL;
2768 break;
2769 }
2770 r = kvm_s390_set_irq_state(vcpu,
2771 (void __user *) irq_state.buf,
2772 irq_state.len);
2773 break;
2774 }
2775 case KVM_S390_GET_IRQ_STATE: {
2776 struct kvm_s390_irq_state irq_state;
2777
2778 r = -EFAULT;
2779 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2780 break;
2781 if (irq_state.len == 0) {
2782 r = -EINVAL;
2783 break;
2784 }
2785 r = kvm_s390_get_irq_state(vcpu,
2786 (__u8 __user *) irq_state.buf,
2787 irq_state.len);
2788 break;
2789 }
b0c632db 2790 default:
3e6afcf1 2791 r = -ENOTTY;
b0c632db 2792 }
bc923cc9 2793 return r;
b0c632db
HC		 2794}
2795
5b1c1493
CO		 2796int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2797{
2798#ifdef CONFIG_KVM_S390_UCONTROL
2799 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2800 && (kvm_is_ucontrol(vcpu->kvm))) {
2801 vmf->page = virt_to_page(vcpu->arch.sie_block);
2802 get_page(vmf->page);
2803 return 0;
2804 }
2805#endif
2806 return VM_FAULT_SIGBUS;
2807}
2808
5587027c
AK		 2809int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2810 unsigned long npages)
db3fe4eb
TY		 2811{
2812 return 0;
2813}
2814
b0c632db 2815/* Section: memory related */
f7784b8e
MT		 2816int kvm_arch_prepare_memory_region(struct kvm *kvm,
2817 struct kvm_memory_slot *memslot,
09170a49 2818 const struct kvm_userspace_memory_region *mem,
7b6195a9 2819 enum kvm_mr_change change)
b0c632db 2820{
dd2887e7
NW		 2821 /* A few sanity checks. We can have memory slots which have to be
2822 located/ended at a segment boundary (1MB). The memory in userland is
2823 ok to be fragmented into various different vmas. It is okay to mmap()
2824 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2825
598841ca 2826 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2827 return -EINVAL;
2828
598841ca 2829 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2830 return -EINVAL;
2831
f7784b8e
MT		 2832 return 0;
2833}
2834
2835void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 2836 const struct kvm_userspace_memory_region *mem,
8482644a 2837 const struct kvm_memory_slot *old,
f36f3f28 2838 const struct kvm_memory_slot *new,
8482644a 2839 enum kvm_mr_change change)
f7784b8e 2840{
f7850c92 2841 int rc;
f7784b8e 2842
2cef4deb
CB		 2843 /* If the basics of the memslot do not change, we do not want
2844 * to update the gmap. Every update causes several unnecessary
2845 * segment translation exceptions. This is usually handled just
2846 * fine by the normal fault handler + gmap, but it will also
2847 * cause faults on the prefix page of running guest CPUs.
2848 */
2849 if (old->userspace_addr == mem->userspace_addr &&
2850 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2851 old->npages * PAGE_SIZE == mem->memory_size)
2852 return;
598841ca
CO		 2853
2854 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2855 mem->guest_phys_addr, mem->memory_size);
2856 if (rc)
ea2cdd27 2857 pr_warn("failed to commit memory region\n");
598841ca 2858 return;
b0c632db
HC		 2859}
2860
b0c632db
HC		 2861static int __init kvm_s390_init(void)
2862{
07197fd0
DH		 2863 if (!sclp.has_sief2) {
2864 pr_info("SIE not available\n");
2865 return -ENODEV;
2866 }
2867
9d8d5786 2868 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2869}
2870
2871static void __exit kvm_s390_exit(void)
2872{
2873 kvm_exit();
2874}
2875
2876module_init(kvm_s390_init);
2877module_exit(kvm_s390_exit);
566af940
CH		 2878
2879/*
2880 * Enable autoloading of the kvm module.
2881 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2882 * since x86 takes a different approach.
2883 */
2884#include <linux/miscdevice.h>
2885MODULE_ALIAS_MISCDEV(KVM_MINOR);
2886MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)