projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: wake up when the VCPU cpu timer expires
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db 30#include <asm/lowcore.h>
fdf03650 31#include <asm/etr.h>
b0c632db 32#include <asm/pgtable.h>
f5daba1d 33#include <asm/nmi.h>
a0616cde 34#include <asm/switch_to.h>
6d3da241 35#include <asm/isc.h>
1526bf9c 36#include <asm/sclp.h>
8f2abe6a 37#include "kvm-s390.h"
b0c632db
HC		 38#include "gaccess.h"
39
ea2cdd27
DH		 40#define KMSG_COMPONENT "kvm-s390"
41#undef pr_fmt
42#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
43
5786fffa
CH		 44#define CREATE_TRACE_POINTS
45#include "trace.h"
ade38c31 46#include "trace-s390.h"
5786fffa 47
41408c28 48#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 49#define LOCAL_IRQS 32
50#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
51 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 52
b0c632db
HC		 53#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
54
55struct kvm_stats_debugfs_item debugfs_entries[] = {
56 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 57 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 58 { "exit_validity", VCPU_STAT(exit_validity) },
59 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
60 { "exit_external_request", VCPU_STAT(exit_external_request) },
61 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 62 { "exit_instruction", VCPU_STAT(exit_instruction) },
63 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
64 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 65 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 66 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
ce2e4f0b 67 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 68 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 69 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 70 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
71 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 72 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 73 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 74 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
75 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
76 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
77 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
78 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
79 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
80 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 81 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 82 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
83 { "instruction_spx", VCPU_STAT(instruction_spx) },
84 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
85 { "instruction_stap", VCPU_STAT(instruction_stap) },
86 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 87 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 88 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
89 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 90 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 91 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
92 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 93 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 94 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 95 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 96 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 97 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 98 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
99 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 100 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 101 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
102 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 103 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 104 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
105 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
106 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 107 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
108 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
109 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 110 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 111 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 112 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 113 { "diagnose_258", VCPU_STAT(diagnose_258) },
114 { "diagnose_308", VCPU_STAT(diagnose_308) },
115 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 116 { NULL }
117};
118
9d8d5786
MM		 119/* upper facilities limit for kvm */
120unsigned long kvm_s390_fac_list_mask[] = {
a3ed8dae 121 0xffe6fffbfcfdfc40UL,
53df84f8 122 0x005e800000000000UL,
9d8d5786 123};
b0c632db 124
9d8d5786 125unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 126{
9d8d5786
MM		 127 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
128 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 129}
130
9d8d5786 131static struct gmap_notifier gmap_notifier;
78f26131 132debug_info_t *kvm_s390_dbf;
9d8d5786 133
b0c632db 134/* Section: not file related */
13a34e06 135int kvm_arch_hardware_enable(void)
b0c632db
HC		 136{
137 /* every s390 is virtualization enabled ;-) */
10474ae8 138 return 0;
b0c632db
HC		 139}
140
2c70fe44
CB		 141static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
142
fdf03650
FZ		 143/*
144 * This callback is executed during stop_machine(). All CPUs are therefore
145 * temporarily stopped. In order not to change guest behavior, we have to
146 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
147 * so a CPU won't be stopped while calculating with the epoch.
148 */
149static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
150 void *v)
151{
152 struct kvm *kvm;
153 struct kvm_vcpu *vcpu;
154 int i;
155 unsigned long long *delta = v;
156
157 list_for_each_entry(kvm, &vm_list, vm_list) {
158 kvm->arch.epoch -= *delta;
159 kvm_for_each_vcpu(i, vcpu, kvm) {
160 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 161 if (vcpu->arch.cputm_enabled)
162 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 163 }
164 }
165 return NOTIFY_OK;
166}
167
168static struct notifier_block kvm_clock_notifier = {
169 .notifier_call = kvm_clock_sync,
170};
171
b0c632db
HC		 172int kvm_arch_hardware_setup(void)
173{
2c70fe44
CB		 174 gmap_notifier.notifier_call = kvm_gmap_notifier;
175 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 176 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
177 &kvm_clock_notifier);
b0c632db
HC		 178 return 0;
179}
180
181void kvm_arch_hardware_unsetup(void)
182{
2c70fe44 183 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 184 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
185 &kvm_clock_notifier);
b0c632db
HC		 186}
187
b0c632db
HC		 188int kvm_arch_init(void *opaque)
189{
78f26131
CB		 190 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
191 if (!kvm_s390_dbf)
192 return -ENOMEM;
193
194 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
195 debug_unregister(kvm_s390_dbf);
196 return -ENOMEM;
197 }
198
84877d93
CH		 199 /* Register floating interrupt controller interface. */
200 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 201}
202
78f26131
CB		 203void kvm_arch_exit(void)
204{
205 debug_unregister(kvm_s390_dbf);
206}
207
b0c632db
HC		 208/* Section: device related */
209long kvm_arch_dev_ioctl(struct file *filp,
210 unsigned int ioctl, unsigned long arg)
211{
212 if (ioctl == KVM_S390_ENABLE_SIE)
213 return s390_enable_sie();
214 return -EINVAL;
215}
216
784aa3d7 217int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 218{
d7b0b5eb
CO		 219 int r;
220
2bd0ac4e 221 switch (ext) {
d7b0b5eb 222 case KVM_CAP_S390_PSW:
b6cf8788 223 case KVM_CAP_S390_GMAP:
52e16b18 224 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 225#ifdef CONFIG_KVM_S390_UCONTROL
226 case KVM_CAP_S390_UCONTROL:
227#endif
3c038e6b 228 case KVM_CAP_ASYNC_PF:
60b413c9 229 case KVM_CAP_SYNC_REGS:
14eebd91 230 case KVM_CAP_ONE_REG:
d6712df9 231 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 232 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 233 case KVM_CAP_IOEVENTFD:
c05c4186 234 case KVM_CAP_DEVICE_CTRL:
d938dc55 235 case KVM_CAP_ENABLE_CAP_VM:
78599d90 236 case KVM_CAP_S390_IRQCHIP:
f2061656 237 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 238 case KVM_CAP_MP_STATE:
47b43c52 239 case KVM_CAP_S390_INJECT_IRQ:
2444b352 240 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 241 case KVM_CAP_S390_USER_STSI:
30ee2a98 242 case KVM_CAP_S390_SKEYS:
816c7667 243 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 244 r = 1;
245 break;
41408c28
TH		 246 case KVM_CAP_S390_MEM_OP:
247 r = MEM_OP_MAX_SIZE;
248 break;
e726b1bd
CB		 249 case KVM_CAP_NR_VCPUS:
250 case KVM_CAP_MAX_VCPUS:
fe0edcb7
ED		 251 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
252 : KVM_S390_BSCA_CPU_SLOTS;
e726b1bd 253 break;
e1e2e605
NW		 254 case KVM_CAP_NR_MEMSLOTS:
255 r = KVM_USER_MEM_SLOTS;
256 break;
1526bf9c 257 case KVM_CAP_S390_COW:
abf09bed 258 r = MACHINE_HAS_ESOP;
1526bf9c 259 break;
68c55750
EF		 260 case KVM_CAP_S390_VECTOR_REGISTERS:
261 r = MACHINE_HAS_VX;
262 break;
c6e5f166
FZ		 263 case KVM_CAP_S390_RI:
264 r = test_facility(64);
265 break;
2bd0ac4e 266 default:
d7b0b5eb 267 r = 0;
2bd0ac4e 268 }
d7b0b5eb 269 return r;
b0c632db
HC		 270}
271
15f36ebd
JH		 272static void kvm_s390_sync_dirty_log(struct kvm *kvm,
273 struct kvm_memory_slot *memslot)
274{
275 gfn_t cur_gfn, last_gfn;
276 unsigned long address;
277 struct gmap *gmap = kvm->arch.gmap;
278
15f36ebd
JH		 279 /* Loop over all guest pages */
280 last_gfn = memslot->base_gfn + memslot->npages;
281 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
282 address = gfn_to_hva_memslot(memslot, cur_gfn);
283
284 if (gmap_test_and_clear_dirty(address, gmap))
285 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 286 if (fatal_signal_pending(current))
287 return;
70c88a00 288 cond_resched();
15f36ebd 289 }
15f36ebd
JH		 290}
291
b0c632db 292/* Section: vm related */
a6e2f683
ED		 293static void sca_del_vcpu(struct kvm_vcpu *vcpu);
294
b0c632db
HC		 295/*
296 * Get (and clear) the dirty memory log for a memory slot.
297 */
298int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
299 struct kvm_dirty_log *log)
300{
15f36ebd
JH		 301 int r;
302 unsigned long n;
9f6b8029 303 struct kvm_memslots *slots;
15f36ebd
JH		 304 struct kvm_memory_slot *memslot;
305 int is_dirty = 0;
306
307 mutex_lock(&kvm->slots_lock);
308
309 r = -EINVAL;
310 if (log->slot >= KVM_USER_MEM_SLOTS)
311 goto out;
312
9f6b8029
PB		 313 slots = kvm_memslots(kvm);
314 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 315 r = -ENOENT;
316 if (!memslot->dirty_bitmap)
317 goto out;
318
319 kvm_s390_sync_dirty_log(kvm, memslot);
320 r = kvm_get_dirty_log(kvm, log, &is_dirty);
321 if (r)
322 goto out;
323
324 /* Clear the dirty log */
325 if (is_dirty) {
326 n = kvm_dirty_bitmap_bytes(memslot);
327 memset(memslot->dirty_bitmap, 0, n);
328 }
329 r = 0;
330out:
331 mutex_unlock(&kvm->slots_lock);
332 return r;
b0c632db
HC		 333}
334
d938dc55
CH		 335static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
336{
337 int r;
338
339 if (cap->flags)
340 return -EINVAL;
341
342 switch (cap->cap) {
84223598 343 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 344 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 345 kvm->arch.use_irqchip = 1;
346 r = 0;
347 break;
2444b352 348 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 349 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 350 kvm->arch.user_sigp = 1;
351 r = 0;
352 break;
68c55750 353 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 354 mutex_lock(&kvm->lock);
355 if (atomic_read(&kvm->online_vcpus)) {
356 r = -EBUSY;
357 } else if (MACHINE_HAS_VX) {
18280d8b
MM		 358 set_kvm_facility(kvm->arch.model.fac->mask, 129);
359 set_kvm_facility(kvm->arch.model.fac->list, 129);
360 r = 0;
361 } else
362 r = -EINVAL;
5967c17b 363 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 364 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
365 r ? "(not available)" : "(success)");
68c55750 366 break;
c6e5f166
FZ		 367 case KVM_CAP_S390_RI:
368 r = -EINVAL;
369 mutex_lock(&kvm->lock);
370 if (atomic_read(&kvm->online_vcpus)) {
371 r = -EBUSY;
372 } else if (test_facility(64)) {
373 set_kvm_facility(kvm->arch.model.fac->mask, 64);
374 set_kvm_facility(kvm->arch.model.fac->list, 64);
375 r = 0;
376 }
377 mutex_unlock(&kvm->lock);
378 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
379 r ? "(not available)" : "(success)");
380 break;
e44fc8c9 381 case KVM_CAP_S390_USER_STSI:
c92ea7b9 382 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 383 kvm->arch.user_stsi = 1;
384 r = 0;
385 break;
d938dc55
CH		 386 default:
387 r = -EINVAL;
388 break;
389 }
390 return r;
391}
392
8c0a7ce6
DD		 393static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
394{
395 int ret;
396
397 switch (attr->attr) {
398 case KVM_S390_VM_MEM_LIMIT_SIZE:
399 ret = 0;
c92ea7b9 400 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 401 kvm->arch.mem_limit);
402 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 403 ret = -EFAULT;
404 break;
405 default:
406 ret = -ENXIO;
407 break;
408 }
409 return ret;
410}
411
412static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 413{
414 int ret;
415 unsigned int idx;
416 switch (attr->attr) {
417 case KVM_S390_VM_MEM_ENABLE_CMMA:
e6db1d61
DD		 418 /* enable CMMA only for z10 and later (EDAT_1) */
419 ret = -EINVAL;
420 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
421 break;
422
4f718eab 423 ret = -EBUSY;
c92ea7b9 424 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 425 mutex_lock(&kvm->lock);
426 if (atomic_read(&kvm->online_vcpus) == 0) {
427 kvm->arch.use_cmma = 1;
428 ret = 0;
429 }
430 mutex_unlock(&kvm->lock);
431 break;
432 case KVM_S390_VM_MEM_CLR_CMMA:
c3489155
DD		 433 ret = -EINVAL;
434 if (!kvm->arch.use_cmma)
435 break;
436
c92ea7b9 437 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 438 mutex_lock(&kvm->lock);
439 idx = srcu_read_lock(&kvm->srcu);
a13cff31 440 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 441 srcu_read_unlock(&kvm->srcu, idx);
442 mutex_unlock(&kvm->lock);
443 ret = 0;
444 break;
8c0a7ce6
DD		 445 case KVM_S390_VM_MEM_LIMIT_SIZE: {
446 unsigned long new_limit;
447
448 if (kvm_is_ucontrol(kvm))
449 return -EINVAL;
450
451 if (get_user(new_limit, (u64 __user *)attr->addr))
452 return -EFAULT;
453
a3a92c31
DD		 454 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
455 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 456 return -E2BIG;
457
a3a92c31
DD		 458 if (!new_limit)
459 return -EINVAL;
460
461 /* gmap_alloc takes last usable address */
462 if (new_limit != KVM_S390_NO_MEM_LIMIT)
463 new_limit -= 1;
464
8c0a7ce6
DD		 465 ret = -EBUSY;
466 mutex_lock(&kvm->lock);
467 if (atomic_read(&kvm->online_vcpus) == 0) {
468 /* gmap_alloc will round the limit up */
469 struct gmap *new = gmap_alloc(current->mm, new_limit);
470
471 if (!new) {
472 ret = -ENOMEM;
473 } else {
474 gmap_free(kvm->arch.gmap);
475 new->private = kvm;
476 kvm->arch.gmap = new;
477 ret = 0;
478 }
479 }
480 mutex_unlock(&kvm->lock);
a3a92c31
DD		 481 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
482 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
483 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 484 break;
485 }
4f718eab
DD		 486 default:
487 ret = -ENXIO;
488 break;
489 }
490 return ret;
491}
492
a374e892
TK		 493static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
494
495static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
496{
497 struct kvm_vcpu *vcpu;
498 int i;
499
9d8d5786 500 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 501 return -EINVAL;
502
503 mutex_lock(&kvm->lock);
504 switch (attr->attr) {
505 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
506 get_random_bytes(
507 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
508 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
509 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 510 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 511 break;
512 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
513 get_random_bytes(
514 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
515 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
516 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 517 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 518 break;
519 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
520 kvm->arch.crypto.aes_kw = 0;
521 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
522 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 523 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 524 break;
525 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
526 kvm->arch.crypto.dea_kw = 0;
527 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
528 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 529 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 530 break;
531 default:
532 mutex_unlock(&kvm->lock);
533 return -ENXIO;
534 }
535
536 kvm_for_each_vcpu(i, vcpu, kvm) {
537 kvm_s390_vcpu_crypto_setup(vcpu);
538 exit_sie(vcpu);
539 }
540 mutex_unlock(&kvm->lock);
541 return 0;
542}
543
72f25020
JH		 544static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
545{
546 u8 gtod_high;
547
548 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
549 sizeof(gtod_high)))
550 return -EFAULT;
551
552 if (gtod_high != 0)
553 return -EINVAL;
58c383c6 554 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 555
556 return 0;
557}
558
559static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
560{
5a3d883a 561 u64 gtod;
72f25020
JH		 562
563 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
564 return -EFAULT;
565
25ed1675 566 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 567 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 568 return 0;
569}
570
571static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
572{
573 int ret;
574
575 if (attr->flags)
576 return -EINVAL;
577
578 switch (attr->attr) {
579 case KVM_S390_VM_TOD_HIGH:
580 ret = kvm_s390_set_tod_high(kvm, attr);
581 break;
582 case KVM_S390_VM_TOD_LOW:
583 ret = kvm_s390_set_tod_low(kvm, attr);
584 break;
585 default:
586 ret = -ENXIO;
587 break;
588 }
589 return ret;
590}
591
592static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
593{
594 u8 gtod_high = 0;
595
596 if (copy_to_user((void __user *)attr->addr, &gtod_high,
597 sizeof(gtod_high)))
598 return -EFAULT;
58c383c6 599 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 600
601 return 0;
602}
603
604static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
605{
5a3d883a 606 u64 gtod;
72f25020 607
60417fcc 608 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 609 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
610 return -EFAULT;
58c383c6 611 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 612
613 return 0;
614}
615
616static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
617{
618 int ret;
619
620 if (attr->flags)
621 return -EINVAL;
622
623 switch (attr->attr) {
624 case KVM_S390_VM_TOD_HIGH:
625 ret = kvm_s390_get_tod_high(kvm, attr);
626 break;
627 case KVM_S390_VM_TOD_LOW:
628 ret = kvm_s390_get_tod_low(kvm, attr);
629 break;
630 default:
631 ret = -ENXIO;
632 break;
633 }
634 return ret;
635}
636
658b6eda
MM		 637static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
638{
639 struct kvm_s390_vm_cpu_processor *proc;
640 int ret = 0;
641
642 mutex_lock(&kvm->lock);
643 if (atomic_read(&kvm->online_vcpus)) {
644 ret = -EBUSY;
645 goto out;
646 }
647 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
648 if (!proc) {
649 ret = -ENOMEM;
650 goto out;
651 }
652 if (!copy_from_user(proc, (void __user *)attr->addr,
653 sizeof(*proc))) {
654 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
655 sizeof(struct cpuid));
656 kvm->arch.model.ibc = proc->ibc;
981467c9 657 memcpy(kvm->arch.model.fac->list, proc->fac_list,
658b6eda
MM		 658 S390_ARCH_FAC_LIST_SIZE_BYTE);
659 } else
660 ret = -EFAULT;
661 kfree(proc);
662out:
663 mutex_unlock(&kvm->lock);
664 return ret;
665}
666
667static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
668{
669 int ret = -ENXIO;
670
671 switch (attr->attr) {
672 case KVM_S390_VM_CPU_PROCESSOR:
673 ret = kvm_s390_set_processor(kvm, attr);
674 break;
675 }
676 return ret;
677}
678
679static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
680{
681 struct kvm_s390_vm_cpu_processor *proc;
682 int ret = 0;
683
684 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
685 if (!proc) {
686 ret = -ENOMEM;
687 goto out;
688 }
689 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
690 proc->ibc = kvm->arch.model.ibc;
981467c9 691 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 692 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
693 ret = -EFAULT;
694 kfree(proc);
695out:
696 return ret;
697}
698
699static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
700{
701 struct kvm_s390_vm_cpu_machine *mach;
702 int ret = 0;
703
704 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
705 if (!mach) {
706 ret = -ENOMEM;
707 goto out;
708 }
709 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 710 mach->ibc = sclp.ibc;
981467c9
MM		 711 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
712 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 713 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 714 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 715 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
716 ret = -EFAULT;
717 kfree(mach);
718out:
719 return ret;
720}
721
722static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
723{
724 int ret = -ENXIO;
725
726 switch (attr->attr) {
727 case KVM_S390_VM_CPU_PROCESSOR:
728 ret = kvm_s390_get_processor(kvm, attr);
729 break;
730 case KVM_S390_VM_CPU_MACHINE:
731 ret = kvm_s390_get_machine(kvm, attr);
732 break;
733 }
734 return ret;
735}
736
f2061656
DD		 737static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
738{
739 int ret;
740
741 switch (attr->group) {
4f718eab 742 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 743 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 744 break;
72f25020
JH		 745 case KVM_S390_VM_TOD:
746 ret = kvm_s390_set_tod(kvm, attr);
747 break;
658b6eda
MM		 748 case KVM_S390_VM_CPU_MODEL:
749 ret = kvm_s390_set_cpu_model(kvm, attr);
750 break;
a374e892
TK		 751 case KVM_S390_VM_CRYPTO:
752 ret = kvm_s390_vm_set_crypto(kvm, attr);
753 break;
f2061656
DD		 754 default:
755 ret = -ENXIO;
756 break;
757 }
758
759 return ret;
760}
761
762static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
763{
8c0a7ce6
DD		 764 int ret;
765
766 switch (attr->group) {
767 case KVM_S390_VM_MEM_CTRL:
768 ret = kvm_s390_get_mem_control(kvm, attr);
769 break;
72f25020
JH		 770 case KVM_S390_VM_TOD:
771 ret = kvm_s390_get_tod(kvm, attr);
772 break;
658b6eda
MM		 773 case KVM_S390_VM_CPU_MODEL:
774 ret = kvm_s390_get_cpu_model(kvm, attr);
775 break;
8c0a7ce6
DD		 776 default:
777 ret = -ENXIO;
778 break;
779 }
780
781 return ret;
f2061656
DD		 782}
783
784static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
785{
786 int ret;
787
788 switch (attr->group) {
4f718eab
DD		 789 case KVM_S390_VM_MEM_CTRL:
790 switch (attr->attr) {
791 case KVM_S390_VM_MEM_ENABLE_CMMA:
792 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 793 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 794 ret = 0;
795 break;
796 default:
797 ret = -ENXIO;
798 break;
799 }
800 break;
72f25020
JH		 801 case KVM_S390_VM_TOD:
802 switch (attr->attr) {
803 case KVM_S390_VM_TOD_LOW:
804 case KVM_S390_VM_TOD_HIGH:
805 ret = 0;
806 break;
807 default:
808 ret = -ENXIO;
809 break;
810 }
811 break;
658b6eda
MM		 812 case KVM_S390_VM_CPU_MODEL:
813 switch (attr->attr) {
814 case KVM_S390_VM_CPU_PROCESSOR:
815 case KVM_S390_VM_CPU_MACHINE:
816 ret = 0;
817 break;
818 default:
819 ret = -ENXIO;
820 break;
821 }
822 break;
a374e892
TK		 823 case KVM_S390_VM_CRYPTO:
824 switch (attr->attr) {
825 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
826 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
827 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
828 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
829 ret = 0;
830 break;
831 default:
832 ret = -ENXIO;
833 break;
834 }
835 break;
f2061656
DD		 836 default:
837 ret = -ENXIO;
838 break;
839 }
840
841 return ret;
842}
843
30ee2a98
JH		 844static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
845{
846 uint8_t *keys;
847 uint64_t hva;
848 unsigned long curkey;
849 int i, r = 0;
850
851 if (args->flags != 0)
852 return -EINVAL;
853
854 /* Is this guest using storage keys? */
855 if (!mm_use_skey(current->mm))
856 return KVM_S390_GET_SKEYS_NONE;
857
858 /* Enforce sane limit on memory allocation */
859 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
860 return -EINVAL;
861
862 keys = kmalloc_array(args->count, sizeof(uint8_t),
863 GFP_KERNEL | __GFP_NOWARN);
864 if (!keys)
865 keys = vmalloc(sizeof(uint8_t) * args->count);
866 if (!keys)
867 return -ENOMEM;
868
869 for (i = 0; i < args->count; i++) {
870 hva = gfn_to_hva(kvm, args->start_gfn + i);
871 if (kvm_is_error_hva(hva)) {
872 r = -EFAULT;
873 goto out;
874 }
875
876 curkey = get_guest_storage_key(current->mm, hva);
877 if (IS_ERR_VALUE(curkey)) {
878 r = curkey;
879 goto out;
880 }
881 keys[i] = curkey;
882 }
883
884 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
885 sizeof(uint8_t) * args->count);
886 if (r)
887 r = -EFAULT;
888out:
889 kvfree(keys);
890 return r;
891}
892
893static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
894{
895 uint8_t *keys;
896 uint64_t hva;
897 int i, r = 0;
898
899 if (args->flags != 0)
900 return -EINVAL;
901
902 /* Enforce sane limit on memory allocation */
903 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
904 return -EINVAL;
905
906 keys = kmalloc_array(args->count, sizeof(uint8_t),
907 GFP_KERNEL | __GFP_NOWARN);
908 if (!keys)
909 keys = vmalloc(sizeof(uint8_t) * args->count);
910 if (!keys)
911 return -ENOMEM;
912
913 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
914 sizeof(uint8_t) * args->count);
915 if (r) {
916 r = -EFAULT;
917 goto out;
918 }
919
920 /* Enable storage key handling for the guest */
14d4a425
DD		 921 r = s390_enable_skey();
922 if (r)
923 goto out;
30ee2a98
JH		 924
925 for (i = 0; i < args->count; i++) {
926 hva = gfn_to_hva(kvm, args->start_gfn + i);
927 if (kvm_is_error_hva(hva)) {
928 r = -EFAULT;
929 goto out;
930 }
931
932 /* Lowest order bit is reserved */
933 if (keys[i] & 0x01) {
934 r = -EINVAL;
935 goto out;
936 }
937
938 r = set_guest_storage_key(current->mm, hva,
939 (unsigned long)keys[i], 0);
940 if (r)
941 goto out;
942 }
943out:
944 kvfree(keys);
945 return r;
946}
947
b0c632db
HC		 948long kvm_arch_vm_ioctl(struct file *filp,
949 unsigned int ioctl, unsigned long arg)
950{
951 struct kvm *kvm = filp->private_data;
952 void __user *argp = (void __user *)arg;
f2061656 953 struct kvm_device_attr attr;
b0c632db
HC		 954 int r;
955
956 switch (ioctl) {
ba5c1e9b
CO		 957 case KVM_S390_INTERRUPT: {
958 struct kvm_s390_interrupt s390int;
959
960 r = -EFAULT;
961 if (copy_from_user(&s390int, argp, sizeof(s390int)))
962 break;
963 r = kvm_s390_inject_vm(kvm, &s390int);
964 break;
965 }
d938dc55
CH		 966 case KVM_ENABLE_CAP: {
967 struct kvm_enable_cap cap;
968 r = -EFAULT;
969 if (copy_from_user(&cap, argp, sizeof(cap)))
970 break;
971 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
972 break;
973 }
84223598
CH		 974 case KVM_CREATE_IRQCHIP: {
975 struct kvm_irq_routing_entry routing;
976
977 r = -EINVAL;
978 if (kvm->arch.use_irqchip) {
979 /* Set up dummy routing. */
980 memset(&routing, 0, sizeof(routing));
152b2839 981 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 982 }
983 break;
984 }
f2061656
DD		 985 case KVM_SET_DEVICE_ATTR: {
986 r = -EFAULT;
987 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
988 break;
989 r = kvm_s390_vm_set_attr(kvm, &attr);
990 break;
991 }
992 case KVM_GET_DEVICE_ATTR: {
993 r = -EFAULT;
994 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
995 break;
996 r = kvm_s390_vm_get_attr(kvm, &attr);
997 break;
998 }
999 case KVM_HAS_DEVICE_ATTR: {
1000 r = -EFAULT;
1001 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1002 break;
1003 r = kvm_s390_vm_has_attr(kvm, &attr);
1004 break;
1005 }
30ee2a98
JH		 1006 case KVM_S390_GET_SKEYS: {
1007 struct kvm_s390_skeys args;
1008
1009 r = -EFAULT;
1010 if (copy_from_user(&args, argp,
1011 sizeof(struct kvm_s390_skeys)))
1012 break;
1013 r = kvm_s390_get_skeys(kvm, &args);
1014 break;
1015 }
1016 case KVM_S390_SET_SKEYS: {
1017 struct kvm_s390_skeys args;
1018
1019 r = -EFAULT;
1020 if (copy_from_user(&args, argp,
1021 sizeof(struct kvm_s390_skeys)))
1022 break;
1023 r = kvm_s390_set_skeys(kvm, &args);
1024 break;
1025 }
b0c632db 1026 default:
367e1319 1027 r = -ENOTTY;
b0c632db
HC		 1028 }
1029
1030 return r;
1031}
1032
45c9b47c
TK		 1033static int kvm_s390_query_ap_config(u8 *config)
1034{
1035 u32 fcn_code = 0x04000000UL;
86044c8c 1036 u32 cc = 0;
45c9b47c 1037
86044c8c 1038 memset(config, 0, 128);
45c9b47c
TK		 1039 asm volatile(
1040 "lgr 0,%1\n"
1041 "lgr 2,%2\n"
1042 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1043 "0: ipm %0\n"
45c9b47c 1044 "srl %0,28\n"
86044c8c
CB		 1045 "1:\n"
1046 EX_TABLE(0b, 1b)
1047 : "+r" (cc)
45c9b47c
TK		 1048 : "r" (fcn_code), "r" (config)
1049 : "cc", "0", "2", "memory"
1050 );
1051
1052 return cc;
1053}
1054
1055static int kvm_s390_apxa_installed(void)
1056{
1057 u8 config[128];
1058 int cc;
1059
a6aacc3f 1060 if (test_facility(12)) {
45c9b47c
TK		 1061 cc = kvm_s390_query_ap_config(config);
1062
1063 if (cc)
1064 pr_err("PQAP(QCI) failed with cc=%d", cc);
1065 else
1066 return config[0] & 0x40;
1067 }
1068
1069 return 0;
1070}
1071
1072static void kvm_s390_set_crycb_format(struct kvm *kvm)
1073{
1074 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1075
1076 if (kvm_s390_apxa_installed())
1077 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1078 else
1079 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1080}
1081
9d8d5786
MM		 1082static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1083{
1084 get_cpu_id(cpu_id);
1085 cpu_id->version = 0xff;
1086}
1087
5102ee87
TK		 1088static int kvm_s390_crypto_init(struct kvm *kvm)
1089{
9d8d5786 1090 if (!test_kvm_facility(kvm, 76))
5102ee87
TK		 1091 return 0;
1092
1093 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1094 GFP_KERNEL | GFP_DMA);
1095 if (!kvm->arch.crypto.crycb)
1096 return -ENOMEM;
1097
45c9b47c 1098 kvm_s390_set_crycb_format(kvm);
5102ee87 1099
ed6f76b4
TK		 1100 /* Enable AES/DEA protected key functions by default */
1101 kvm->arch.crypto.aes_kw = 1;
1102 kvm->arch.crypto.dea_kw = 1;
1103 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1104 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1105 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1106 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
a374e892 1107
5102ee87
TK		 1108 return 0;
1109}
1110
7d43bafc
ED		 1111static void sca_dispose(struct kvm *kvm)
1112{
1113 if (kvm->arch.use_esca)
5e044315 1114 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1115 else
1116 free_page((unsigned long)(kvm->arch.sca));
1117 kvm->arch.sca = NULL;
1118}
1119
e08b9637 1120int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1121{
9d8d5786 1122 int i, rc;
b0c632db 1123 char debug_name[16];
f6c137ff 1124 static unsigned long sca_offset;
b0c632db 1125
e08b9637
CO		 1126 rc = -EINVAL;
1127#ifdef CONFIG_KVM_S390_UCONTROL
1128 if (type & ~KVM_VM_S390_UCONTROL)
1129 goto out_err;
1130 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1131 goto out_err;
1132#else
1133 if (type)
1134 goto out_err;
1135#endif
1136
b0c632db
HC		 1137 rc = s390_enable_sie();
1138 if (rc)
d89f5eff 1139 goto out_err;
b0c632db 1140
b290411a
CO		 1141 rc = -ENOMEM;
1142
7d43bafc 1143 kvm->arch.use_esca = 0; /* start with basic SCA */
5e044315 1144 rwlock_init(&kvm->arch.sca_lock);
bc784cce 1145 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
b0c632db 1146 if (!kvm->arch.sca)
d89f5eff 1147 goto out_err;
f6c137ff 1148 spin_lock(&kvm_lock);
c5c2c393 1149 sca_offset += 16;
bc784cce 1150 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1151 sca_offset = 0;
bc784cce
ED		 1152 kvm->arch.sca = (struct bsca_block *)
1153 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1154 spin_unlock(&kvm_lock);
b0c632db
HC		 1155
1156 sprintf(debug_name, "kvm-%u", current->pid);
1157
1cb9cf72 1158 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1159 if (!kvm->arch.dbf)
40f5b735 1160 goto out_err;
b0c632db 1161
9d8d5786
MM		 1162 /*
1163 * The architectural maximum amount of facilities is 16 kbit. To store
1164 * this amount, 2 kbyte of memory is required. Thus we need a full
981467c9
MM		 1165 * page to hold the guest facility list (arch.model.fac->list) and the
1166 * facility mask (arch.model.fac->mask). Its address size has to be
9d8d5786
MM		 1167 * 31 bits and word aligned.
1168 */
1169 kvm->arch.model.fac =
981467c9 1170 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
9d8d5786 1171 if (!kvm->arch.model.fac)
40f5b735 1172 goto out_err;
9d8d5786 1173
fb5bf93f 1174 /* Populate the facility mask initially. */
981467c9 1175 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
94422ee8 1176 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1177 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1178 if (i < kvm_s390_fac_list_mask_size())
981467c9 1179 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1180 else
981467c9 1181 kvm->arch.model.fac->mask[i] = 0UL;
9d8d5786
MM		 1182 }
1183
981467c9
MM		 1184 /* Populate the facility list initially. */
1185 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1186 S390_ARCH_FAC_LIST_SIZE_BYTE);
1187
9d8d5786 1188 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
37c5f6c8 1189 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1190
5102ee87 1191 if (kvm_s390_crypto_init(kvm) < 0)
40f5b735 1192 goto out_err;
5102ee87 1193
ba5c1e9b 1194 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1195 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1196 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1197 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1198 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1199
b0c632db 1200 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1201 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1202
e08b9637
CO		 1203 if (type & KVM_VM_S390_UCONTROL) {
1204 kvm->arch.gmap = NULL;
a3a92c31 1205 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1206 } else {
32e6b236
GH		 1207 if (sclp.hamax == U64_MAX)
1208 kvm->arch.mem_limit = TASK_MAX_SIZE;
1209 else
1210 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1211 sclp.hamax + 1);
a3a92c31 1212 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1213 if (!kvm->arch.gmap)
40f5b735 1214 goto out_err;
2c70fe44 1215 kvm->arch.gmap->private = kvm;
24eb3a82 1216 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1217 }
fa6b7fe9
CH		 1218
1219 kvm->arch.css_support = 0;
84223598 1220 kvm->arch.use_irqchip = 0;
72f25020 1221 kvm->arch.epoch = 0;
fa6b7fe9 1222
8ad35755 1223 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1224 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1225
d89f5eff 1226 return 0;
40f5b735 1227out_err:
5102ee87 1228 kfree(kvm->arch.crypto.crycb);
9d8d5786 1229 free_page((unsigned long)kvm->arch.model.fac);
598841ca 1230 debug_unregister(kvm->arch.dbf);
7d43bafc 1231 sca_dispose(kvm);
78f26131 1232 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1233 return rc;
b0c632db
HC		 1234}
1235
d329c035
CB		 1236void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1237{
1238 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1239 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1240 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1241 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1242 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1243 sca_del_vcpu(vcpu);
27e0393f
CO		 1244
1245 if (kvm_is_ucontrol(vcpu->kvm))
1246 gmap_free(vcpu->arch.gmap);
1247
e6db1d61 1248 if (vcpu->kvm->arch.use_cmma)
b31605c1 1249 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1250 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1251
6692cef3 1252 kvm_vcpu_uninit(vcpu);
b110feaf 1253 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1254}
1255
1256static void kvm_free_vcpus(struct kvm *kvm)
1257{
1258 unsigned int i;
988a2cae 1259 struct kvm_vcpu *vcpu;
d329c035 1260
988a2cae
GN		 1261 kvm_for_each_vcpu(i, vcpu, kvm)
1262 kvm_arch_vcpu_destroy(vcpu);
1263
1264 mutex_lock(&kvm->lock);
1265 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1266 kvm->vcpus[i] = NULL;
1267
1268 atomic_set(&kvm->online_vcpus, 0);
1269 mutex_unlock(&kvm->lock);
d329c035
CB		 1270}
1271
b0c632db
HC		 1272void kvm_arch_destroy_vm(struct kvm *kvm)
1273{
d329c035 1274 kvm_free_vcpus(kvm);
9d8d5786 1275 free_page((unsigned long)kvm->arch.model.fac);
7d43bafc 1276 sca_dispose(kvm);
d329c035 1277 debug_unregister(kvm->arch.dbf);
5102ee87 1278 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 1279 if (!kvm_is_ucontrol(kvm))
1280 gmap_free(kvm->arch.gmap);
841b91c5 1281 kvm_s390_destroy_adapters(kvm);
67335e63 1282 kvm_s390_clear_float_irqs(kvm);
8335713a 1283 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1284}
1285
1286/* Section: vcpu related */
dafd032a
DD		 1287static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1288{
1289 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1290 if (!vcpu->arch.gmap)
1291 return -ENOMEM;
1292 vcpu->arch.gmap->private = vcpu->kvm;
1293
1294 return 0;
1295}
1296
a6e2f683
ED		 1297static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1298{
5e044315 1299 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1300 if (vcpu->kvm->arch.use_esca) {
1301 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1302
7d43bafc 1303 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1304 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1305 } else {
1306 struct bsca_block *sca = vcpu->kvm->arch.sca;
1307
1308 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1309 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1310 }
5e044315 1311 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1312}
1313
eaa78f34 1314static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1315{
eaa78f34
DH		 1316 read_lock(&vcpu->kvm->arch.sca_lock);
1317 if (vcpu->kvm->arch.use_esca) {
1318 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1319
eaa78f34 1320 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1321 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1322 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1323 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1324 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1325 } else {
eaa78f34 1326 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1327
eaa78f34 1328 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1329 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1330 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1331 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1332 }
eaa78f34 1333 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1334}
1335
1336/* Basic SCA to Extended SCA data copy routines */
1337static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1338{
1339 d->sda = s->sda;
1340 d->sigp_ctrl.c = s->sigp_ctrl.c;
1341 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1342}
1343
1344static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1345{
1346 int i;
1347
1348 d->ipte_control = s->ipte_control;
1349 d->mcn[0] = s->mcn;
1350 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1351 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1352}
1353
1354static int sca_switch_to_extended(struct kvm *kvm)
1355{
1356 struct bsca_block *old_sca = kvm->arch.sca;
1357 struct esca_block *new_sca;
1358 struct kvm_vcpu *vcpu;
1359 unsigned int vcpu_idx;
1360 u32 scaol, scaoh;
1361
1362 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1363 if (!new_sca)
1364 return -ENOMEM;
1365
1366 scaoh = (u32)((u64)(new_sca) >> 32);
1367 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1368
1369 kvm_s390_vcpu_block_all(kvm);
1370 write_lock(&kvm->arch.sca_lock);
1371
1372 sca_copy_b_to_e(new_sca, old_sca);
1373
1374 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1375 vcpu->arch.sie_block->scaoh = scaoh;
1376 vcpu->arch.sie_block->scaol = scaol;
1377 vcpu->arch.sie_block->ecb2 |= 0x04U;
1378 }
1379 kvm->arch.sca = new_sca;
1380 kvm->arch.use_esca = 1;
1381
1382 write_unlock(&kvm->arch.sca_lock);
1383 kvm_s390_vcpu_unblock_all(kvm);
1384
1385 free_page((unsigned long)old_sca);
1386
8335713a
CB		 1387 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1388 old_sca, kvm->arch.sca);
5e044315 1389 return 0;
a6e2f683
ED		 1390}
1391
1392static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1393{
5e044315
ED		 1394 int rc;
1395
1396 if (id < KVM_S390_BSCA_CPU_SLOTS)
1397 return true;
1398 if (!sclp.has_esca)
1399 return false;
1400
1401 mutex_lock(&kvm->lock);
1402 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1403 mutex_unlock(&kvm->lock);
1404
1405 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1406}
1407
b0c632db
HC		 1408int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1409{
3c038e6b
DD		 1410 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1411 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1412 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1413 KVM_SYNC_GPRS |
9eed0735 1414 KVM_SYNC_ACRS |
b028ee3e
DH		 1415 KVM_SYNC_CRS |
1416 KVM_SYNC_ARCH0 |
1417 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1418 if (test_kvm_facility(vcpu->kvm, 64))
1419 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1420 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1421 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1422 */
1423 if (MACHINE_HAS_VX)
68c55750 1424 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1425 else
1426 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1427
1428 if (kvm_is_ucontrol(vcpu->kvm))
1429 return __kvm_ucontrol_vcpu_init(vcpu);
1430
b0c632db
HC		 1431 return 0;
1432}
1433
db0758b2
DH		 1434/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1435static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1436{
1437 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1438 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1439 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1440 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1441}
1442
1443/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1444static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1445{
1446 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1447 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1448 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1449 vcpu->arch.cputm_start = 0;
9c23a131 1450 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1451}
1452
1453/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1454static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1455{
1456 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1457 vcpu->arch.cputm_enabled = true;
1458 __start_cpu_timer_accounting(vcpu);
1459}
1460
1461/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1462static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1463{
1464 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1465 __stop_cpu_timer_accounting(vcpu);
1466 vcpu->arch.cputm_enabled = false;
1467}
1468
1469static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1470{
1471 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1472 __enable_cpu_timer_accounting(vcpu);
1473 preempt_enable();
1474}
1475
1476static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1477{
1478 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1479 __disable_cpu_timer_accounting(vcpu);
1480 preempt_enable();
1481}
1482
4287f247
DH		 1483/* set the cpu timer - may only be called from the VCPU thread itself */
1484void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1485{
db0758b2 1486 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1487 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1488 if (vcpu->arch.cputm_enabled)
1489 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1490 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1491 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1492 preempt_enable();
4287f247
DH		 1493}
1494
db0758b2 1495/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1496__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1497{
9c23a131 1498 unsigned int seq;
db0758b2 1499 __u64 value;
db0758b2
DH		 1500
1501 if (unlikely(!vcpu->arch.cputm_enabled))
1502 return vcpu->arch.sie_block->cputm;
1503
9c23a131
DH		 1504 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1505 do {
1506 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1507 /*
1508 * If the writer would ever execute a read in the critical
1509 * section, e.g. in irq context, we have a deadlock.
1510 */
1511 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1512 value = vcpu->arch.sie_block->cputm;
1513 /* if cputm_start is 0, accounting is being started/stopped */
1514 if (likely(vcpu->arch.cputm_start))
1515 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1516 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1517 preempt_enable();
db0758b2 1518 return value;
4287f247
DH		 1519}
1520
b0c632db
HC		 1521void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1522{
9977e886 1523 /* Save host register state */
d0164ee2 1524 save_fpu_regs();
9abc2a08
DH		 1525 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1526 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1527
6fd8e67d
DH		 1528 if (MACHINE_HAS_VX)
1529 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1530 else
1531 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1532 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1533 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1534 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1535 current->thread.fpu.fpc = 0;
1536
1537 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1538 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1539 gmap_enable(vcpu->arch.gmap);
805de8f4 1540 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1541 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1542 __start_cpu_timer_accounting(vcpu);
01a745ac 1543 vcpu->cpu = cpu;
b0c632db
HC		 1544}
1545
1546void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1547{
01a745ac 1548 vcpu->cpu = -1;
5ebda316 1549 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1550 __stop_cpu_timer_accounting(vcpu);
805de8f4 1551 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1552 gmap_disable(vcpu->arch.gmap);
9977e886 1553
9abc2a08 1554 /* Save guest register state */
d0164ee2 1555 save_fpu_regs();
9abc2a08 1556 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1557
9abc2a08
DH		 1558 /* Restore host register state */
1559 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1560 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1561
1562 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1563 restore_access_regs(vcpu->arch.host_acrs);
1564}
1565
1566static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1567{
1568 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1569 vcpu->arch.sie_block->gpsw.mask = 0UL;
1570 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1571 kvm_s390_set_prefix(vcpu, 0);
4287f247 1572 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1573 vcpu->arch.sie_block->ckc = 0UL;
1574 vcpu->arch.sie_block->todpr = 0;
1575 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1576 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1577 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1578 /* make sure the new fpc will be lazily loaded */
1579 save_fpu_regs();
1580 current->thread.fpu.fpc = 0;
b0c632db 1581 vcpu->arch.sie_block->gbea = 1;
672550fb 1582 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1583 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1584 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1585 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1586 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1587 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1588}
1589
31928aa5 1590void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1591{
72f25020 1592 mutex_lock(&vcpu->kvm->lock);
fdf03650 1593 preempt_disable();
72f25020 1594 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1595 preempt_enable();
72f25020 1596 mutex_unlock(&vcpu->kvm->lock);
25508824 1597 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1598 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1599 sca_add_vcpu(vcpu);
25508824
DH		 1600 }
1601
42897d86
MT		 1602}
1603
5102ee87
TK		 1604static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1605{
9d8d5786 1606 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1607 return;
1608
a374e892
TK		 1609 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1610
1611 if (vcpu->kvm->arch.crypto.aes_kw)
1612 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1613 if (vcpu->kvm->arch.crypto.dea_kw)
1614 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1615
5102ee87
TK		 1616 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1617}
1618
b31605c1
DD		 1619void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1620{
1621 free_page(vcpu->arch.sie_block->cbrlo);
1622 vcpu->arch.sie_block->cbrlo = 0;
1623}
1624
1625int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1626{
1627 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1628 if (!vcpu->arch.sie_block->cbrlo)
1629 return -ENOMEM;
1630
1631 vcpu->arch.sie_block->ecb2 |= 0x80;
1632 vcpu->arch.sie_block->ecb2 &= ~0x08;
1633 return 0;
1634}
1635
91520f1a
MM		 1636static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1637{
1638 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1639
1640 vcpu->arch.cpu_id = model->cpu_id;
1641 vcpu->arch.sie_block->ibc = model->ibc;
1642 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1643}
1644
b0c632db
HC		 1645int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1646{
b31605c1 1647 int rc = 0;
b31288fa 1648
9e6dabef
CH		 1649 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1650 CPUSTAT_SM |
a4a4f191
GH		 1651 CPUSTAT_STOPPED);
1652
53df84f8 1653 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1654 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1655 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1656 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1657
91520f1a
MM		 1658 kvm_s390_vcpu_setup_model(vcpu);
1659
fc34531d 1660 vcpu->arch.sie_block->ecb = 6;
9d8d5786 1661 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1662 vcpu->arch.sie_block->ecb |= 0x10;
1663
69d0d3a3 1664 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 1665 vcpu->arch.sie_block->eca = 0xC1002000U;
37c5f6c8 1666 if (sclp.has_siif)
217a4406 1667 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1668 if (sclp.has_sigpif)
ea5f4969 1669 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1670 if (test_kvm_facility(vcpu->kvm, 64))
1671 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1672 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1673 vcpu->arch.sie_block->eca |= 0x00020000;
1674 vcpu->arch.sie_block->ecd |= 0x20000000;
1675 }
c6e5f166 1676 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1677 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1678
e6db1d61 1679 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1680 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1681 if (rc)
1682 return rc;
b31288fa 1683 }
0ac96caf 1684 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1685 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1686
5102ee87
TK		 1687 kvm_s390_vcpu_crypto_setup(vcpu);
1688
b31605c1 1689 return rc;
b0c632db
HC		 1690}
1691
1692struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1693 unsigned int id)
1694{
4d47555a 1695 struct kvm_vcpu *vcpu;
7feb6bb8 1696 struct sie_page *sie_page;
4d47555a
CO		 1697 int rc = -EINVAL;
1698
4215825e 1699 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1700 goto out;
1701
1702 rc = -ENOMEM;
b0c632db 1703
b110feaf 1704 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1705 if (!vcpu)
4d47555a 1706 goto out;
b0c632db 1707
7feb6bb8
MM		 1708 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1709 if (!sie_page)
b0c632db
HC		 1710 goto out_free_cpu;
1711
7feb6bb8
MM		 1712 vcpu->arch.sie_block = &sie_page->sie_block;
1713 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1714
b0c632db 1715 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1716 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1717 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1718 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1719 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1720 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1721
b0c632db
HC		 1722 rc = kvm_vcpu_init(vcpu, kvm, id);
1723 if (rc)
9abc2a08 1724 goto out_free_sie_block;
8335713a 1725 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1726 vcpu->arch.sie_block);
ade38c31 1727 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1728
b0c632db 1729 return vcpu;
7b06bf2f
WY		 1730out_free_sie_block:
1731 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1732out_free_cpu:
b110feaf 1733 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1734out:
b0c632db
HC		 1735 return ERR_PTR(rc);
1736}
1737
b0c632db
HC		 1738int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1739{
9a022067 1740 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1741}
1742
27406cd5 1743void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1744{
805de8f4 1745 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1746 exit_sie(vcpu);
49b99e1e
CB		 1747}
1748
27406cd5 1749void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1750{
805de8f4 1751 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1752}
1753
8e236546
CB		 1754static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1755{
805de8f4 1756 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1757 exit_sie(vcpu);
8e236546
CB		 1758}
1759
1760static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1761{
9bf9fde2 1762 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1763}
1764
49b99e1e
CB		 1765/*
1766 * Kick a guest cpu out of SIE and wait until SIE is not running.
1767 * If the CPU is not running (e.g. waiting as idle) the function will
1768 * return immediately. */
1769void exit_sie(struct kvm_vcpu *vcpu)
1770{
805de8f4 1771 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1772 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1773 cpu_relax();
1774}
1775
8e236546
CB		 1776/* Kick a guest cpu out of SIE to process a request synchronously */
1777void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1778{
8e236546
CB		 1779 kvm_make_request(req, vcpu);
1780 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1781}
1782
2c70fe44
CB		 1783static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1784{
1785 int i;
1786 struct kvm *kvm = gmap->private;
1787 struct kvm_vcpu *vcpu;
1788
1789 kvm_for_each_vcpu(i, vcpu, kvm) {
1790 /* match against both prefix pages */
fda902cb 1791 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1792 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1793 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1794 }
1795 }
1796}
1797
b6d33834
CD		 1798int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1799{
1800 /* kvm common code refers to this, but never calls it */
1801 BUG();
1802 return 0;
1803}
1804
14eebd91
CO		 1805static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1806 struct kvm_one_reg *reg)
1807{
1808 int r = -EINVAL;
1809
1810 switch (reg->id) {
29b7c71b
CO		 1811 case KVM_REG_S390_TODPR:
1812 r = put_user(vcpu->arch.sie_block->todpr,
1813 (u32 __user *)reg->addr);
1814 break;
1815 case KVM_REG_S390_EPOCHDIFF:
1816 r = put_user(vcpu->arch.sie_block->epoch,
1817 (u64 __user *)reg->addr);
1818 break;
46a6dd1c 1819 case KVM_REG_S390_CPU_TIMER:
4287f247 1820 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 1821 (u64 __user *)reg->addr);
1822 break;
1823 case KVM_REG_S390_CLOCK_COMP:
1824 r = put_user(vcpu->arch.sie_block->ckc,
1825 (u64 __user *)reg->addr);
1826 break;
536336c2
DD		 1827 case KVM_REG_S390_PFTOKEN:
1828 r = put_user(vcpu->arch.pfault_token,
1829 (u64 __user *)reg->addr);
1830 break;
1831 case KVM_REG_S390_PFCOMPARE:
1832 r = put_user(vcpu->arch.pfault_compare,
1833 (u64 __user *)reg->addr);
1834 break;
1835 case KVM_REG_S390_PFSELECT:
1836 r = put_user(vcpu->arch.pfault_select,
1837 (u64 __user *)reg->addr);
1838 break;
672550fb
CB		 1839 case KVM_REG_S390_PP:
1840 r = put_user(vcpu->arch.sie_block->pp,
1841 (u64 __user *)reg->addr);
1842 break;
afa45ff5
CB		 1843 case KVM_REG_S390_GBEA:
1844 r = put_user(vcpu->arch.sie_block->gbea,
1845 (u64 __user *)reg->addr);
1846 break;
14eebd91
CO		 1847 default:
1848 break;
1849 }
1850
1851 return r;
1852}
1853
1854static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1855 struct kvm_one_reg *reg)
1856{
1857 int r = -EINVAL;
4287f247 1858 __u64 val;
14eebd91
CO		 1859
1860 switch (reg->id) {
29b7c71b
CO		 1861 case KVM_REG_S390_TODPR:
1862 r = get_user(vcpu->arch.sie_block->todpr,
1863 (u32 __user *)reg->addr);
1864 break;
1865 case KVM_REG_S390_EPOCHDIFF:
1866 r = get_user(vcpu->arch.sie_block->epoch,
1867 (u64 __user *)reg->addr);
1868 break;
46a6dd1c 1869 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 1870 r = get_user(val, (u64 __user *)reg->addr);
1871 if (!r)
1872 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 1873 break;
1874 case KVM_REG_S390_CLOCK_COMP:
1875 r = get_user(vcpu->arch.sie_block->ckc,
1876 (u64 __user *)reg->addr);
1877 break;
536336c2
DD		 1878 case KVM_REG_S390_PFTOKEN:
1879 r = get_user(vcpu->arch.pfault_token,
1880 (u64 __user *)reg->addr);
9fbd8082
DH		 1881 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1882 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1883 break;
1884 case KVM_REG_S390_PFCOMPARE:
1885 r = get_user(vcpu->arch.pfault_compare,
1886 (u64 __user *)reg->addr);
1887 break;
1888 case KVM_REG_S390_PFSELECT:
1889 r = get_user(vcpu->arch.pfault_select,
1890 (u64 __user *)reg->addr);
1891 break;
672550fb
CB		 1892 case KVM_REG_S390_PP:
1893 r = get_user(vcpu->arch.sie_block->pp,
1894 (u64 __user *)reg->addr);
1895 break;
afa45ff5
CB		 1896 case KVM_REG_S390_GBEA:
1897 r = get_user(vcpu->arch.sie_block->gbea,
1898 (u64 __user *)reg->addr);
1899 break;
14eebd91
CO		 1900 default:
1901 break;
1902 }
1903
1904 return r;
1905}
b6d33834 1906
b0c632db
HC		 1907static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1908{
b0c632db 1909 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1910 return 0;
1911}
1912
1913int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1914{
5a32c1af 1915 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1916 return 0;
1917}
1918
1919int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1920{
5a32c1af 1921 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1922 return 0;
1923}
1924
1925int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1926 struct kvm_sregs *sregs)
1927{
59674c1a 1928 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1929 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1930 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1931 return 0;
1932}
1933
1934int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1935 struct kvm_sregs *sregs)
1936{
59674c1a 1937 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1938 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1939 return 0;
1940}
1941
1942int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1943{
9abc2a08
DH		 1944 /* make sure the new values will be lazily loaded */
1945 save_fpu_regs();
4725c860
MS		 1946 if (test_fp_ctl(fpu->fpc))
1947 return -EINVAL;
9abc2a08
DH		 1948 current->thread.fpu.fpc = fpu->fpc;
1949 if (MACHINE_HAS_VX)
1950 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
1951 else
1952 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 1953 return 0;
1954}
1955
1956int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1957{
9abc2a08
DH		 1958 /* make sure we have the latest values */
1959 save_fpu_regs();
1960 if (MACHINE_HAS_VX)
1961 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
1962 else
1963 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
1964 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 1965 return 0;
1966}
1967
1968static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1969{
1970 int rc = 0;
1971
7a42fdc2 1972 if (!is_vcpu_stopped(vcpu))
b0c632db 1973 rc = -EBUSY;
d7b0b5eb
CO		 1974 else {
1975 vcpu->run->psw_mask = psw.mask;
1976 vcpu->run->psw_addr = psw.addr;
1977 }
b0c632db
HC		 1978 return rc;
1979}
1980
1981int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1982 struct kvm_translation *tr)
1983{
1984 return -EINVAL; /* not implemented yet */
1985}
1986
27291e21
DH		 1987#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1988 KVM_GUESTDBG_USE_HW_BP | \
1989 KVM_GUESTDBG_ENABLE)
1990
d0bfb940
JK		 1991int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1992 struct kvm_guest_debug *dbg)
b0c632db 1993{
27291e21
DH		 1994 int rc = 0;
1995
1996 vcpu->guest_debug = 0;
1997 kvm_s390_clear_bp_data(vcpu);
1998
2de3bfc2 1999 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 2000 return -EINVAL;
2001
2002 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2003 vcpu->guest_debug = dbg->control;
2004 /* enforce guest PER */
805de8f4 2005 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2006
2007 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2008 rc = kvm_s390_import_bp_data(vcpu, dbg);
2009 } else {
805de8f4 2010 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2011 vcpu->arch.guestdbg.last_bp = 0;
2012 }
2013
2014 if (rc) {
2015 vcpu->guest_debug = 0;
2016 kvm_s390_clear_bp_data(vcpu);
805de8f4 2017 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2018 }
2019
2020 return rc;
b0c632db
HC		 2021}
2022
62d9f0db
MT		 2023int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2024 struct kvm_mp_state *mp_state)
2025{
6352e4d2
DH		 2026 /* CHECK_STOP and LOAD are not supported yet */
2027 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2028 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2029}
2030
2031int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2032 struct kvm_mp_state *mp_state)
2033{
6352e4d2
DH		 2034 int rc = 0;
2035
2036 /* user space knows about this interface - let it control the state */
2037 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2038
2039 switch (mp_state->mp_state) {
2040 case KVM_MP_STATE_STOPPED:
2041 kvm_s390_vcpu_stop(vcpu);
2042 break;
2043 case KVM_MP_STATE_OPERATING:
2044 kvm_s390_vcpu_start(vcpu);
2045 break;
2046 case KVM_MP_STATE_LOAD:
2047 case KVM_MP_STATE_CHECK_STOP:
2048 /* fall through - CHECK_STOP and LOAD are not supported yet */
2049 default:
2050 rc = -ENXIO;
2051 }
2052
2053 return rc;
62d9f0db
MT		 2054}
2055
8ad35755
DH		 2056static bool ibs_enabled(struct kvm_vcpu *vcpu)
2057{
2058 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2059}
2060
2c70fe44
CB		 2061static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2062{
8ad35755 2063retry:
8e236546 2064 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2065 if (!vcpu->requests)
2066 return 0;
2c70fe44
CB		 2067 /*
2068 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2069 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2070 * This ensures that the ipte instruction for this request has
2071 * already finished. We might race against a second unmapper that
2072 * wants to set the blocking bit. Lets just retry the request loop.
2073 */
8ad35755 2074 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2075 int rc;
2076 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2077 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2078 PAGE_SIZE * 2);
2079 if (rc)
2080 return rc;
8ad35755 2081 goto retry;
2c70fe44 2082 }
8ad35755 2083
d3d692c8
DH		 2084 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2085 vcpu->arch.sie_block->ihcpu = 0xffff;
2086 goto retry;
2087 }
2088
8ad35755
DH		 2089 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2090 if (!ibs_enabled(vcpu)) {
2091 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2092 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2093 &vcpu->arch.sie_block->cpuflags);
2094 }
2095 goto retry;
2c70fe44 2096 }
8ad35755
DH		 2097
2098 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2099 if (ibs_enabled(vcpu)) {
2100 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2101 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2102 &vcpu->arch.sie_block->cpuflags);
2103 }
2104 goto retry;
2105 }
2106
0759d068
DH		 2107 /* nothing to do, just clear the request */
2108 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2109
2c70fe44
CB		 2110 return 0;
2111}
2112
25ed1675
DH		 2113void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2114{
2115 struct kvm_vcpu *vcpu;
2116 int i;
2117
2118 mutex_lock(&kvm->lock);
2119 preempt_disable();
2120 kvm->arch.epoch = tod - get_tod_clock();
2121 kvm_s390_vcpu_block_all(kvm);
2122 kvm_for_each_vcpu(i, vcpu, kvm)
2123 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2124 kvm_s390_vcpu_unblock_all(kvm);
2125 preempt_enable();
2126 mutex_unlock(&kvm->lock);
2127}
2128
fa576c58
TH		 2129/**
2130 * kvm_arch_fault_in_page - fault-in guest page if necessary
2131 * @vcpu: The corresponding virtual cpu
2132 * @gpa: Guest physical address
2133 * @writable: Whether the page should be writable or not
2134 *
2135 * Make sure that a guest page has been faulted-in on the host.
2136 *
2137 * Return: Zero on success, negative error code otherwise.
2138 */
2139long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2140{
527e30b4
MS		 2141 return gmap_fault(vcpu->arch.gmap, gpa,
2142 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2143}
2144
3c038e6b
DD		 2145static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2146 unsigned long token)
2147{
2148 struct kvm_s390_interrupt inti;
383d0b05 2149 struct kvm_s390_irq irq;
3c038e6b
DD		 2150
2151 if (start_token) {
383d0b05
JF		 2152 irq.u.ext.ext_params2 = token;
2153 irq.type = KVM_S390_INT_PFAULT_INIT;
2154 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2155 } else {
2156 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2157 inti.parm64 = token;
3c038e6b
DD		 2158 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2159 }
2160}
2161
2162void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2163 struct kvm_async_pf *work)
2164{
2165 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2166 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2167}
2168
2169void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2170 struct kvm_async_pf *work)
2171{
2172 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2173 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2174}
2175
2176void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2177 struct kvm_async_pf *work)
2178{
2179 /* s390 will always inject the page directly */
2180}
2181
2182bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2183{
2184 /*
2185 * s390 will always inject the page directly,
2186 * but we still want check_async_completion to cleanup
2187 */
2188 return true;
2189}
2190
2191static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2192{
2193 hva_t hva;
2194 struct kvm_arch_async_pf arch;
2195 int rc;
2196
2197 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2198 return 0;
2199 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2200 vcpu->arch.pfault_compare)
2201 return 0;
2202 if (psw_extint_disabled(vcpu))
2203 return 0;
9a022067 2204 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2205 return 0;
2206 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2207 return 0;
2208 if (!vcpu->arch.gmap->pfault_enabled)
2209 return 0;
2210
81480cc1
HC		 2211 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2212 hva += current->thread.gmap_addr & ~PAGE_MASK;
2213 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2214 return 0;
2215
2216 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2217 return rc;
2218}
2219
3fb4c40f 2220static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2221{
3fb4c40f 2222 int rc, cpuflags;
e168bf8d 2223
3c038e6b
DD		 2224 /*
2225 * On s390 notifications for arriving pages will be delivered directly
2226 * to the guest but the house keeping for completed pfaults is
2227 * handled outside the worker.
2228 */
2229 kvm_check_async_pf_completion(vcpu);
2230
7ec7c8c7
CB		 2231 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2232 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2233
2234 if (need_resched())
2235 schedule();
2236
d3a73acb 2237 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2238 s390_handle_mcck();
2239
79395031
JF		 2240 if (!kvm_is_ucontrol(vcpu->kvm)) {
2241 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2242 if (rc)
2243 return rc;
2244 }
0ff31867 2245
2c70fe44
CB		 2246 rc = kvm_s390_handle_requests(vcpu);
2247 if (rc)
2248 return rc;
2249
27291e21
DH		 2250 if (guestdbg_enabled(vcpu)) {
2251 kvm_s390_backup_guest_per_regs(vcpu);
2252 kvm_s390_patch_guest_per_regs(vcpu);
2253 }
2254
b0c632db 2255 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2256 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2257 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2258 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2259
3fb4c40f
TH		 2260 return 0;
2261}
2262
492d8642
TH		 2263static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2264{
56317920
DH		 2265 struct kvm_s390_pgm_info pgm_info = {
2266 .code = PGM_ADDRESSING,
2267 };
2268 u8 opcode, ilen;
492d8642
TH		 2269 int rc;
2270
2271 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2272 trace_kvm_s390_sie_fault(vcpu);
2273
2274 /*
2275 * We want to inject an addressing exception, which is defined as a
2276 * suppressing or terminating exception. However, since we came here
2277 * by a DAT access exception, the PSW still points to the faulting
2278 * instruction since DAT exceptions are nullifying. So we've got
2279 * to look up the current opcode to get the length of the instruction
2280 * to be able to forward the PSW.
2281 */
65977322 2282 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2283 ilen = insn_length(opcode);
9b0d721a
DH		 2284 if (rc < 0) {
2285 return rc;
2286 } else if (rc) {
2287 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2288 * Forward by arbitrary ilc, injection will take care of
2289 * nullification if necessary.
2290 */
2291 pgm_info = vcpu->arch.pgm;
2292 ilen = 4;
2293 }
56317920
DH		 2294 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2295 kvm_s390_forward_psw(vcpu, ilen);
2296 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2297}
2298
3fb4c40f
TH		 2299static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2300{
2b29a9fd
DD		 2301 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2302 vcpu->arch.sie_block->icptcode);
2303 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2304
27291e21
DH		 2305 if (guestdbg_enabled(vcpu))
2306 kvm_s390_restore_guest_per_regs(vcpu);
2307
7ec7c8c7
CB		 2308 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2309 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2310
2311 if (vcpu->arch.sie_block->icptcode > 0) {
2312 int rc = kvm_handle_sie_intercept(vcpu);
2313
2314 if (rc != -EOPNOTSUPP)
2315 return rc;
2316 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2317 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2318 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2319 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2320 return -EREMOTE;
2321 } else if (exit_reason != -EFAULT) {
2322 vcpu->stat.exit_null++;
2323 return 0;
210b1607
TH		 2324 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2325 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2326 vcpu->run->s390_ucontrol.trans_exc_code =
2327 current->thread.gmap_addr;
2328 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2329 return -EREMOTE;
24eb3a82 2330 } else if (current->thread.gmap_pfault) {
3c038e6b 2331 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2332 current->thread.gmap_pfault = 0;
71f116bf
DH		 2333 if (kvm_arch_setup_async_pf(vcpu))
2334 return 0;
2335 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2336 }
71f116bf 2337 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2338}
2339
2340static int __vcpu_run(struct kvm_vcpu *vcpu)
2341{
2342 int rc, exit_reason;
2343
800c1065
TH		 2344 /*
2345 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2346 * ning the guest), so that memslots (and other stuff) are protected
2347 */
2348 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2349
a76ccff6
TH		 2350 do {
2351 rc = vcpu_pre_run(vcpu);
2352 if (rc)
2353 break;
3fb4c40f 2354
800c1065 2355 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2356 /*
2357 * As PF_VCPU will be used in fault handler, between
2358 * guest_enter and guest_exit should be no uaccess.
2359 */
0097d12e
CB		 2360 local_irq_disable();
2361 __kvm_guest_enter();
db0758b2 2362 __disable_cpu_timer_accounting(vcpu);
0097d12e 2363 local_irq_enable();
a76ccff6
TH		 2364 exit_reason = sie64a(vcpu->arch.sie_block,
2365 vcpu->run->s.regs.gprs);
0097d12e 2366 local_irq_disable();
db0758b2 2367 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2368 __kvm_guest_exit();
2369 local_irq_enable();
800c1065 2370 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2371
2372 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2373 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2374
800c1065 2375 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2376 return rc;
b0c632db
HC		 2377}
2378
b028ee3e
DH		 2379static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2380{
2381 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2382 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2383 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2384 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2385 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2386 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2387 /* some control register changes require a tlb flush */
2388 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2389 }
2390 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2391 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2392 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2393 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2394 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2395 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2396 }
2397 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2398 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2399 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2400 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2401 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2402 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2403 }
2404 kvm_run->kvm_dirty_regs = 0;
2405}
2406
2407static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2408{
2409 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2410 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2411 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2412 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2413 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2414 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2415 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2416 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2417 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2418 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2419 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2420 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2421}
2422
b0c632db
HC		 2423int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2424{
8f2abe6a 2425 int rc;
b0c632db
HC		 2426 sigset_t sigsaved;
2427
27291e21
DH		 2428 if (guestdbg_exit_pending(vcpu)) {
2429 kvm_s390_prepare_debug_exit(vcpu);
2430 return 0;
2431 }
2432
b0c632db
HC		 2433 if (vcpu->sigset_active)
2434 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2435
6352e4d2
DH		 2436 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2437 kvm_s390_vcpu_start(vcpu);
2438 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2439 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2440 vcpu->vcpu_id);
2441 return -EINVAL;
2442 }
b0c632db 2443
b028ee3e 2444 sync_regs(vcpu, kvm_run);
db0758b2 2445 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2446
dab4079d 2447 might_fault();
a76ccff6 2448 rc = __vcpu_run(vcpu);
9ace903d 2449
b1d16c49
CE		 2450 if (signal_pending(current) && !rc) {
2451 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2452 rc = -EINTR;
b1d16c49 2453 }
8f2abe6a 2454
27291e21
DH		 2455 if (guestdbg_exit_pending(vcpu) && !rc) {
2456 kvm_s390_prepare_debug_exit(vcpu);
2457 rc = 0;
2458 }
2459
8f2abe6a 2460 if (rc == -EREMOTE) {
71f116bf 2461 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2462 rc = 0;
2463 }
b0c632db 2464
db0758b2 2465 disable_cpu_timer_accounting(vcpu);
b028ee3e 2466 store_regs(vcpu, kvm_run);
d7b0b5eb 2467
b0c632db
HC		 2468 if (vcpu->sigset_active)
2469 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2470
b0c632db 2471 vcpu->stat.exit_userspace++;
7e8e6ab4 2472 return rc;
b0c632db
HC		 2473}
2474
b0c632db
HC		 2475/*
2476 * store status at address
2477 * we use have two special cases:
2478 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2479 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2480 */
d0bce605 2481int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2482{
092670cd 2483 unsigned char archmode = 1;
9abc2a08 2484 freg_t fprs[NUM_FPRS];
fda902cb 2485 unsigned int px;
4287f247 2486 u64 clkcomp, cputm;
d0bce605 2487 int rc;
b0c632db 2488
d9a3a09a 2489 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2490 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2491 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2492 return -EFAULT;
d9a3a09a 2493 gpa = 0;
d0bce605
HC		 2494 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2495 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2496 return -EFAULT;
d9a3a09a
MS		 2497 gpa = px;
2498 } else
2499 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2500
2501 /* manually convert vector registers if necessary */
2502 if (MACHINE_HAS_VX) {
2503 convert_vx_to_fp(fprs, current->thread.fpu.vxrs);
2504 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2505 fprs, 128);
2506 } else {
2507 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2508 vcpu->run->s.regs.fprs, 128);
9abc2a08 2509 }
d9a3a09a 2510 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2511 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2512 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2513 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2514 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2515 &px, 4);
d9a3a09a 2516 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2517 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2518 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2519 &vcpu->arch.sie_block->todpr, 4);
4287f247 2520 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2521 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2522 &cputm, 8);
178bd789 2523 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2524 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2525 &clkcomp, 8);
d9a3a09a 2526 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2527 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2528 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2529 &vcpu->arch.sie_block->gcr, 128);
2530 return rc ? -EFAULT : 0;
b0c632db
HC		 2531}
2532
e879892c
TH		 2533int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2534{
2535 /*
2536 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2537 * copying in vcpu load/put. Lets update our copies before we save
2538 * it into the save area
2539 */
d0164ee2 2540 save_fpu_regs();
9abc2a08 2541 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2542 save_access_regs(vcpu->run->s.regs.acrs);
2543
2544 return kvm_s390_store_status_unloaded(vcpu, addr);
2545}
2546
bc17de7c
EF		 2547/*
2548 * store additional status at address
2549 */
2550int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2551 unsigned long gpa)
2552{
2553 /* Only bits 0-53 are used for address formation */
2554 if (!(gpa & ~0x3ff))
2555 return 0;
2556
2557 return write_guest_abs(vcpu, gpa & ~0x3ff,
2558 (void *)&vcpu->run->s.regs.vrs, 512);
2559}
2560
2561int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2562{
2563 if (!test_kvm_facility(vcpu->kvm, 129))
2564 return 0;
2565
2566 /*
2567 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2568 * copying in vcpu load/put. We can simply call save_fpu_regs()
2569 * to save the current register state because we are in the
2570 * middle of a load/put cycle.
2571 *
2572 * Let's update our copies before we save it into the save area.
bc17de7c 2573 */
d0164ee2 2574 save_fpu_regs();
bc17de7c
EF		 2575
2576 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2577}
2578
8ad35755
DH		 2579static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2580{
2581 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2582 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2583}
2584
2585static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2586{
2587 unsigned int i;
2588 struct kvm_vcpu *vcpu;
2589
2590 kvm_for_each_vcpu(i, vcpu, kvm) {
2591 __disable_ibs_on_vcpu(vcpu);
2592 }
2593}
2594
2595static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2596{
2597 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2598 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2599}
2600
6852d7b6
DH		 2601void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2602{
8ad35755
DH		 2603 int i, online_vcpus, started_vcpus = 0;
2604
2605 if (!is_vcpu_stopped(vcpu))
2606 return;
2607
6852d7b6 2608 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2609 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2610 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2611 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2612
2613 for (i = 0; i < online_vcpus; i++) {
2614 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2615 started_vcpus++;
2616 }
2617
2618 if (started_vcpus == 0) {
2619 /* we're the only active VCPU -> speed it up */
2620 __enable_ibs_on_vcpu(vcpu);
2621 } else if (started_vcpus == 1) {
2622 /*
2623 * As we are starting a second VCPU, we have to disable
2624 * the IBS facility on all VCPUs to remove potentially
2625 * oustanding ENABLE requests.
2626 */
2627 __disable_ibs_on_all_vcpus(vcpu->kvm);
2628 }
2629
805de8f4 2630 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2631 /*
2632 * Another VCPU might have used IBS while we were offline.
2633 * Let's play safe and flush the VCPU at startup.
2634 */
d3d692c8 2635 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2636 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2637 return;
6852d7b6
DH		 2638}
2639
2640void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2641{
8ad35755
DH		 2642 int i, online_vcpus, started_vcpus = 0;
2643 struct kvm_vcpu *started_vcpu = NULL;
2644
2645 if (is_vcpu_stopped(vcpu))
2646 return;
2647
6852d7b6 2648 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2649 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2650 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2651 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2652
32f5ff63 2653 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2654 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2655
805de8f4 2656 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2657 __disable_ibs_on_vcpu(vcpu);
2658
2659 for (i = 0; i < online_vcpus; i++) {
2660 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2661 started_vcpus++;
2662 started_vcpu = vcpu->kvm->vcpus[i];
2663 }
2664 }
2665
2666 if (started_vcpus == 1) {
2667 /*
2668 * As we only have one VCPU left, we want to enable the
2669 * IBS facility for that VCPU to speed it up.
2670 */
2671 __enable_ibs_on_vcpu(started_vcpu);
2672 }
2673
433b9ee4 2674 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2675 return;
6852d7b6
DH		 2676}
2677
d6712df9
CH		 2678static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2679 struct kvm_enable_cap *cap)
2680{
2681 int r;
2682
2683 if (cap->flags)
2684 return -EINVAL;
2685
2686 switch (cap->cap) {
fa6b7fe9
CH		 2687 case KVM_CAP_S390_CSS_SUPPORT:
2688 if (!vcpu->kvm->arch.css_support) {
2689 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2690 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2691 trace_kvm_s390_enable_css(vcpu->kvm);
2692 }
2693 r = 0;
2694 break;
d6712df9
CH		 2695 default:
2696 r = -EINVAL;
2697 break;
2698 }
2699 return r;
2700}
2701
41408c28
TH		 2702static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2703 struct kvm_s390_mem_op *mop)
2704{
2705 void __user *uaddr = (void __user *)mop->buf;
2706 void *tmpbuf = NULL;
2707 int r, srcu_idx;
2708 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2709 | KVM_S390_MEMOP_F_CHECK_ONLY;
2710
2711 if (mop->flags & ~supported_flags)
2712 return -EINVAL;
2713
2714 if (mop->size > MEM_OP_MAX_SIZE)
2715 return -E2BIG;
2716
2717 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2718 tmpbuf = vmalloc(mop->size);
2719 if (!tmpbuf)
2720 return -ENOMEM;
2721 }
2722
2723 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2724
2725 switch (mop->op) {
2726 case KVM_S390_MEMOP_LOGICAL_READ:
2727 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2728 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2729 mop->size, GACC_FETCH);
41408c28
TH		 2730 break;
2731 }
2732 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2733 if (r == 0) {
2734 if (copy_to_user(uaddr, tmpbuf, mop->size))
2735 r = -EFAULT;
2736 }
2737 break;
2738 case KVM_S390_MEMOP_LOGICAL_WRITE:
2739 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2740 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2741 mop->size, GACC_STORE);
41408c28
TH		 2742 break;
2743 }
2744 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2745 r = -EFAULT;
2746 break;
2747 }
2748 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2749 break;
2750 default:
2751 r = -EINVAL;
2752 }
2753
2754 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2755
2756 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2757 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2758
2759 vfree(tmpbuf);
2760 return r;
2761}
2762
b0c632db
HC		 2763long kvm_arch_vcpu_ioctl(struct file *filp,
2764 unsigned int ioctl, unsigned long arg)
2765{
2766 struct kvm_vcpu *vcpu = filp->private_data;
2767 void __user *argp = (void __user *)arg;
800c1065 2768 int idx;
bc923cc9 2769 long r;
b0c632db 2770
93736624 2771 switch (ioctl) {
47b43c52
JF		 2772 case KVM_S390_IRQ: {
2773 struct kvm_s390_irq s390irq;
2774
2775 r = -EFAULT;
2776 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2777 break;
2778 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2779 break;
2780 }
93736624 2781 case KVM_S390_INTERRUPT: {
ba5c1e9b 2782 struct kvm_s390_interrupt s390int;
383d0b05 2783 struct kvm_s390_irq s390irq;
ba5c1e9b 2784
93736624 2785 r = -EFAULT;
ba5c1e9b 2786 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2787 break;
383d0b05
JF		 2788 if (s390int_to_s390irq(&s390int, &s390irq))
2789 return -EINVAL;
2790 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2791 break;
ba5c1e9b 2792 }
b0c632db 2793 case KVM_S390_STORE_STATUS:
800c1065 2794 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2795 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2796 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2797 break;
b0c632db
HC		 2798 case KVM_S390_SET_INITIAL_PSW: {
2799 psw_t psw;
2800
bc923cc9 2801 r = -EFAULT;
b0c632db 2802 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2803 break;
2804 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2805 break;
b0c632db
HC		 2806 }
2807 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2808 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2809 break;
14eebd91
CO		 2810 case KVM_SET_ONE_REG:
2811 case KVM_GET_ONE_REG: {
2812 struct kvm_one_reg reg;
2813 r = -EFAULT;
2814 if (copy_from_user(&reg, argp, sizeof(reg)))
2815 break;
2816 if (ioctl == KVM_SET_ONE_REG)
2817 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2818 else
2819 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2820 break;
2821 }
27e0393f
CO		 2822#ifdef CONFIG_KVM_S390_UCONTROL
2823 case KVM_S390_UCAS_MAP: {
2824 struct kvm_s390_ucas_mapping ucasmap;
2825
2826 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2827 r = -EFAULT;
2828 break;
2829 }
2830
2831 if (!kvm_is_ucontrol(vcpu->kvm)) {
2832 r = -EINVAL;
2833 break;
2834 }
2835
2836 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2837 ucasmap.vcpu_addr, ucasmap.length);
2838 break;
2839 }
2840 case KVM_S390_UCAS_UNMAP: {
2841 struct kvm_s390_ucas_mapping ucasmap;
2842
2843 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2844 r = -EFAULT;
2845 break;
2846 }
2847
2848 if (!kvm_is_ucontrol(vcpu->kvm)) {
2849 r = -EINVAL;
2850 break;
2851 }
2852
2853 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2854 ucasmap.length);
2855 break;
2856 }
2857#endif
ccc7910f 2858 case KVM_S390_VCPU_FAULT: {
527e30b4 2859 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2860 break;
2861 }
d6712df9
CH		 2862 case KVM_ENABLE_CAP:
2863 {
2864 struct kvm_enable_cap cap;
2865 r = -EFAULT;
2866 if (copy_from_user(&cap, argp, sizeof(cap)))
2867 break;
2868 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2869 break;
2870 }
41408c28
TH		 2871 case KVM_S390_MEM_OP: {
2872 struct kvm_s390_mem_op mem_op;
2873
2874 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2875 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2876 else
2877 r = -EFAULT;
2878 break;
2879 }
816c7667
JF		 2880 case KVM_S390_SET_IRQ_STATE: {
2881 struct kvm_s390_irq_state irq_state;
2882
2883 r = -EFAULT;
2884 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2885 break;
2886 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2887 irq_state.len == 0 ||
2888 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2889 r = -EINVAL;
2890 break;
2891 }
2892 r = kvm_s390_set_irq_state(vcpu,
2893 (void __user *) irq_state.buf,
2894 irq_state.len);
2895 break;
2896 }
2897 case KVM_S390_GET_IRQ_STATE: {
2898 struct kvm_s390_irq_state irq_state;
2899
2900 r = -EFAULT;
2901 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2902 break;
2903 if (irq_state.len == 0) {
2904 r = -EINVAL;
2905 break;
2906 }
2907 r = kvm_s390_get_irq_state(vcpu,
2908 (__u8 __user *) irq_state.buf,
2909 irq_state.len);
2910 break;
2911 }
b0c632db 2912 default:
3e6afcf1 2913 r = -ENOTTY;
b0c632db 2914 }
bc923cc9 2915 return r;
b0c632db
HC		 2916}
2917
5b1c1493
CO		 2918int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2919{
2920#ifdef CONFIG_KVM_S390_UCONTROL
2921 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2922 && (kvm_is_ucontrol(vcpu->kvm))) {
2923 vmf->page = virt_to_page(vcpu->arch.sie_block);
2924 get_page(vmf->page);
2925 return 0;
2926 }
2927#endif
2928 return VM_FAULT_SIGBUS;
2929}
2930
5587027c
AK		 2931int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2932 unsigned long npages)
db3fe4eb
TY		 2933{
2934 return 0;
2935}
2936
b0c632db 2937/* Section: memory related */
f7784b8e
MT		 2938int kvm_arch_prepare_memory_region(struct kvm *kvm,
2939 struct kvm_memory_slot *memslot,
09170a49 2940 const struct kvm_userspace_memory_region *mem,
7b6195a9 2941 enum kvm_mr_change change)
b0c632db 2942{
dd2887e7
NW		 2943 /* A few sanity checks. We can have memory slots which have to be
2944 located/ended at a segment boundary (1MB). The memory in userland is
2945 ok to be fragmented into various different vmas. It is okay to mmap()
2946 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2947
598841ca 2948 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2949 return -EINVAL;
2950
598841ca 2951 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2952 return -EINVAL;
2953
a3a92c31
DD		 2954 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2955 return -EINVAL;
2956
f7784b8e
MT		 2957 return 0;
2958}
2959
2960void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 2961 const struct kvm_userspace_memory_region *mem,
8482644a 2962 const struct kvm_memory_slot *old,
f36f3f28 2963 const struct kvm_memory_slot *new,
8482644a 2964 enum kvm_mr_change change)
f7784b8e 2965{
f7850c92 2966 int rc;
f7784b8e 2967
2cef4deb
CB		 2968 /* If the basics of the memslot do not change, we do not want
2969 * to update the gmap. Every update causes several unnecessary
2970 * segment translation exceptions. This is usually handled just
2971 * fine by the normal fault handler + gmap, but it will also
2972 * cause faults on the prefix page of running guest CPUs.
2973 */
2974 if (old->userspace_addr == mem->userspace_addr &&
2975 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2976 old->npages * PAGE_SIZE == mem->memory_size)
2977 return;
598841ca
CO		 2978
2979 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2980 mem->guest_phys_addr, mem->memory_size);
2981 if (rc)
ea2cdd27 2982 pr_warn("failed to commit memory region\n");
598841ca 2983 return;
b0c632db
HC		 2984}
2985
b0c632db
HC		 2986static int __init kvm_s390_init(void)
2987{
07197fd0
DH		 2988 if (!sclp.has_sief2) {
2989 pr_info("SIE not available\n");
2990 return -ENODEV;
2991 }
2992
9d8d5786 2993 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2994}
2995
2996static void __exit kvm_s390_exit(void)
2997{
2998 kvm_exit();
2999}
3000
3001module_init(kvm_s390_init);
3002module_exit(kvm_s390_exit);
566af940
CH		 3003
3004/*
3005 * Enable autoloading of the kvm module.
3006 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3007 * since x86 takes a different approach.
3008 */
3009#include <linux/miscdevice.h>
3010MODULE_ALIAS_MISCDEV(KVM_MINOR);
3011MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)