projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/mm: simplify get_guest_storage_key
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
15c9705f 29#include <linux/bitmap.h>
cbb870c8 30#include <asm/asm-offsets.h>
b0c632db 31#include <asm/lowcore.h>
fdf03650 32#include <asm/etr.h>
b0c632db 33#include <asm/pgtable.h>
1e133ab2 34#include <asm/gmap.h>
f5daba1d 35#include <asm/nmi.h>
a0616cde 36#include <asm/switch_to.h>
6d3da241 37#include <asm/isc.h>
1526bf9c 38#include <asm/sclp.h>
0a763c78
DH		 39#include <asm/cpacf.h>
40#include <asm/etr.h>
8f2abe6a 41#include "kvm-s390.h"
b0c632db
HC		 42#include "gaccess.h"
43
ea2cdd27
DH		 44#define KMSG_COMPONENT "kvm-s390"
45#undef pr_fmt
46#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
47
5786fffa
CH		 48#define CREATE_TRACE_POINTS
49#include "trace.h"
ade38c31 50#include "trace-s390.h"
5786fffa 51
41408c28 52#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 53#define LOCAL_IRQS 32
54#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
55 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 56
b0c632db
HC		 57#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
58
59struct kvm_stats_debugfs_item debugfs_entries[] = {
60 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 61 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 62 { "exit_validity", VCPU_STAT(exit_validity) },
63 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
64 { "exit_external_request", VCPU_STAT(exit_external_request) },
65 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 66 { "exit_instruction", VCPU_STAT(exit_instruction) },
67 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
68 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 69 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 70 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 71 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 72 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 73 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 74 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 75 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 76 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
77 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 78 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 79 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 80 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
81 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
82 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
83 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
84 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
85 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
86 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 87 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 88 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
89 { "instruction_spx", VCPU_STAT(instruction_spx) },
90 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
91 { "instruction_stap", VCPU_STAT(instruction_stap) },
92 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 93 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 94 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
95 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 96 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 97 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
98 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 99 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
95ca2cb5 100 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
5288fbf0 101 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 102 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 103 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 104 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 105 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
106 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 107 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 108 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
109 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 110 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 111 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
112 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
113 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 114 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
115 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
116 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 117 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 118 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 119 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
175a5c9e
CB		 120 { "diagnose_258", VCPU_STAT(diagnose_258) },
121 { "diagnose_308", VCPU_STAT(diagnose_308) },
122 { "diagnose_500", VCPU_STAT(diagnose_500) },
b0c632db
HC		 123 { NULL }
124};
125
9d8d5786 126/* upper facilities limit for kvm */
60a37709
AY		 127unsigned long kvm_s390_fac_list_mask[16] = {
128 0xffe6000000000000UL,
129 0x005e000000000000UL,
9d8d5786 130};
b0c632db 131
9d8d5786 132unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 133{
9d8d5786
MM		 134 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
135 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 136}
137
15c9705f
DH		 138/* available cpu features supported by kvm */
139static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 140/* available subfunctions indicated via query / "test bit" */
141static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 142
9d8d5786 143static struct gmap_notifier gmap_notifier;
78f26131 144debug_info_t *kvm_s390_dbf;
9d8d5786 145
b0c632db 146/* Section: not file related */
13a34e06 147int kvm_arch_hardware_enable(void)
b0c632db
HC		 148{
149 /* every s390 is virtualization enabled ;-) */
10474ae8 150 return 0;
b0c632db
HC		 151}
152
2c70fe44
CB		 153static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
154
fdf03650
FZ		 155/*
156 * This callback is executed during stop_machine(). All CPUs are therefore
157 * temporarily stopped. In order not to change guest behavior, we have to
158 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
159 * so a CPU won't be stopped while calculating with the epoch.
160 */
161static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
162 void *v)
163{
164 struct kvm *kvm;
165 struct kvm_vcpu *vcpu;
166 int i;
167 unsigned long long *delta = v;
168
169 list_for_each_entry(kvm, &vm_list, vm_list) {
170 kvm->arch.epoch -= *delta;
171 kvm_for_each_vcpu(i, vcpu, kvm) {
172 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 173 if (vcpu->arch.cputm_enabled)
174 vcpu->arch.cputm_start += *delta;
fdf03650
FZ		 175 }
176 }
177 return NOTIFY_OK;
178}
179
180static struct notifier_block kvm_clock_notifier = {
181 .notifier_call = kvm_clock_sync,
182};
183
b0c632db
HC		 184int kvm_arch_hardware_setup(void)
185{
2c70fe44
CB		 186 gmap_notifier.notifier_call = kvm_gmap_notifier;
187 gmap_register_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 188 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
189 &kvm_clock_notifier);
b0c632db
HC		 190 return 0;
191}
192
193void kvm_arch_hardware_unsetup(void)
194{
2c70fe44 195 gmap_unregister_ipte_notifier(&gmap_notifier);
fdf03650
FZ		 196 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
197 &kvm_clock_notifier);
b0c632db
HC		 198}
199
22be5a13
DH		 200static void allow_cpu_feat(unsigned long nr)
201{
202 set_bit_inv(nr, kvm_s390_available_cpu_feat);
203}
204
0a763c78
DH		 205static inline int plo_test_bit(unsigned char nr)
206{
207 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
208 int cc = 3; /* subfunction not available */
209
210 asm volatile(
211 /* Parameter registers are ignored for "test bit" */
212 " plo 0,0,0,0(0)\n"
213 " ipm %0\n"
214 " srl %0,28\n"
215 : "=d" (cc)
216 : "d" (r0)
217 : "cc");
218 return cc == 0;
219}
220
22be5a13
DH		 221static void kvm_s390_cpu_feat_init(void)
222{
0a763c78
DH		 223 int i;
224
225 for (i = 0; i < 256; ++i) {
226 if (plo_test_bit(i))
227 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
228 }
229
230 if (test_facility(28)) /* TOD-clock steering */
231 etr_ptff(kvm_s390_available_subfunc.ptff, ETR_PTFF_QAF);
232
233 if (test_facility(17)) { /* MSA */
234 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
235 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
236 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
237 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
238 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
239 }
240 if (test_facility(76)) /* MSA3 */
241 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
242 if (test_facility(77)) { /* MSA4 */
243 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
244 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
245 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
246 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
247 }
248 if (test_facility(57)) /* MSA5 */
249 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
250
22be5a13
DH		 251 if (MACHINE_HAS_ESOP)
252 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
253}
254
b0c632db
HC		 255int kvm_arch_init(void *opaque)
256{
78f26131
CB		 257 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
258 if (!kvm_s390_dbf)
259 return -ENOMEM;
260
261 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
262 debug_unregister(kvm_s390_dbf);
263 return -ENOMEM;
264 }
265
22be5a13
DH		 266 kvm_s390_cpu_feat_init();
267
84877d93
CH		 268 /* Register floating interrupt controller interface. */
269 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 270}
271
78f26131
CB		 272void kvm_arch_exit(void)
273{
274 debug_unregister(kvm_s390_dbf);
275}
276
b0c632db
HC		 277/* Section: device related */
278long kvm_arch_dev_ioctl(struct file *filp,
279 unsigned int ioctl, unsigned long arg)
280{
281 if (ioctl == KVM_S390_ENABLE_SIE)
282 return s390_enable_sie();
283 return -EINVAL;
284}
285
784aa3d7 286int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 287{
d7b0b5eb
CO		 288 int r;
289
2bd0ac4e 290 switch (ext) {
d7b0b5eb 291 case KVM_CAP_S390_PSW:
b6cf8788 292 case KVM_CAP_S390_GMAP:
52e16b18 293 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 294#ifdef CONFIG_KVM_S390_UCONTROL
295 case KVM_CAP_S390_UCONTROL:
296#endif
3c038e6b 297 case KVM_CAP_ASYNC_PF:
60b413c9 298 case KVM_CAP_SYNC_REGS:
14eebd91 299 case KVM_CAP_ONE_REG:
d6712df9 300 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 301 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 302 case KVM_CAP_IOEVENTFD:
c05c4186 303 case KVM_CAP_DEVICE_CTRL:
d938dc55 304 case KVM_CAP_ENABLE_CAP_VM:
78599d90 305 case KVM_CAP_S390_IRQCHIP:
f2061656 306 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 307 case KVM_CAP_MP_STATE:
47b43c52 308 case KVM_CAP_S390_INJECT_IRQ:
2444b352 309 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 310 case KVM_CAP_S390_USER_STSI:
30ee2a98 311 case KVM_CAP_S390_SKEYS:
816c7667 312 case KVM_CAP_S390_IRQ_STATE:
d7b0b5eb
CO		 313 r = 1;
314 break;
41408c28
TH		 315 case KVM_CAP_S390_MEM_OP:
316 r = MEM_OP_MAX_SIZE;
317 break;
e726b1bd
CB		 318 case KVM_CAP_NR_VCPUS:
319 case KVM_CAP_MAX_VCPUS:
76a6dd72
DH		 320 r = KVM_S390_BSCA_CPU_SLOTS;
321 if (sclp.has_esca && sclp.has_64bscao)
322 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 323 break;
e1e2e605
NW		 324 case KVM_CAP_NR_MEMSLOTS:
325 r = KVM_USER_MEM_SLOTS;
326 break;
1526bf9c 327 case KVM_CAP_S390_COW:
abf09bed 328 r = MACHINE_HAS_ESOP;
1526bf9c 329 break;
68c55750
EF		 330 case KVM_CAP_S390_VECTOR_REGISTERS:
331 r = MACHINE_HAS_VX;
332 break;
c6e5f166
FZ		 333 case KVM_CAP_S390_RI:
334 r = test_facility(64);
335 break;
2bd0ac4e 336 default:
d7b0b5eb 337 r = 0;
2bd0ac4e 338 }
d7b0b5eb 339 return r;
b0c632db
HC		 340}
341
15f36ebd
JH		 342static void kvm_s390_sync_dirty_log(struct kvm *kvm,
343 struct kvm_memory_slot *memslot)
344{
345 gfn_t cur_gfn, last_gfn;
346 unsigned long address;
347 struct gmap *gmap = kvm->arch.gmap;
348
15f36ebd
JH		 349 /* Loop over all guest pages */
350 last_gfn = memslot->base_gfn + memslot->npages;
351 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
352 address = gfn_to_hva_memslot(memslot, cur_gfn);
353
1e133ab2 354 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 355 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 356 if (fatal_signal_pending(current))
357 return;
70c88a00 358 cond_resched();
15f36ebd 359 }
15f36ebd
JH		 360}
361
b0c632db 362/* Section: vm related */
a6e2f683
ED		 363static void sca_del_vcpu(struct kvm_vcpu *vcpu);
364
b0c632db
HC		 365/*
366 * Get (and clear) the dirty memory log for a memory slot.
367 */
368int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
369 struct kvm_dirty_log *log)
370{
15f36ebd
JH		 371 int r;
372 unsigned long n;
9f6b8029 373 struct kvm_memslots *slots;
15f36ebd
JH		 374 struct kvm_memory_slot *memslot;
375 int is_dirty = 0;
376
377 mutex_lock(&kvm->slots_lock);
378
379 r = -EINVAL;
380 if (log->slot >= KVM_USER_MEM_SLOTS)
381 goto out;
382
9f6b8029
PB		 383 slots = kvm_memslots(kvm);
384 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 385 r = -ENOENT;
386 if (!memslot->dirty_bitmap)
387 goto out;
388
389 kvm_s390_sync_dirty_log(kvm, memslot);
390 r = kvm_get_dirty_log(kvm, log, &is_dirty);
391 if (r)
392 goto out;
393
394 /* Clear the dirty log */
395 if (is_dirty) {
396 n = kvm_dirty_bitmap_bytes(memslot);
397 memset(memslot->dirty_bitmap, 0, n);
398 }
399 r = 0;
400out:
401 mutex_unlock(&kvm->slots_lock);
402 return r;
b0c632db
HC		 403}
404
d938dc55
CH		 405static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
406{
407 int r;
408
409 if (cap->flags)
410 return -EINVAL;
411
412 switch (cap->cap) {
84223598 413 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 414 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 415 kvm->arch.use_irqchip = 1;
416 r = 0;
417 break;
2444b352 418 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 419 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 420 kvm->arch.user_sigp = 1;
421 r = 0;
422 break;
68c55750 423 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b
DH		 424 mutex_lock(&kvm->lock);
425 if (atomic_read(&kvm->online_vcpus)) {
426 r = -EBUSY;
427 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 428 set_kvm_facility(kvm->arch.model.fac_mask, 129);
429 set_kvm_facility(kvm->arch.model.fac_list, 129);
18280d8b
MM		 430 r = 0;
431 } else
432 r = -EINVAL;
5967c17b 433 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 434 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
435 r ? "(not available)" : "(success)");
68c55750 436 break;
c6e5f166
FZ		 437 case KVM_CAP_S390_RI:
438 r = -EINVAL;
439 mutex_lock(&kvm->lock);
440 if (atomic_read(&kvm->online_vcpus)) {
441 r = -EBUSY;
442 } else if (test_facility(64)) {
c54f0d6a
DH		 443 set_kvm_facility(kvm->arch.model.fac_mask, 64);
444 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 445 r = 0;
446 }
447 mutex_unlock(&kvm->lock);
448 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
449 r ? "(not available)" : "(success)");
450 break;
e44fc8c9 451 case KVM_CAP_S390_USER_STSI:
c92ea7b9 452 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 453 kvm->arch.user_stsi = 1;
454 r = 0;
455 break;
d938dc55
CH		 456 default:
457 r = -EINVAL;
458 break;
459 }
460 return r;
461}
462
8c0a7ce6
DD		 463static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
464{
465 int ret;
466
467 switch (attr->attr) {
468 case KVM_S390_VM_MEM_LIMIT_SIZE:
469 ret = 0;
c92ea7b9 470 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 471 kvm->arch.mem_limit);
472 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 473 ret = -EFAULT;
474 break;
475 default:
476 ret = -ENXIO;
477 break;
478 }
479 return ret;
480}
481
482static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 483{
484 int ret;
485 unsigned int idx;
486 switch (attr->attr) {
487 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 488 ret = -ENXIO;
c24cc9c8 489 if (!sclp.has_cmma)
e6db1d61
DD		 490 break;
491
4f718eab 492 ret = -EBUSY;
c92ea7b9 493 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab
DD		 494 mutex_lock(&kvm->lock);
495 if (atomic_read(&kvm->online_vcpus) == 0) {
496 kvm->arch.use_cmma = 1;
497 ret = 0;
498 }
499 mutex_unlock(&kvm->lock);
500 break;
501 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 502 ret = -ENXIO;
503 if (!sclp.has_cmma)
504 break;
c3489155
DD		 505 ret = -EINVAL;
506 if (!kvm->arch.use_cmma)
507 break;
508
c92ea7b9 509 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 510 mutex_lock(&kvm->lock);
511 idx = srcu_read_lock(&kvm->srcu);
a13cff31 512 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 513 srcu_read_unlock(&kvm->srcu, idx);
514 mutex_unlock(&kvm->lock);
515 ret = 0;
516 break;
8c0a7ce6
DD		 517 case KVM_S390_VM_MEM_LIMIT_SIZE: {
518 unsigned long new_limit;
519
520 if (kvm_is_ucontrol(kvm))
521 return -EINVAL;
522
523 if (get_user(new_limit, (u64 __user *)attr->addr))
524 return -EFAULT;
525
a3a92c31
DD		 526 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
527 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 528 return -E2BIG;
529
a3a92c31
DD		 530 if (!new_limit)
531 return -EINVAL;
532
533 /* gmap_alloc takes last usable address */
534 if (new_limit != KVM_S390_NO_MEM_LIMIT)
535 new_limit -= 1;
536
8c0a7ce6
DD		 537 ret = -EBUSY;
538 mutex_lock(&kvm->lock);
539 if (atomic_read(&kvm->online_vcpus) == 0) {
540 /* gmap_alloc will round the limit up */
541 struct gmap *new = gmap_alloc(current->mm, new_limit);
542
543 if (!new) {
544 ret = -ENOMEM;
545 } else {
546 gmap_free(kvm->arch.gmap);
547 new->private = kvm;
548 kvm->arch.gmap = new;
549 ret = 0;
550 }
551 }
552 mutex_unlock(&kvm->lock);
a3a92c31
DD		 553 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
554 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
555 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 556 break;
557 }
4f718eab
DD		 558 default:
559 ret = -ENXIO;
560 break;
561 }
562 return ret;
563}
564
a374e892
TK		 565static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
566
567static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
568{
569 struct kvm_vcpu *vcpu;
570 int i;
571
9d8d5786 572 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 573 return -EINVAL;
574
575 mutex_lock(&kvm->lock);
576 switch (attr->attr) {
577 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
578 get_random_bytes(
579 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
580 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
581 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 582 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 583 break;
584 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
585 get_random_bytes(
586 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
587 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
588 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 589 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 590 break;
591 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
592 kvm->arch.crypto.aes_kw = 0;
593 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
594 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 595 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 596 break;
597 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
598 kvm->arch.crypto.dea_kw = 0;
599 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
600 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 601 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 602 break;
603 default:
604 mutex_unlock(&kvm->lock);
605 return -ENXIO;
606 }
607
608 kvm_for_each_vcpu(i, vcpu, kvm) {
609 kvm_s390_vcpu_crypto_setup(vcpu);
610 exit_sie(vcpu);
611 }
612 mutex_unlock(&kvm->lock);
613 return 0;
614}
615
72f25020
JH		 616static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
617{
618 u8 gtod_high;
619
620 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
621 sizeof(gtod_high)))
622 return -EFAULT;
623
624 if (gtod_high != 0)
625 return -EINVAL;
58c383c6 626 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 627
628 return 0;
629}
630
631static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
632{
5a3d883a 633 u64 gtod;
72f25020
JH		 634
635 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
636 return -EFAULT;
637
25ed1675 638 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 639 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 640 return 0;
641}
642
643static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
644{
645 int ret;
646
647 if (attr->flags)
648 return -EINVAL;
649
650 switch (attr->attr) {
651 case KVM_S390_VM_TOD_HIGH:
652 ret = kvm_s390_set_tod_high(kvm, attr);
653 break;
654 case KVM_S390_VM_TOD_LOW:
655 ret = kvm_s390_set_tod_low(kvm, attr);
656 break;
657 default:
658 ret = -ENXIO;
659 break;
660 }
661 return ret;
662}
663
664static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
665{
666 u8 gtod_high = 0;
667
668 if (copy_to_user((void __user *)attr->addr, &gtod_high,
669 sizeof(gtod_high)))
670 return -EFAULT;
58c383c6 671 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 672
673 return 0;
674}
675
676static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
677{
5a3d883a 678 u64 gtod;
72f25020 679
60417fcc 680 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 681 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
682 return -EFAULT;
58c383c6 683 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 684
685 return 0;
686}
687
688static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
689{
690 int ret;
691
692 if (attr->flags)
693 return -EINVAL;
694
695 switch (attr->attr) {
696 case KVM_S390_VM_TOD_HIGH:
697 ret = kvm_s390_get_tod_high(kvm, attr);
698 break;
699 case KVM_S390_VM_TOD_LOW:
700 ret = kvm_s390_get_tod_low(kvm, attr);
701 break;
702 default:
703 ret = -ENXIO;
704 break;
705 }
706 return ret;
707}
708
658b6eda
MM		 709static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
710{
711 struct kvm_s390_vm_cpu_processor *proc;
053dd230 712 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 713 int ret = 0;
714
715 mutex_lock(&kvm->lock);
716 if (atomic_read(&kvm->online_vcpus)) {
717 ret = -EBUSY;
718 goto out;
719 }
720 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
721 if (!proc) {
722 ret = -ENOMEM;
723 goto out;
724 }
725 if (!copy_from_user(proc, (void __user *)attr->addr,
726 sizeof(*proc))) {
9bb0ec09 727 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 728 lowest_ibc = sclp.ibc >> 16 & 0xfff;
729 unblocked_ibc = sclp.ibc & 0xfff;
730 if (lowest_ibc) {
731 if (proc->ibc > unblocked_ibc)
732 kvm->arch.model.ibc = unblocked_ibc;
733 else if (proc->ibc < lowest_ibc)
734 kvm->arch.model.ibc = lowest_ibc;
735 else
736 kvm->arch.model.ibc = proc->ibc;
737 }
c54f0d6a 738 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda
MM		 739 S390_ARCH_FAC_LIST_SIZE_BYTE);
740 } else
741 ret = -EFAULT;
742 kfree(proc);
743out:
744 mutex_unlock(&kvm->lock);
745 return ret;
746}
747
15c9705f
DH		 748static int kvm_s390_set_processor_feat(struct kvm *kvm,
749 struct kvm_device_attr *attr)
750{
751 struct kvm_s390_vm_cpu_feat data;
752 int ret = -EBUSY;
753
754 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
755 return -EFAULT;
756 if (!bitmap_subset((unsigned long *) data.feat,
757 kvm_s390_available_cpu_feat,
758 KVM_S390_VM_CPU_FEAT_NR_BITS))
759 return -EINVAL;
760
761 mutex_lock(&kvm->lock);
762 if (!atomic_read(&kvm->online_vcpus)) {
763 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
764 KVM_S390_VM_CPU_FEAT_NR_BITS);
765 ret = 0;
766 }
767 mutex_unlock(&kvm->lock);
768 return ret;
769}
770
0a763c78
DH		 771static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
772 struct kvm_device_attr *attr)
773{
774 /*
775 * Once supported by kernel + hw, we have to store the subfunctions
776 * in kvm->arch and remember that user space configured them.
777 */
778 return -ENXIO;
779}
780
658b6eda
MM		 781static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
782{
783 int ret = -ENXIO;
784
785 switch (attr->attr) {
786 case KVM_S390_VM_CPU_PROCESSOR:
787 ret = kvm_s390_set_processor(kvm, attr);
788 break;
15c9705f
DH		 789 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
790 ret = kvm_s390_set_processor_feat(kvm, attr);
791 break;
0a763c78
DH		 792 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
793 ret = kvm_s390_set_processor_subfunc(kvm, attr);
794 break;
658b6eda
MM		 795 }
796 return ret;
797}
798
799static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
800{
801 struct kvm_s390_vm_cpu_processor *proc;
802 int ret = 0;
803
804 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
805 if (!proc) {
806 ret = -ENOMEM;
807 goto out;
808 }
9bb0ec09 809 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 810 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 811 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
812 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 813 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
814 ret = -EFAULT;
815 kfree(proc);
816out:
817 return ret;
818}
819
820static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
821{
822 struct kvm_s390_vm_cpu_machine *mach;
823 int ret = 0;
824
825 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
826 if (!mach) {
827 ret = -ENOMEM;
828 goto out;
829 }
830 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 831 mach->ibc = sclp.ibc;
c54f0d6a 832 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 833 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 834 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 835 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 836 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
837 ret = -EFAULT;
838 kfree(mach);
839out:
840 return ret;
841}
842
15c9705f
DH		 843static int kvm_s390_get_processor_feat(struct kvm *kvm,
844 struct kvm_device_attr *attr)
845{
846 struct kvm_s390_vm_cpu_feat data;
847
848 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
849 KVM_S390_VM_CPU_FEAT_NR_BITS);
850 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
851 return -EFAULT;
852 return 0;
853}
854
855static int kvm_s390_get_machine_feat(struct kvm *kvm,
856 struct kvm_device_attr *attr)
857{
858 struct kvm_s390_vm_cpu_feat data;
859
860 bitmap_copy((unsigned long *) data.feat,
861 kvm_s390_available_cpu_feat,
862 KVM_S390_VM_CPU_FEAT_NR_BITS);
863 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
864 return -EFAULT;
865 return 0;
866}
867
0a763c78
DH		 868static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
869 struct kvm_device_attr *attr)
870{
871 /*
872 * Once we can actually configure subfunctions (kernel + hw support),
873 * we have to check if they were already set by user space, if so copy
874 * them from kvm->arch.
875 */
876 return -ENXIO;
877}
878
879static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
880 struct kvm_device_attr *attr)
881{
882 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
883 sizeof(struct kvm_s390_vm_cpu_subfunc)))
884 return -EFAULT;
885 return 0;
886}
658b6eda
MM		 887static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
888{
889 int ret = -ENXIO;
890
891 switch (attr->attr) {
892 case KVM_S390_VM_CPU_PROCESSOR:
893 ret = kvm_s390_get_processor(kvm, attr);
894 break;
895 case KVM_S390_VM_CPU_MACHINE:
896 ret = kvm_s390_get_machine(kvm, attr);
897 break;
15c9705f
DH		 898 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
899 ret = kvm_s390_get_processor_feat(kvm, attr);
900 break;
901 case KVM_S390_VM_CPU_MACHINE_FEAT:
902 ret = kvm_s390_get_machine_feat(kvm, attr);
903 break;
0a763c78
DH		 904 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
905 ret = kvm_s390_get_processor_subfunc(kvm, attr);
906 break;
907 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
908 ret = kvm_s390_get_machine_subfunc(kvm, attr);
909 break;
658b6eda
MM		 910 }
911 return ret;
912}
913
f2061656
DD		 914static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
915{
916 int ret;
917
918 switch (attr->group) {
4f718eab 919 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 920 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 921 break;
72f25020
JH		 922 case KVM_S390_VM_TOD:
923 ret = kvm_s390_set_tod(kvm, attr);
924 break;
658b6eda
MM		 925 case KVM_S390_VM_CPU_MODEL:
926 ret = kvm_s390_set_cpu_model(kvm, attr);
927 break;
a374e892
TK		 928 case KVM_S390_VM_CRYPTO:
929 ret = kvm_s390_vm_set_crypto(kvm, attr);
930 break;
f2061656
DD		 931 default:
932 ret = -ENXIO;
933 break;
934 }
935
936 return ret;
937}
938
939static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
940{
8c0a7ce6
DD		 941 int ret;
942
943 switch (attr->group) {
944 case KVM_S390_VM_MEM_CTRL:
945 ret = kvm_s390_get_mem_control(kvm, attr);
946 break;
72f25020
JH		 947 case KVM_S390_VM_TOD:
948 ret = kvm_s390_get_tod(kvm, attr);
949 break;
658b6eda
MM		 950 case KVM_S390_VM_CPU_MODEL:
951 ret = kvm_s390_get_cpu_model(kvm, attr);
952 break;
8c0a7ce6
DD		 953 default:
954 ret = -ENXIO;
955 break;
956 }
957
958 return ret;
f2061656
DD		 959}
960
961static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
962{
963 int ret;
964
965 switch (attr->group) {
4f718eab
DD		 966 case KVM_S390_VM_MEM_CTRL:
967 switch (attr->attr) {
968 case KVM_S390_VM_MEM_ENABLE_CMMA:
969 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 970 ret = sclp.has_cmma ? 0 : -ENXIO;
971 break;
8c0a7ce6 972 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 973 ret = 0;
974 break;
975 default:
976 ret = -ENXIO;
977 break;
978 }
979 break;
72f25020
JH		 980 case KVM_S390_VM_TOD:
981 switch (attr->attr) {
982 case KVM_S390_VM_TOD_LOW:
983 case KVM_S390_VM_TOD_HIGH:
984 ret = 0;
985 break;
986 default:
987 ret = -ENXIO;
988 break;
989 }
990 break;
658b6eda
MM		 991 case KVM_S390_VM_CPU_MODEL:
992 switch (attr->attr) {
993 case KVM_S390_VM_CPU_PROCESSOR:
994 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 995 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
996 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 997 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 998 ret = 0;
999 break;
0a763c78
DH		 1000 /* configuring subfunctions is not supported yet */
1001 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1002 default:
1003 ret = -ENXIO;
1004 break;
1005 }
1006 break;
a374e892
TK		 1007 case KVM_S390_VM_CRYPTO:
1008 switch (attr->attr) {
1009 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1010 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1011 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1012 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1013 ret = 0;
1014 break;
1015 default:
1016 ret = -ENXIO;
1017 break;
1018 }
1019 break;
f2061656
DD		 1020 default:
1021 ret = -ENXIO;
1022 break;
1023 }
1024
1025 return ret;
1026}
1027
30ee2a98
JH		 1028static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1029{
1030 uint8_t *keys;
1031 uint64_t hva;
1032 unsigned long curkey;
1033 int i, r = 0;
1034
1035 if (args->flags != 0)
1036 return -EINVAL;
1037
1038 /* Is this guest using storage keys? */
1039 if (!mm_use_skey(current->mm))
1040 return KVM_S390_GET_SKEYS_NONE;
1041
1042 /* Enforce sane limit on memory allocation */
1043 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1044 return -EINVAL;
1045
1046 keys = kmalloc_array(args->count, sizeof(uint8_t),
1047 GFP_KERNEL | __GFP_NOWARN);
1048 if (!keys)
1049 keys = vmalloc(sizeof(uint8_t) * args->count);
1050 if (!keys)
1051 return -ENOMEM;
1052
d3ed1cee 1053 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1054 for (i = 0; i < args->count; i++) {
1055 hva = gfn_to_hva(kvm, args->start_gfn + i);
1056 if (kvm_is_error_hva(hva)) {
1057 r = -EFAULT;
d3ed1cee 1058 break;
30ee2a98
JH		 1059 }
1060
1061 curkey = get_guest_storage_key(current->mm, hva);
1062 if (IS_ERR_VALUE(curkey)) {
1063 r = curkey;
d3ed1cee 1064 break;
30ee2a98
JH		 1065 }
1066 keys[i] = curkey;
1067 }
d3ed1cee
MS		 1068 up_read(&current->mm->mmap_sem);
1069
1070 if (!r) {
1071 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1072 sizeof(uint8_t) * args->count);
1073 if (r)
1074 r = -EFAULT;
1075 }
30ee2a98 1076
30ee2a98
JH		 1077 kvfree(keys);
1078 return r;
1079}
1080
1081static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1082{
1083 uint8_t *keys;
1084 uint64_t hva;
1085 int i, r = 0;
1086
1087 if (args->flags != 0)
1088 return -EINVAL;
1089
1090 /* Enforce sane limit on memory allocation */
1091 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1092 return -EINVAL;
1093
1094 keys = kmalloc_array(args->count, sizeof(uint8_t),
1095 GFP_KERNEL | __GFP_NOWARN);
1096 if (!keys)
1097 keys = vmalloc(sizeof(uint8_t) * args->count);
1098 if (!keys)
1099 return -ENOMEM;
1100
1101 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1102 sizeof(uint8_t) * args->count);
1103 if (r) {
1104 r = -EFAULT;
1105 goto out;
1106 }
1107
1108 /* Enable storage key handling for the guest */
14d4a425
DD		 1109 r = s390_enable_skey();
1110 if (r)
1111 goto out;
30ee2a98 1112
d3ed1cee 1113 down_read(&current->mm->mmap_sem);
30ee2a98
JH		 1114 for (i = 0; i < args->count; i++) {
1115 hva = gfn_to_hva(kvm, args->start_gfn + i);
1116 if (kvm_is_error_hva(hva)) {
1117 r = -EFAULT;
d3ed1cee 1118 break;
30ee2a98
JH		 1119 }
1120
1121 /* Lowest order bit is reserved */
1122 if (keys[i] & 0x01) {
1123 r = -EINVAL;
d3ed1cee 1124 break;
30ee2a98
JH		 1125 }
1126
1127 r = set_guest_storage_key(current->mm, hva,
1128 (unsigned long)keys[i], 0);
1129 if (r)
d3ed1cee 1130 break;
30ee2a98 1131 }
d3ed1cee 1132 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1133out:
1134 kvfree(keys);
1135 return r;
1136}
1137
b0c632db
HC		 1138long kvm_arch_vm_ioctl(struct file *filp,
1139 unsigned int ioctl, unsigned long arg)
1140{
1141 struct kvm *kvm = filp->private_data;
1142 void __user *argp = (void __user *)arg;
f2061656 1143 struct kvm_device_attr attr;
b0c632db
HC		 1144 int r;
1145
1146 switch (ioctl) {
ba5c1e9b
CO		 1147 case KVM_S390_INTERRUPT: {
1148 struct kvm_s390_interrupt s390int;
1149
1150 r = -EFAULT;
1151 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1152 break;
1153 r = kvm_s390_inject_vm(kvm, &s390int);
1154 break;
1155 }
d938dc55
CH		 1156 case KVM_ENABLE_CAP: {
1157 struct kvm_enable_cap cap;
1158 r = -EFAULT;
1159 if (copy_from_user(&cap, argp, sizeof(cap)))
1160 break;
1161 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1162 break;
1163 }
84223598
CH		 1164 case KVM_CREATE_IRQCHIP: {
1165 struct kvm_irq_routing_entry routing;
1166
1167 r = -EINVAL;
1168 if (kvm->arch.use_irqchip) {
1169 /* Set up dummy routing. */
1170 memset(&routing, 0, sizeof(routing));
152b2839 1171 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1172 }
1173 break;
1174 }
f2061656
DD		 1175 case KVM_SET_DEVICE_ATTR: {
1176 r = -EFAULT;
1177 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1178 break;
1179 r = kvm_s390_vm_set_attr(kvm, &attr);
1180 break;
1181 }
1182 case KVM_GET_DEVICE_ATTR: {
1183 r = -EFAULT;
1184 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1185 break;
1186 r = kvm_s390_vm_get_attr(kvm, &attr);
1187 break;
1188 }
1189 case KVM_HAS_DEVICE_ATTR: {
1190 r = -EFAULT;
1191 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1192 break;
1193 r = kvm_s390_vm_has_attr(kvm, &attr);
1194 break;
1195 }
30ee2a98
JH		 1196 case KVM_S390_GET_SKEYS: {
1197 struct kvm_s390_skeys args;
1198
1199 r = -EFAULT;
1200 if (copy_from_user(&args, argp,
1201 sizeof(struct kvm_s390_skeys)))
1202 break;
1203 r = kvm_s390_get_skeys(kvm, &args);
1204 break;
1205 }
1206 case KVM_S390_SET_SKEYS: {
1207 struct kvm_s390_skeys args;
1208
1209 r = -EFAULT;
1210 if (copy_from_user(&args, argp,
1211 sizeof(struct kvm_s390_skeys)))
1212 break;
1213 r = kvm_s390_set_skeys(kvm, &args);
1214 break;
1215 }
b0c632db 1216 default:
367e1319 1217 r = -ENOTTY;
b0c632db
HC		 1218 }
1219
1220 return r;
1221}
1222
45c9b47c
TK		 1223static int kvm_s390_query_ap_config(u8 *config)
1224{
1225 u32 fcn_code = 0x04000000UL;
86044c8c 1226 u32 cc = 0;
45c9b47c 1227
86044c8c 1228 memset(config, 0, 128);
45c9b47c
TK		 1229 asm volatile(
1230 "lgr 0,%1\n"
1231 "lgr 2,%2\n"
1232 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1233 "0: ipm %0\n"
45c9b47c 1234 "srl %0,28\n"
86044c8c
CB		 1235 "1:\n"
1236 EX_TABLE(0b, 1b)
1237 : "+r" (cc)
45c9b47c
TK		 1238 : "r" (fcn_code), "r" (config)
1239 : "cc", "0", "2", "memory"
1240 );
1241
1242 return cc;
1243}
1244
1245static int kvm_s390_apxa_installed(void)
1246{
1247 u8 config[128];
1248 int cc;
1249
a6aacc3f 1250 if (test_facility(12)) {
45c9b47c
TK		 1251 cc = kvm_s390_query_ap_config(config);
1252
1253 if (cc)
1254 pr_err("PQAP(QCI) failed with cc=%d", cc);
1255 else
1256 return config[0] & 0x40;
1257 }
1258
1259 return 0;
1260}
1261
1262static void kvm_s390_set_crycb_format(struct kvm *kvm)
1263{
1264 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1265
1266 if (kvm_s390_apxa_installed())
1267 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1268 else
1269 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1270}
1271
9bb0ec09 1272static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1273{
9bb0ec09
DH		 1274 struct cpuid cpuid;
1275
1276 get_cpu_id(&cpuid);
1277 cpuid.version = 0xff;
1278 return *((u64 *) &cpuid);
9d8d5786
MM		 1279}
1280
c54f0d6a 1281static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1282{
9d8d5786 1283 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1284 return;
5102ee87 1285
c54f0d6a 1286 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1287 kvm_s390_set_crycb_format(kvm);
5102ee87 1288
ed6f76b4
TK		 1289 /* Enable AES/DEA protected key functions by default */
1290 kvm->arch.crypto.aes_kw = 1;
1291 kvm->arch.crypto.dea_kw = 1;
1292 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1293 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1294 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1295 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1296}
1297
7d43bafc
ED		 1298static void sca_dispose(struct kvm *kvm)
1299{
1300 if (kvm->arch.use_esca)
5e044315 1301 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1302 else
1303 free_page((unsigned long)(kvm->arch.sca));
1304 kvm->arch.sca = NULL;
1305}
1306
e08b9637 1307int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1308{
76a6dd72 1309 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1310 int i, rc;
b0c632db 1311 char debug_name[16];
f6c137ff 1312 static unsigned long sca_offset;
b0c632db 1313
e08b9637
CO		 1314 rc = -EINVAL;
1315#ifdef CONFIG_KVM_S390_UCONTROL
1316 if (type & ~KVM_VM_S390_UCONTROL)
1317 goto out_err;
1318 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1319 goto out_err;
1320#else
1321 if (type)
1322 goto out_err;
1323#endif
1324
b0c632db
HC		 1325 rc = s390_enable_sie();
1326 if (rc)
d89f5eff 1327 goto out_err;
b0c632db 1328
b290411a
CO		 1329 rc = -ENOMEM;
1330
7d0a5e62
JF		 1331 ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1332
7d43bafc 1333 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1334 if (!sclp.has_64bscao)
1335 alloc_flags |= GFP_DMA;
5e044315 1336 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1337 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1338 if (!kvm->arch.sca)
d89f5eff 1339 goto out_err;
f6c137ff 1340 spin_lock(&kvm_lock);
c5c2c393 1341 sca_offset += 16;
bc784cce 1342 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1343 sca_offset = 0;
bc784cce
ED		 1344 kvm->arch.sca = (struct bsca_block *)
1345 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1346 spin_unlock(&kvm_lock);
b0c632db
HC		 1347
1348 sprintf(debug_name, "kvm-%u", current->pid);
1349
1cb9cf72 1350 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1351 if (!kvm->arch.dbf)
40f5b735 1352 goto out_err;
b0c632db 1353
c54f0d6a
DH		 1354 kvm->arch.sie_page2 =
1355 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1356 if (!kvm->arch.sie_page2)
40f5b735 1357 goto out_err;
9d8d5786 1358
fb5bf93f 1359 /* Populate the facility mask initially. */
c54f0d6a 1360 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
94422ee8 1361 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1362 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1363 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1364 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1365 else
c54f0d6a 1366 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1367 }
1368
981467c9 1369 /* Populate the facility list initially. */
c54f0d6a
DH		 1370 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1371 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1372 S390_ARCH_FAC_LIST_SIZE_BYTE);
1373
95ca2cb5
JF		 1374 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1375 set_kvm_facility(kvm->arch.model.fac_list, 74);
1376
9bb0ec09 1377 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1378 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1379
c54f0d6a 1380 kvm_s390_crypto_init(kvm);
5102ee87 1381
ba5c1e9b 1382 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1383 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1384 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1385 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1386 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1387
b0c632db 1388 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1389 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1390
e08b9637
CO		 1391 if (type & KVM_VM_S390_UCONTROL) {
1392 kvm->arch.gmap = NULL;
a3a92c31 1393 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1394 } else {
32e6b236
GH		 1395 if (sclp.hamax == U64_MAX)
1396 kvm->arch.mem_limit = TASK_MAX_SIZE;
1397 else
1398 kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1399 sclp.hamax + 1);
a3a92c31 1400 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1401 if (!kvm->arch.gmap)
40f5b735 1402 goto out_err;
2c70fe44 1403 kvm->arch.gmap->private = kvm;
24eb3a82 1404 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1405 }
fa6b7fe9
CH		 1406
1407 kvm->arch.css_support = 0;
84223598 1408 kvm->arch.use_irqchip = 0;
72f25020 1409 kvm->arch.epoch = 0;
fa6b7fe9 1410
8ad35755 1411 spin_lock_init(&kvm->arch.start_stop_lock);
8335713a 1412 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 1413
d89f5eff 1414 return 0;
40f5b735 1415out_err:
c54f0d6a 1416 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 1417 debug_unregister(kvm->arch.dbf);
7d43bafc 1418 sca_dispose(kvm);
78f26131 1419 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 1420 return rc;
b0c632db
HC		 1421}
1422
d329c035
CB		 1423void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1424{
1425 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1426 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1427 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1428 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 1429 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 1430 sca_del_vcpu(vcpu);
27e0393f
CO		 1431
1432 if (kvm_is_ucontrol(vcpu->kvm))
1433 gmap_free(vcpu->arch.gmap);
1434
e6db1d61 1435 if (vcpu->kvm->arch.use_cmma)
b31605c1 1436 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1437 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1438
6692cef3 1439 kvm_vcpu_uninit(vcpu);
b110feaf 1440 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1441}
1442
1443static void kvm_free_vcpus(struct kvm *kvm)
1444{
1445 unsigned int i;
988a2cae 1446 struct kvm_vcpu *vcpu;
d329c035 1447
988a2cae
GN		 1448 kvm_for_each_vcpu(i, vcpu, kvm)
1449 kvm_arch_vcpu_destroy(vcpu);
1450
1451 mutex_lock(&kvm->lock);
1452 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1453 kvm->vcpus[i] = NULL;
1454
1455 atomic_set(&kvm->online_vcpus, 0);
1456 mutex_unlock(&kvm->lock);
d329c035
CB		 1457}
1458
b0c632db
HC		 1459void kvm_arch_destroy_vm(struct kvm *kvm)
1460{
d329c035 1461 kvm_free_vcpus(kvm);
7d43bafc 1462 sca_dispose(kvm);
d329c035 1463 debug_unregister(kvm->arch.dbf);
c54f0d6a 1464 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f
CO		 1465 if (!kvm_is_ucontrol(kvm))
1466 gmap_free(kvm->arch.gmap);
841b91c5 1467 kvm_s390_destroy_adapters(kvm);
67335e63 1468 kvm_s390_clear_float_irqs(kvm);
8335713a 1469 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 1470}
1471
1472/* Section: vcpu related */
dafd032a
DD		 1473static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1474{
1475 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1476 if (!vcpu->arch.gmap)
1477 return -ENOMEM;
1478 vcpu->arch.gmap->private = vcpu->kvm;
1479
1480 return 0;
1481}
1482
a6e2f683
ED		 1483static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1484{
5e044315 1485 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 1486 if (vcpu->kvm->arch.use_esca) {
1487 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1488
7d43bafc 1489 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 1490 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 1491 } else {
1492 struct bsca_block *sca = vcpu->kvm->arch.sca;
1493
1494 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 1495 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 1496 }
5e044315 1497 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 1498}
1499
eaa78f34 1500static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 1501{
eaa78f34
DH		 1502 read_lock(&vcpu->kvm->arch.sca_lock);
1503 if (vcpu->kvm->arch.use_esca) {
1504 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 1505
eaa78f34 1506 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1507 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1508 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
25508824 1509 vcpu->arch.sie_block->ecb2 |= 0x04U;
eaa78f34 1510 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 1511 } else {
eaa78f34 1512 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 1513
eaa78f34 1514 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 1515 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1516 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 1517 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 1518 }
eaa78f34 1519 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 1520}
1521
1522/* Basic SCA to Extended SCA data copy routines */
1523static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1524{
1525 d->sda = s->sda;
1526 d->sigp_ctrl.c = s->sigp_ctrl.c;
1527 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1528}
1529
1530static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1531{
1532 int i;
1533
1534 d->ipte_control = s->ipte_control;
1535 d->mcn[0] = s->mcn;
1536 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1537 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1538}
1539
1540static int sca_switch_to_extended(struct kvm *kvm)
1541{
1542 struct bsca_block *old_sca = kvm->arch.sca;
1543 struct esca_block *new_sca;
1544 struct kvm_vcpu *vcpu;
1545 unsigned int vcpu_idx;
1546 u32 scaol, scaoh;
1547
1548 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1549 if (!new_sca)
1550 return -ENOMEM;
1551
1552 scaoh = (u32)((u64)(new_sca) >> 32);
1553 scaol = (u32)(u64)(new_sca) & ~0x3fU;
1554
1555 kvm_s390_vcpu_block_all(kvm);
1556 write_lock(&kvm->arch.sca_lock);
1557
1558 sca_copy_b_to_e(new_sca, old_sca);
1559
1560 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1561 vcpu->arch.sie_block->scaoh = scaoh;
1562 vcpu->arch.sie_block->scaol = scaol;
1563 vcpu->arch.sie_block->ecb2 |= 0x04U;
1564 }
1565 kvm->arch.sca = new_sca;
1566 kvm->arch.use_esca = 1;
1567
1568 write_unlock(&kvm->arch.sca_lock);
1569 kvm_s390_vcpu_unblock_all(kvm);
1570
1571 free_page((unsigned long)old_sca);
1572
8335713a
CB		 1573 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1574 old_sca, kvm->arch.sca);
5e044315 1575 return 0;
a6e2f683
ED		 1576}
1577
1578static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1579{
5e044315
ED		 1580 int rc;
1581
1582 if (id < KVM_S390_BSCA_CPU_SLOTS)
1583 return true;
76a6dd72 1584 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 1585 return false;
1586
1587 mutex_lock(&kvm->lock);
1588 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1589 mutex_unlock(&kvm->lock);
1590
1591 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 1592}
1593
b0c632db
HC		 1594int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1595{
3c038e6b
DD		 1596 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1597 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1598 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1599 KVM_SYNC_GPRS |
9eed0735 1600 KVM_SYNC_ACRS |
b028ee3e
DH		 1601 KVM_SYNC_CRS |
1602 KVM_SYNC_ARCH0 |
1603 KVM_SYNC_PFAULT;
c6e5f166
FZ		 1604 if (test_kvm_facility(vcpu->kvm, 64))
1605 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
f6aa6dc4
DH		 1606 /* fprs can be synchronized via vrs, even if the guest has no vx. With
1607 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1608 */
1609 if (MACHINE_HAS_VX)
68c55750 1610 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 1611 else
1612 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 1613
1614 if (kvm_is_ucontrol(vcpu->kvm))
1615 return __kvm_ucontrol_vcpu_init(vcpu);
1616
b0c632db
HC		 1617 return 0;
1618}
1619
db0758b2
DH		 1620/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1621static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1622{
1623 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 1624 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 1625 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 1626 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1627}
1628
1629/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1630static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1631{
1632 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 1633 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1634 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1635 vcpu->arch.cputm_start = 0;
9c23a131 1636 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1637}
1638
1639/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1640static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1641{
1642 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1643 vcpu->arch.cputm_enabled = true;
1644 __start_cpu_timer_accounting(vcpu);
1645}
1646
1647/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1648static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1649{
1650 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1651 __stop_cpu_timer_accounting(vcpu);
1652 vcpu->arch.cputm_enabled = false;
1653}
1654
1655static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1656{
1657 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1658 __enable_cpu_timer_accounting(vcpu);
1659 preempt_enable();
1660}
1661
1662static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1663{
1664 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1665 __disable_cpu_timer_accounting(vcpu);
1666 preempt_enable();
1667}
1668
4287f247
DH		 1669/* set the cpu timer - may only be called from the VCPU thread itself */
1670void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1671{
db0758b2 1672 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 1673 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 1674 if (vcpu->arch.cputm_enabled)
1675 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 1676 vcpu->arch.sie_block->cputm = cputm;
9c23a131 1677 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 1678 preempt_enable();
4287f247
DH		 1679}
1680
db0758b2 1681/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 1682__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1683{
9c23a131 1684 unsigned int seq;
db0758b2 1685 __u64 value;
db0758b2
DH		 1686
1687 if (unlikely(!vcpu->arch.cputm_enabled))
1688 return vcpu->arch.sie_block->cputm;
1689
9c23a131
DH		 1690 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1691 do {
1692 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1693 /*
1694 * If the writer would ever execute a read in the critical
1695 * section, e.g. in irq context, we have a deadlock.
1696 */
1697 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1698 value = vcpu->arch.sie_block->cputm;
1699 /* if cputm_start is 0, accounting is being started/stopped */
1700 if (likely(vcpu->arch.cputm_start))
1701 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1702 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1703 preempt_enable();
db0758b2 1704 return value;
4287f247
DH		 1705}
1706
b0c632db
HC		 1707void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1708{
9977e886 1709 /* Save host register state */
d0164ee2 1710 save_fpu_regs();
9abc2a08
DH		 1711 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1712 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
9977e886 1713
6fd8e67d
DH		 1714 if (MACHINE_HAS_VX)
1715 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1716 else
1717 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
9abc2a08 1718 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
9977e886 1719 if (test_fp_ctl(current->thread.fpu.fpc))
96b2d7a8 1720 /* User space provided an invalid FPC, let's clear it */
9977e886
HB		 1721 current->thread.fpu.fpc = 0;
1722
1723 save_access_regs(vcpu->arch.host_acrs);
59674c1a 1724 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1725 gmap_enable(vcpu->arch.gmap);
805de8f4 1726 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
5ebda316 1727 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1728 __start_cpu_timer_accounting(vcpu);
01a745ac 1729 vcpu->cpu = cpu;
b0c632db
HC		 1730}
1731
1732void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1733{
01a745ac 1734 vcpu->cpu = -1;
5ebda316 1735 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 1736 __stop_cpu_timer_accounting(vcpu);
805de8f4 1737 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1738 gmap_disable(vcpu->arch.gmap);
9977e886 1739
9abc2a08 1740 /* Save guest register state */
d0164ee2 1741 save_fpu_regs();
9abc2a08 1742 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
9977e886 1743
9abc2a08
DH		 1744 /* Restore host register state */
1745 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1746 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
9977e886
HB		 1747
1748 save_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1749 restore_access_regs(vcpu->arch.host_acrs);
1750}
1751
1752static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1753{
1754 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1755 vcpu->arch.sie_block->gpsw.mask = 0UL;
1756 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1757 kvm_s390_set_prefix(vcpu, 0);
4287f247 1758 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 1759 vcpu->arch.sie_block->ckc = 0UL;
1760 vcpu->arch.sie_block->todpr = 0;
1761 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1762 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1763 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 1764 /* make sure the new fpc will be lazily loaded */
1765 save_fpu_regs();
1766 current->thread.fpu.fpc = 0;
b0c632db 1767 vcpu->arch.sie_block->gbea = 1;
672550fb 1768 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1769 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1770 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1771 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1772 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1773 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1774}
1775
31928aa5 1776void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1777{
72f25020 1778 mutex_lock(&vcpu->kvm->lock);
fdf03650 1779 preempt_disable();
72f25020 1780 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 1781 preempt_enable();
72f25020 1782 mutex_unlock(&vcpu->kvm->lock);
25508824 1783 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 1784 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 1785 sca_add_vcpu(vcpu);
25508824
DH		 1786 }
1787
42897d86
MT		 1788}
1789
5102ee87
TK		 1790static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1791{
9d8d5786 1792 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1793 return;
1794
a374e892
TK		 1795 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1796
1797 if (vcpu->kvm->arch.crypto.aes_kw)
1798 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1799 if (vcpu->kvm->arch.crypto.dea_kw)
1800 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1801
5102ee87
TK		 1802 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1803}
1804
b31605c1
DD		 1805void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1806{
1807 free_page(vcpu->arch.sie_block->cbrlo);
1808 vcpu->arch.sie_block->cbrlo = 0;
1809}
1810
1811int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1812{
1813 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1814 if (!vcpu->arch.sie_block->cbrlo)
1815 return -ENOMEM;
1816
1817 vcpu->arch.sie_block->ecb2 |= 0x80;
1818 vcpu->arch.sie_block->ecb2 &= ~0x08;
1819 return 0;
1820}
1821
91520f1a
MM		 1822static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1823{
1824 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1825
91520f1a 1826 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 1827 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 1828 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 1829}
1830
b0c632db
HC		 1831int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1832{
b31605c1 1833 int rc = 0;
b31288fa 1834
9e6dabef
CH		 1835 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1836 CPUSTAT_SM |
a4a4f191
GH		 1837 CPUSTAT_STOPPED);
1838
53df84f8 1839 if (test_kvm_facility(vcpu->kvm, 78))
805de8f4 1840 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
53df84f8 1841 else if (test_kvm_facility(vcpu->kvm, 8))
805de8f4 1842 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
a4a4f191 1843
91520f1a
MM		 1844 kvm_s390_vcpu_setup_model(vcpu);
1845
bdab09f3
DH		 1846 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1847 if (MACHINE_HAS_ESOP)
1848 vcpu->arch.sie_block->ecb |= 0x02;
bd50e8ec
DH		 1849 if (test_kvm_facility(vcpu->kvm, 9))
1850 vcpu->arch.sie_block->ecb |= 0x04;
f597d24e 1851 if (test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1852 vcpu->arch.sie_block->ecb |= 0x10;
1853
873b425e 1854 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
d6af0b49 1855 vcpu->arch.sie_block->ecb2 |= 0x08;
48ee7d3a
DH		 1856 vcpu->arch.sie_block->eca = 0x1002000U;
1857 if (sclp.has_cei)
1858 vcpu->arch.sie_block->eca |= 0x80000000U;
11ad65b7
DH		 1859 if (sclp.has_ib)
1860 vcpu->arch.sie_block->eca |= 0x40000000U;
37c5f6c8 1861 if (sclp.has_siif)
217a4406 1862 vcpu->arch.sie_block->eca |= 1;
37c5f6c8 1863 if (sclp.has_sigpif)
ea5f4969 1864 vcpu->arch.sie_block->eca |= 0x10000000U;
c6e5f166
FZ		 1865 if (test_kvm_facility(vcpu->kvm, 64))
1866 vcpu->arch.sie_block->ecb3 |= 0x01;
18280d8b 1867 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1868 vcpu->arch.sie_block->eca |= 0x00020000;
1869 vcpu->arch.sie_block->ecd |= 0x20000000;
1870 }
c6e5f166 1871 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
492d8642 1872 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
95ca2cb5
JF		 1873 if (test_kvm_facility(vcpu->kvm, 74))
1874 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
5a5e6536 1875
e6db1d61 1876 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 1877 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1878 if (rc)
1879 return rc;
b31288fa 1880 }
0ac96caf 1881 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1882 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1883
5102ee87
TK		 1884 kvm_s390_vcpu_crypto_setup(vcpu);
1885
b31605c1 1886 return rc;
b0c632db
HC		 1887}
1888
1889struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1890 unsigned int id)
1891{
4d47555a 1892 struct kvm_vcpu *vcpu;
7feb6bb8 1893 struct sie_page *sie_page;
4d47555a
CO		 1894 int rc = -EINVAL;
1895
4215825e 1896 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 1897 goto out;
1898
1899 rc = -ENOMEM;
b0c632db 1900
b110feaf 1901 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1902 if (!vcpu)
4d47555a 1903 goto out;
b0c632db 1904
7feb6bb8
MM		 1905 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1906 if (!sie_page)
b0c632db
HC		 1907 goto out_free_cpu;
1908
7feb6bb8
MM		 1909 vcpu->arch.sie_block = &sie_page->sie_block;
1910 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1911
efed1104
DH		 1912 /* the real guest size will always be smaller than msl */
1913 vcpu->arch.sie_block->mso = 0;
1914 vcpu->arch.sie_block->msl = sclp.hamax;
1915
b0c632db 1916 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 1917 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1918 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1919 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1920 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
9c23a131 1921 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 1922
b0c632db
HC		 1923 rc = kvm_vcpu_init(vcpu, kvm, id);
1924 if (rc)
9abc2a08 1925 goto out_free_sie_block;
8335713a 1926 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 1927 vcpu->arch.sie_block);
ade38c31 1928 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1929
b0c632db 1930 return vcpu;
7b06bf2f
WY		 1931out_free_sie_block:
1932 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1933out_free_cpu:
b110feaf 1934 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1935out:
b0c632db
HC		 1936 return ERR_PTR(rc);
1937}
1938
b0c632db
HC		 1939int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1940{
9a022067 1941 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1942}
1943
27406cd5 1944void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 1945{
805de8f4 1946 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 1947 exit_sie(vcpu);
49b99e1e
CB		 1948}
1949
27406cd5 1950void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 1951{
805de8f4 1952 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 1953}
1954
8e236546
CB		 1955static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1956{
805de8f4 1957 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 1958 exit_sie(vcpu);
8e236546
CB		 1959}
1960
1961static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1962{
9bf9fde2 1963 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 1964}
1965
49b99e1e
CB		 1966/*
1967 * Kick a guest cpu out of SIE and wait until SIE is not running.
1968 * If the CPU is not running (e.g. waiting as idle) the function will
1969 * return immediately. */
1970void exit_sie(struct kvm_vcpu *vcpu)
1971{
805de8f4 1972 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
49b99e1e
CB		 1973 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1974 cpu_relax();
1975}
1976
8e236546
CB		 1977/* Kick a guest cpu out of SIE to process a request synchronously */
1978void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 1979{
8e236546
CB		 1980 kvm_make_request(req, vcpu);
1981 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 1982}
1983
2c70fe44
CB		 1984static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1985{
1986 int i;
1987 struct kvm *kvm = gmap->private;
1988 struct kvm_vcpu *vcpu;
1989
1990 kvm_for_each_vcpu(i, vcpu, kvm) {
1991 /* match against both prefix pages */
fda902cb 1992 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44 1993 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
8e236546 1994 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 1995 }
1996 }
1997}
1998
b6d33834
CD		 1999int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2000{
2001 /* kvm common code refers to this, but never calls it */
2002 BUG();
2003 return 0;
2004}
2005
14eebd91
CO		 2006static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2007 struct kvm_one_reg *reg)
2008{
2009 int r = -EINVAL;
2010
2011 switch (reg->id) {
29b7c71b
CO		 2012 case KVM_REG_S390_TODPR:
2013 r = put_user(vcpu->arch.sie_block->todpr,
2014 (u32 __user *)reg->addr);
2015 break;
2016 case KVM_REG_S390_EPOCHDIFF:
2017 r = put_user(vcpu->arch.sie_block->epoch,
2018 (u64 __user *)reg->addr);
2019 break;
46a6dd1c 2020 case KVM_REG_S390_CPU_TIMER:
4287f247 2021 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2022 (u64 __user *)reg->addr);
2023 break;
2024 case KVM_REG_S390_CLOCK_COMP:
2025 r = put_user(vcpu->arch.sie_block->ckc,
2026 (u64 __user *)reg->addr);
2027 break;
536336c2
DD		 2028 case KVM_REG_S390_PFTOKEN:
2029 r = put_user(vcpu->arch.pfault_token,
2030 (u64 __user *)reg->addr);
2031 break;
2032 case KVM_REG_S390_PFCOMPARE:
2033 r = put_user(vcpu->arch.pfault_compare,
2034 (u64 __user *)reg->addr);
2035 break;
2036 case KVM_REG_S390_PFSELECT:
2037 r = put_user(vcpu->arch.pfault_select,
2038 (u64 __user *)reg->addr);
2039 break;
672550fb
CB		 2040 case KVM_REG_S390_PP:
2041 r = put_user(vcpu->arch.sie_block->pp,
2042 (u64 __user *)reg->addr);
2043 break;
afa45ff5
CB		 2044 case KVM_REG_S390_GBEA:
2045 r = put_user(vcpu->arch.sie_block->gbea,
2046 (u64 __user *)reg->addr);
2047 break;
14eebd91
CO		 2048 default:
2049 break;
2050 }
2051
2052 return r;
2053}
2054
2055static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2056 struct kvm_one_reg *reg)
2057{
2058 int r = -EINVAL;
4287f247 2059 __u64 val;
14eebd91
CO		 2060
2061 switch (reg->id) {
29b7c71b
CO		 2062 case KVM_REG_S390_TODPR:
2063 r = get_user(vcpu->arch.sie_block->todpr,
2064 (u32 __user *)reg->addr);
2065 break;
2066 case KVM_REG_S390_EPOCHDIFF:
2067 r = get_user(vcpu->arch.sie_block->epoch,
2068 (u64 __user *)reg->addr);
2069 break;
46a6dd1c 2070 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2071 r = get_user(val, (u64 __user *)reg->addr);
2072 if (!r)
2073 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2074 break;
2075 case KVM_REG_S390_CLOCK_COMP:
2076 r = get_user(vcpu->arch.sie_block->ckc,
2077 (u64 __user *)reg->addr);
2078 break;
536336c2
DD		 2079 case KVM_REG_S390_PFTOKEN:
2080 r = get_user(vcpu->arch.pfault_token,
2081 (u64 __user *)reg->addr);
9fbd8082
DH		 2082 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2083 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2084 break;
2085 case KVM_REG_S390_PFCOMPARE:
2086 r = get_user(vcpu->arch.pfault_compare,
2087 (u64 __user *)reg->addr);
2088 break;
2089 case KVM_REG_S390_PFSELECT:
2090 r = get_user(vcpu->arch.pfault_select,
2091 (u64 __user *)reg->addr);
2092 break;
672550fb
CB		 2093 case KVM_REG_S390_PP:
2094 r = get_user(vcpu->arch.sie_block->pp,
2095 (u64 __user *)reg->addr);
2096 break;
afa45ff5
CB		 2097 case KVM_REG_S390_GBEA:
2098 r = get_user(vcpu->arch.sie_block->gbea,
2099 (u64 __user *)reg->addr);
2100 break;
14eebd91
CO		 2101 default:
2102 break;
2103 }
2104
2105 return r;
2106}
b6d33834 2107
b0c632db
HC		 2108static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2109{
b0c632db 2110 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2111 return 0;
2112}
2113
2114int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2115{
5a32c1af 2116 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 2117 return 0;
2118}
2119
2120int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2121{
5a32c1af 2122 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 2123 return 0;
2124}
2125
2126int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2127 struct kvm_sregs *sregs)
2128{
59674c1a 2129 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2130 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 2131 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 2132 return 0;
2133}
2134
2135int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2136 struct kvm_sregs *sregs)
2137{
59674c1a 2138 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2139 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 2140 return 0;
2141}
2142
2143int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2144{
9abc2a08
DH		 2145 /* make sure the new values will be lazily loaded */
2146 save_fpu_regs();
4725c860
MS		 2147 if (test_fp_ctl(fpu->fpc))
2148 return -EINVAL;
9abc2a08
DH		 2149 current->thread.fpu.fpc = fpu->fpc;
2150 if (MACHINE_HAS_VX)
2151 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2152 else
2153 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
b0c632db
HC		 2154 return 0;
2155}
2156
2157int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2158{
9abc2a08
DH		 2159 /* make sure we have the latest values */
2160 save_fpu_regs();
2161 if (MACHINE_HAS_VX)
2162 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2163 else
2164 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2165 fpu->fpc = current->thread.fpu.fpc;
b0c632db
HC		 2166 return 0;
2167}
2168
2169static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2170{
2171 int rc = 0;
2172
7a42fdc2 2173 if (!is_vcpu_stopped(vcpu))
b0c632db 2174 rc = -EBUSY;
d7b0b5eb
CO		 2175 else {
2176 vcpu->run->psw_mask = psw.mask;
2177 vcpu->run->psw_addr = psw.addr;
2178 }
b0c632db
HC		 2179 return rc;
2180}
2181
2182int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2183 struct kvm_translation *tr)
2184{
2185 return -EINVAL; /* not implemented yet */
2186}
2187
27291e21
DH		 2188#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2189 KVM_GUESTDBG_USE_HW_BP | \
2190 KVM_GUESTDBG_ENABLE)
2191
d0bfb940
JK		 2192int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2193 struct kvm_guest_debug *dbg)
b0c632db 2194{
27291e21
DH		 2195 int rc = 0;
2196
2197 vcpu->guest_debug = 0;
2198 kvm_s390_clear_bp_data(vcpu);
2199
2de3bfc2 2200 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21 2201 return -EINVAL;
89b5b4de
DH		 2202 if (!sclp.has_gpere)
2203 return -EINVAL;
27291e21
DH		 2204
2205 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2206 vcpu->guest_debug = dbg->control;
2207 /* enforce guest PER */
805de8f4 2208 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2209
2210 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2211 rc = kvm_s390_import_bp_data(vcpu, dbg);
2212 } else {
805de8f4 2213 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2214 vcpu->arch.guestdbg.last_bp = 0;
2215 }
2216
2217 if (rc) {
2218 vcpu->guest_debug = 0;
2219 kvm_s390_clear_bp_data(vcpu);
805de8f4 2220 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
27291e21
DH		 2221 }
2222
2223 return rc;
b0c632db
HC		 2224}
2225
62d9f0db
MT		 2226int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2227 struct kvm_mp_state *mp_state)
2228{
6352e4d2
DH		 2229 /* CHECK_STOP and LOAD are not supported yet */
2230 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2231 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 2232}
2233
2234int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2235 struct kvm_mp_state *mp_state)
2236{
6352e4d2
DH		 2237 int rc = 0;
2238
2239 /* user space knows about this interface - let it control the state */
2240 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2241
2242 switch (mp_state->mp_state) {
2243 case KVM_MP_STATE_STOPPED:
2244 kvm_s390_vcpu_stop(vcpu);
2245 break;
2246 case KVM_MP_STATE_OPERATING:
2247 kvm_s390_vcpu_start(vcpu);
2248 break;
2249 case KVM_MP_STATE_LOAD:
2250 case KVM_MP_STATE_CHECK_STOP:
2251 /* fall through - CHECK_STOP and LOAD are not supported yet */
2252 default:
2253 rc = -ENXIO;
2254 }
2255
2256 return rc;
62d9f0db
MT		 2257}
2258
8ad35755
DH		 2259static bool ibs_enabled(struct kvm_vcpu *vcpu)
2260{
2261 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2262}
2263
2c70fe44
CB		 2264static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2265{
8ad35755 2266retry:
8e236546 2267 kvm_s390_vcpu_request_handled(vcpu);
586b7ccd
CB		 2268 if (!vcpu->requests)
2269 return 0;
2c70fe44
CB		 2270 /*
2271 * We use MMU_RELOAD just to re-arm the ipte notifier for the
2272 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2273 * This ensures that the ipte instruction for this request has
2274 * already finished. We might race against a second unmapper that
2275 * wants to set the blocking bit. Lets just retry the request loop.
2276 */
8ad35755 2277 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 2278 int rc;
2279 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 2280 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 2281 PAGE_SIZE * 2);
2282 if (rc)
2283 return rc;
8ad35755 2284 goto retry;
2c70fe44 2285 }
8ad35755 2286
d3d692c8
DH		 2287 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2288 vcpu->arch.sie_block->ihcpu = 0xffff;
2289 goto retry;
2290 }
2291
8ad35755
DH		 2292 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2293 if (!ibs_enabled(vcpu)) {
2294 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
805de8f4 2295 atomic_or(CPUSTAT_IBS,
8ad35755
DH		 2296 &vcpu->arch.sie_block->cpuflags);
2297 }
2298 goto retry;
2c70fe44 2299 }
8ad35755
DH		 2300
2301 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2302 if (ibs_enabled(vcpu)) {
2303 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
805de8f4 2304 atomic_andnot(CPUSTAT_IBS,
8ad35755
DH		 2305 &vcpu->arch.sie_block->cpuflags);
2306 }
2307 goto retry;
2308 }
2309
0759d068
DH		 2310 /* nothing to do, just clear the request */
2311 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2312
2c70fe44
CB		 2313 return 0;
2314}
2315
25ed1675
DH		 2316void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2317{
2318 struct kvm_vcpu *vcpu;
2319 int i;
2320
2321 mutex_lock(&kvm->lock);
2322 preempt_disable();
2323 kvm->arch.epoch = tod - get_tod_clock();
2324 kvm_s390_vcpu_block_all(kvm);
2325 kvm_for_each_vcpu(i, vcpu, kvm)
2326 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2327 kvm_s390_vcpu_unblock_all(kvm);
2328 preempt_enable();
2329 mutex_unlock(&kvm->lock);
2330}
2331
fa576c58
TH		 2332/**
2333 * kvm_arch_fault_in_page - fault-in guest page if necessary
2334 * @vcpu: The corresponding virtual cpu
2335 * @gpa: Guest physical address
2336 * @writable: Whether the page should be writable or not
2337 *
2338 * Make sure that a guest page has been faulted-in on the host.
2339 *
2340 * Return: Zero on success, negative error code otherwise.
2341 */
2342long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 2343{
527e30b4
MS		 2344 return gmap_fault(vcpu->arch.gmap, gpa,
2345 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 2346}
2347
3c038e6b
DD		 2348static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2349 unsigned long token)
2350{
2351 struct kvm_s390_interrupt inti;
383d0b05 2352 struct kvm_s390_irq irq;
3c038e6b
DD		 2353
2354 if (start_token) {
383d0b05
JF		 2355 irq.u.ext.ext_params2 = token;
2356 irq.type = KVM_S390_INT_PFAULT_INIT;
2357 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 2358 } else {
2359 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 2360 inti.parm64 = token;
3c038e6b
DD		 2361 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2362 }
2363}
2364
2365void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2366 struct kvm_async_pf *work)
2367{
2368 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2369 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2370}
2371
2372void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2373 struct kvm_async_pf *work)
2374{
2375 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2376 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2377}
2378
2379void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2380 struct kvm_async_pf *work)
2381{
2382 /* s390 will always inject the page directly */
2383}
2384
2385bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2386{
2387 /*
2388 * s390 will always inject the page directly,
2389 * but we still want check_async_completion to cleanup
2390 */
2391 return true;
2392}
2393
2394static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2395{
2396 hva_t hva;
2397 struct kvm_arch_async_pf arch;
2398 int rc;
2399
2400 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2401 return 0;
2402 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2403 vcpu->arch.pfault_compare)
2404 return 0;
2405 if (psw_extint_disabled(vcpu))
2406 return 0;
9a022067 2407 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 2408 return 0;
2409 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2410 return 0;
2411 if (!vcpu->arch.gmap->pfault_enabled)
2412 return 0;
2413
81480cc1
HC		 2414 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2415 hva += current->thread.gmap_addr & ~PAGE_MASK;
2416 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 2417 return 0;
2418
2419 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2420 return rc;
2421}
2422
3fb4c40f 2423static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 2424{
3fb4c40f 2425 int rc, cpuflags;
e168bf8d 2426
3c038e6b
DD		 2427 /*
2428 * On s390 notifications for arriving pages will be delivered directly
2429 * to the guest but the house keeping for completed pfaults is
2430 * handled outside the worker.
2431 */
2432 kvm_check_async_pf_completion(vcpu);
2433
7ec7c8c7
CB		 2434 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2435 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 2436
2437 if (need_resched())
2438 schedule();
2439
d3a73acb 2440 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 2441 s390_handle_mcck();
2442
79395031
JF		 2443 if (!kvm_is_ucontrol(vcpu->kvm)) {
2444 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2445 if (rc)
2446 return rc;
2447 }
0ff31867 2448
2c70fe44
CB		 2449 rc = kvm_s390_handle_requests(vcpu);
2450 if (rc)
2451 return rc;
2452
27291e21
DH		 2453 if (guestdbg_enabled(vcpu)) {
2454 kvm_s390_backup_guest_per_regs(vcpu);
2455 kvm_s390_patch_guest_per_regs(vcpu);
2456 }
2457
b0c632db 2458 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 2459 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2460 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2461 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 2462
3fb4c40f
TH		 2463 return 0;
2464}
2465
492d8642
TH		 2466static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2467{
56317920
DH		 2468 struct kvm_s390_pgm_info pgm_info = {
2469 .code = PGM_ADDRESSING,
2470 };
2471 u8 opcode, ilen;
492d8642
TH		 2472 int rc;
2473
2474 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2475 trace_kvm_s390_sie_fault(vcpu);
2476
2477 /*
2478 * We want to inject an addressing exception, which is defined as a
2479 * suppressing or terminating exception. However, since we came here
2480 * by a DAT access exception, the PSW still points to the faulting
2481 * instruction since DAT exceptions are nullifying. So we've got
2482 * to look up the current opcode to get the length of the instruction
2483 * to be able to forward the PSW.
2484 */
65977322 2485 rc = read_guest_instr(vcpu, &opcode, 1);
56317920 2486 ilen = insn_length(opcode);
9b0d721a
DH		 2487 if (rc < 0) {
2488 return rc;
2489 } else if (rc) {
2490 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2491 * Forward by arbitrary ilc, injection will take care of
2492 * nullification if necessary.
2493 */
2494 pgm_info = vcpu->arch.pgm;
2495 ilen = 4;
2496 }
56317920
DH		 2497 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2498 kvm_s390_forward_psw(vcpu, ilen);
2499 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 2500}
2501
3fb4c40f
TH		 2502static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2503{
2b29a9fd
DD		 2504 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2505 vcpu->arch.sie_block->icptcode);
2506 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2507
27291e21
DH		 2508 if (guestdbg_enabled(vcpu))
2509 kvm_s390_restore_guest_per_regs(vcpu);
2510
7ec7c8c7
CB		 2511 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2512 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf
DH		 2513
2514 if (vcpu->arch.sie_block->icptcode > 0) {
2515 int rc = kvm_handle_sie_intercept(vcpu);
2516
2517 if (rc != -EOPNOTSUPP)
2518 return rc;
2519 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2520 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2521 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2522 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2523 return -EREMOTE;
2524 } else if (exit_reason != -EFAULT) {
2525 vcpu->stat.exit_null++;
2526 return 0;
210b1607
TH		 2527 } else if (kvm_is_ucontrol(vcpu->kvm)) {
2528 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2529 vcpu->run->s390_ucontrol.trans_exc_code =
2530 current->thread.gmap_addr;
2531 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 2532 return -EREMOTE;
24eb3a82 2533 } else if (current->thread.gmap_pfault) {
3c038e6b 2534 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 2535 current->thread.gmap_pfault = 0;
71f116bf
DH		 2536 if (kvm_arch_setup_async_pf(vcpu))
2537 return 0;
2538 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 2539 }
71f116bf 2540 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 2541}
2542
2543static int __vcpu_run(struct kvm_vcpu *vcpu)
2544{
2545 int rc, exit_reason;
2546
800c1065
TH		 2547 /*
2548 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2549 * ning the guest), so that memslots (and other stuff) are protected
2550 */
2551 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2552
a76ccff6
TH		 2553 do {
2554 rc = vcpu_pre_run(vcpu);
2555 if (rc)
2556 break;
3fb4c40f 2557
800c1065 2558 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 2559 /*
2560 * As PF_VCPU will be used in fault handler, between
2561 * guest_enter and guest_exit should be no uaccess.
2562 */
0097d12e
CB		 2563 local_irq_disable();
2564 __kvm_guest_enter();
db0758b2 2565 __disable_cpu_timer_accounting(vcpu);
0097d12e 2566 local_irq_enable();
a76ccff6
TH		 2567 exit_reason = sie64a(vcpu->arch.sie_block,
2568 vcpu->run->s.regs.gprs);
0097d12e 2569 local_irq_disable();
db0758b2 2570 __enable_cpu_timer_accounting(vcpu);
0097d12e
CB		 2571 __kvm_guest_exit();
2572 local_irq_enable();
800c1065 2573 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2574
2575 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2576 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2577
800c1065 2578 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2579 return rc;
b0c632db
HC		 2580}
2581
b028ee3e
DH		 2582static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2583{
2584 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2585 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2586 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2587 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2588 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2589 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2590 /* some control register changes require a tlb flush */
2591 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2592 }
2593 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 2594 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 2595 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2596 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2597 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2598 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2599 }
2600 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2601 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2602 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2603 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2604 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2605 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2606 }
2607 kvm_run->kvm_dirty_regs = 0;
2608}
2609
2610static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2611{
2612 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2613 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2614 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2615 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 2616 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 2617 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2618 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2619 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2620 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2621 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2622 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2623 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2624}
2625
b0c632db
HC		 2626int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2627{
8f2abe6a 2628 int rc;
b0c632db
HC		 2629 sigset_t sigsaved;
2630
27291e21
DH		 2631 if (guestdbg_exit_pending(vcpu)) {
2632 kvm_s390_prepare_debug_exit(vcpu);
2633 return 0;
2634 }
2635
b0c632db
HC		 2636 if (vcpu->sigset_active)
2637 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2638
6352e4d2
DH		 2639 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2640 kvm_s390_vcpu_start(vcpu);
2641 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 2642 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2
DH		 2643 vcpu->vcpu_id);
2644 return -EINVAL;
2645 }
b0c632db 2646
b028ee3e 2647 sync_regs(vcpu, kvm_run);
db0758b2 2648 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 2649
dab4079d 2650 might_fault();
a76ccff6 2651 rc = __vcpu_run(vcpu);
9ace903d 2652
b1d16c49
CE		 2653 if (signal_pending(current) && !rc) {
2654 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2655 rc = -EINTR;
b1d16c49 2656 }
8f2abe6a 2657
27291e21
DH		 2658 if (guestdbg_exit_pending(vcpu) && !rc) {
2659 kvm_s390_prepare_debug_exit(vcpu);
2660 rc = 0;
2661 }
2662
8f2abe6a 2663 if (rc == -EREMOTE) {
71f116bf 2664 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 2665 rc = 0;
2666 }
b0c632db 2667
db0758b2 2668 disable_cpu_timer_accounting(vcpu);
b028ee3e 2669 store_regs(vcpu, kvm_run);
d7b0b5eb 2670
b0c632db
HC		 2671 if (vcpu->sigset_active)
2672 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2673
b0c632db 2674 vcpu->stat.exit_userspace++;
7e8e6ab4 2675 return rc;
b0c632db
HC		 2676}
2677
b0c632db
HC		 2678/*
2679 * store status at address
2680 * we use have two special cases:
2681 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2682 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2683 */
d0bce605 2684int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2685{
092670cd 2686 unsigned char archmode = 1;
9abc2a08 2687 freg_t fprs[NUM_FPRS];
fda902cb 2688 unsigned int px;
4287f247 2689 u64 clkcomp, cputm;
d0bce605 2690 int rc;
b0c632db 2691
d9a3a09a 2692 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 2693 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2694 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2695 return -EFAULT;
d9a3a09a 2696 gpa = 0;
d0bce605
HC		 2697 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2698 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2699 return -EFAULT;
d9a3a09a
MS		 2700 gpa = px;
2701 } else
2702 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 2703
2704 /* manually convert vector registers if necessary */
2705 if (MACHINE_HAS_VX) {
9522b37f 2706 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 2707 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2708 fprs, 128);
2709 } else {
2710 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 2711 vcpu->run->s.regs.fprs, 128);
9abc2a08 2712 }
d9a3a09a 2713 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 2714 vcpu->run->s.regs.gprs, 128);
d9a3a09a 2715 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 2716 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 2717 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 2718 &px, 4);
d9a3a09a 2719 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 2720 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 2721 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 2722 &vcpu->arch.sie_block->todpr, 4);
4287f247 2723 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 2724 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 2725 &cputm, 8);
178bd789 2726 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 2727 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 2728 &clkcomp, 8);
d9a3a09a 2729 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 2730 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 2731 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 2732 &vcpu->arch.sie_block->gcr, 128);
2733 return rc ? -EFAULT : 0;
b0c632db
HC		 2734}
2735
e879892c
TH		 2736int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2737{
2738 /*
2739 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2740 * copying in vcpu load/put. Lets update our copies before we save
2741 * it into the save area
2742 */
d0164ee2 2743 save_fpu_regs();
9abc2a08 2744 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 2745 save_access_regs(vcpu->run->s.regs.acrs);
2746
2747 return kvm_s390_store_status_unloaded(vcpu, addr);
2748}
2749
bc17de7c
EF		 2750/*
2751 * store additional status at address
2752 */
2753int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2754 unsigned long gpa)
2755{
2756 /* Only bits 0-53 are used for address formation */
2757 if (!(gpa & ~0x3ff))
2758 return 0;
2759
2760 return write_guest_abs(vcpu, gpa & ~0x3ff,
2761 (void *)&vcpu->run->s.regs.vrs, 512);
2762}
2763
2764int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2765{
2766 if (!test_kvm_facility(vcpu->kvm, 129))
2767 return 0;
2768
2769 /*
2770 * The guest VXRS are in the host VXRs due to the lazy
9977e886
HB		 2771 * copying in vcpu load/put. We can simply call save_fpu_regs()
2772 * to save the current register state because we are in the
2773 * middle of a load/put cycle.
2774 *
2775 * Let's update our copies before we save it into the save area.
bc17de7c 2776 */
d0164ee2 2777 save_fpu_regs();
bc17de7c
EF		 2778
2779 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2780}
2781
8ad35755
DH		 2782static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2783{
2784 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 2785 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 2786}
2787
2788static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2789{
2790 unsigned int i;
2791 struct kvm_vcpu *vcpu;
2792
2793 kvm_for_each_vcpu(i, vcpu, kvm) {
2794 __disable_ibs_on_vcpu(vcpu);
2795 }
2796}
2797
2798static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2799{
09a400e7
DH		 2800 if (!sclp.has_ibs)
2801 return;
8ad35755 2802 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 2803 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 2804}
2805
6852d7b6
DH		 2806void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2807{
8ad35755
DH		 2808 int i, online_vcpus, started_vcpus = 0;
2809
2810 if (!is_vcpu_stopped(vcpu))
2811 return;
2812
6852d7b6 2813 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2814 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2815 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2816 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2817
2818 for (i = 0; i < online_vcpus; i++) {
2819 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2820 started_vcpus++;
2821 }
2822
2823 if (started_vcpus == 0) {
2824 /* we're the only active VCPU -> speed it up */
2825 __enable_ibs_on_vcpu(vcpu);
2826 } else if (started_vcpus == 1) {
2827 /*
2828 * As we are starting a second VCPU, we have to disable
2829 * the IBS facility on all VCPUs to remove potentially
2830 * oustanding ENABLE requests.
2831 */
2832 __disable_ibs_on_all_vcpus(vcpu->kvm);
2833 }
2834
805de8f4 2835 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2836 /*
2837 * Another VCPU might have used IBS while we were offline.
2838 * Let's play safe and flush the VCPU at startup.
2839 */
d3d692c8 2840 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2841 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2842 return;
6852d7b6
DH		 2843}
2844
2845void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2846{
8ad35755
DH		 2847 int i, online_vcpus, started_vcpus = 0;
2848 struct kvm_vcpu *started_vcpu = NULL;
2849
2850 if (is_vcpu_stopped(vcpu))
2851 return;
2852
6852d7b6 2853 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2854 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2855 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2856 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2857
32f5ff63 2858 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2859 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2860
805de8f4 2861 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2862 __disable_ibs_on_vcpu(vcpu);
2863
2864 for (i = 0; i < online_vcpus; i++) {
2865 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2866 started_vcpus++;
2867 started_vcpu = vcpu->kvm->vcpus[i];
2868 }
2869 }
2870
2871 if (started_vcpus == 1) {
2872 /*
2873 * As we only have one VCPU left, we want to enable the
2874 * IBS facility for that VCPU to speed it up.
2875 */
2876 __enable_ibs_on_vcpu(started_vcpu);
2877 }
2878
433b9ee4 2879 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2880 return;
6852d7b6
DH		 2881}
2882
d6712df9
CH		 2883static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2884 struct kvm_enable_cap *cap)
2885{
2886 int r;
2887
2888 if (cap->flags)
2889 return -EINVAL;
2890
2891 switch (cap->cap) {
fa6b7fe9
CH		 2892 case KVM_CAP_S390_CSS_SUPPORT:
2893 if (!vcpu->kvm->arch.css_support) {
2894 vcpu->kvm->arch.css_support = 1;
c92ea7b9 2895 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 2896 trace_kvm_s390_enable_css(vcpu->kvm);
2897 }
2898 r = 0;
2899 break;
d6712df9
CH		 2900 default:
2901 r = -EINVAL;
2902 break;
2903 }
2904 return r;
2905}
2906
41408c28
TH		 2907static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2908 struct kvm_s390_mem_op *mop)
2909{
2910 void __user *uaddr = (void __user *)mop->buf;
2911 void *tmpbuf = NULL;
2912 int r, srcu_idx;
2913 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2914 | KVM_S390_MEMOP_F_CHECK_ONLY;
2915
2916 if (mop->flags & ~supported_flags)
2917 return -EINVAL;
2918
2919 if (mop->size > MEM_OP_MAX_SIZE)
2920 return -E2BIG;
2921
2922 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2923 tmpbuf = vmalloc(mop->size);
2924 if (!tmpbuf)
2925 return -ENOMEM;
2926 }
2927
2928 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2929
2930 switch (mop->op) {
2931 case KVM_S390_MEMOP_LOGICAL_READ:
2932 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2933 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2934 mop->size, GACC_FETCH);
41408c28
TH		 2935 break;
2936 }
2937 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2938 if (r == 0) {
2939 if (copy_to_user(uaddr, tmpbuf, mop->size))
2940 r = -EFAULT;
2941 }
2942 break;
2943 case KVM_S390_MEMOP_LOGICAL_WRITE:
2944 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 2945 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
2946 mop->size, GACC_STORE);
41408c28
TH		 2947 break;
2948 }
2949 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2950 r = -EFAULT;
2951 break;
2952 }
2953 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2954 break;
2955 default:
2956 r = -EINVAL;
2957 }
2958
2959 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2960
2961 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2962 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2963
2964 vfree(tmpbuf);
2965 return r;
2966}
2967
b0c632db
HC		 2968long kvm_arch_vcpu_ioctl(struct file *filp,
2969 unsigned int ioctl, unsigned long arg)
2970{
2971 struct kvm_vcpu *vcpu = filp->private_data;
2972 void __user *argp = (void __user *)arg;
800c1065 2973 int idx;
bc923cc9 2974 long r;
b0c632db 2975
93736624 2976 switch (ioctl) {
47b43c52
JF		 2977 case KVM_S390_IRQ: {
2978 struct kvm_s390_irq s390irq;
2979
2980 r = -EFAULT;
2981 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2982 break;
2983 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2984 break;
2985 }
93736624 2986 case KVM_S390_INTERRUPT: {
ba5c1e9b 2987 struct kvm_s390_interrupt s390int;
383d0b05 2988 struct kvm_s390_irq s390irq;
ba5c1e9b 2989
93736624 2990 r = -EFAULT;
ba5c1e9b 2991 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2992 break;
383d0b05
JF		 2993 if (s390int_to_s390irq(&s390int, &s390irq))
2994 return -EINVAL;
2995 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2996 break;
ba5c1e9b 2997 }
b0c632db 2998 case KVM_S390_STORE_STATUS:
800c1065 2999 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3000 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3001 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3002 break;
b0c632db
HC		 3003 case KVM_S390_SET_INITIAL_PSW: {
3004 psw_t psw;
3005
bc923cc9 3006 r = -EFAULT;
b0c632db 3007 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3008 break;
3009 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3010 break;
b0c632db
HC		 3011 }
3012 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3013 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3014 break;
14eebd91
CO		 3015 case KVM_SET_ONE_REG:
3016 case KVM_GET_ONE_REG: {
3017 struct kvm_one_reg reg;
3018 r = -EFAULT;
3019 if (copy_from_user(&reg, argp, sizeof(reg)))
3020 break;
3021 if (ioctl == KVM_SET_ONE_REG)
3022 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3023 else
3024 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3025 break;
3026 }
27e0393f
CO		 3027#ifdef CONFIG_KVM_S390_UCONTROL
3028 case KVM_S390_UCAS_MAP: {
3029 struct kvm_s390_ucas_mapping ucasmap;
3030
3031 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3032 r = -EFAULT;
3033 break;
3034 }
3035
3036 if (!kvm_is_ucontrol(vcpu->kvm)) {
3037 r = -EINVAL;
3038 break;
3039 }
3040
3041 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3042 ucasmap.vcpu_addr, ucasmap.length);
3043 break;
3044 }
3045 case KVM_S390_UCAS_UNMAP: {
3046 struct kvm_s390_ucas_mapping ucasmap;
3047
3048 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3049 r = -EFAULT;
3050 break;
3051 }
3052
3053 if (!kvm_is_ucontrol(vcpu->kvm)) {
3054 r = -EINVAL;
3055 break;
3056 }
3057
3058 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3059 ucasmap.length);
3060 break;
3061 }
3062#endif
ccc7910f 3063 case KVM_S390_VCPU_FAULT: {
527e30b4 3064 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3065 break;
3066 }
d6712df9
CH		 3067 case KVM_ENABLE_CAP:
3068 {
3069 struct kvm_enable_cap cap;
3070 r = -EFAULT;
3071 if (copy_from_user(&cap, argp, sizeof(cap)))
3072 break;
3073 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3074 break;
3075 }
41408c28
TH		 3076 case KVM_S390_MEM_OP: {
3077 struct kvm_s390_mem_op mem_op;
3078
3079 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3080 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3081 else
3082 r = -EFAULT;
3083 break;
3084 }
816c7667
JF		 3085 case KVM_S390_SET_IRQ_STATE: {
3086 struct kvm_s390_irq_state irq_state;
3087
3088 r = -EFAULT;
3089 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3090 break;
3091 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3092 irq_state.len == 0 ||
3093 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3094 r = -EINVAL;
3095 break;
3096 }
3097 r = kvm_s390_set_irq_state(vcpu,
3098 (void __user *) irq_state.buf,
3099 irq_state.len);
3100 break;
3101 }
3102 case KVM_S390_GET_IRQ_STATE: {
3103 struct kvm_s390_irq_state irq_state;
3104
3105 r = -EFAULT;
3106 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3107 break;
3108 if (irq_state.len == 0) {
3109 r = -EINVAL;
3110 break;
3111 }
3112 r = kvm_s390_get_irq_state(vcpu,
3113 (__u8 __user *) irq_state.buf,
3114 irq_state.len);
3115 break;
3116 }
b0c632db 3117 default:
3e6afcf1 3118 r = -ENOTTY;
b0c632db 3119 }
bc923cc9 3120 return r;
b0c632db
HC		 3121}
3122
5b1c1493
CO		 3123int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3124{
3125#ifdef CONFIG_KVM_S390_UCONTROL
3126 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3127 && (kvm_is_ucontrol(vcpu->kvm))) {
3128 vmf->page = virt_to_page(vcpu->arch.sie_block);
3129 get_page(vmf->page);
3130 return 0;
3131 }
3132#endif
3133 return VM_FAULT_SIGBUS;
3134}
3135
5587027c
AK		 3136int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3137 unsigned long npages)
db3fe4eb
TY		 3138{
3139 return 0;
3140}
3141
b0c632db 3142/* Section: memory related */
f7784b8e
MT		 3143int kvm_arch_prepare_memory_region(struct kvm *kvm,
3144 struct kvm_memory_slot *memslot,
09170a49 3145 const struct kvm_userspace_memory_region *mem,
7b6195a9 3146 enum kvm_mr_change change)
b0c632db 3147{
dd2887e7
NW		 3148 /* A few sanity checks. We can have memory slots which have to be
3149 located/ended at a segment boundary (1MB). The memory in userland is
3150 ok to be fragmented into various different vmas. It is okay to mmap()
3151 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3152
598841ca 3153 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3154 return -EINVAL;
3155
598841ca 3156 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3157 return -EINVAL;
3158
a3a92c31
DD		 3159 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3160 return -EINVAL;
3161
f7784b8e
MT		 3162 return 0;
3163}
3164
3165void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3166 const struct kvm_userspace_memory_region *mem,
8482644a 3167 const struct kvm_memory_slot *old,
f36f3f28 3168 const struct kvm_memory_slot *new,
8482644a 3169 enum kvm_mr_change change)
f7784b8e 3170{
f7850c92 3171 int rc;
f7784b8e 3172
2cef4deb
CB		 3173 /* If the basics of the memslot do not change, we do not want
3174 * to update the gmap. Every update causes several unnecessary
3175 * segment translation exceptions. This is usually handled just
3176 * fine by the normal fault handler + gmap, but it will also
3177 * cause faults on the prefix page of running guest CPUs.
3178 */
3179 if (old->userspace_addr == mem->userspace_addr &&
3180 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3181 old->npages * PAGE_SIZE == mem->memory_size)
3182 return;
598841ca
CO		 3183
3184 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3185 mem->guest_phys_addr, mem->memory_size);
3186 if (rc)
ea2cdd27 3187 pr_warn("failed to commit memory region\n");
598841ca 3188 return;
b0c632db
HC		 3189}
3190
60a37709
AY		 3191static inline unsigned long nonhyp_mask(int i)
3192{
3193 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3194
3195 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3196}
3197
3491caf2
CB		 3198void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3199{
3200 vcpu->valid_wakeup = false;
3201}
3202
b0c632db
HC		 3203static int __init kvm_s390_init(void)
3204{
60a37709
AY		 3205 int i;
3206
07197fd0
DH		 3207 if (!sclp.has_sief2) {
3208 pr_info("SIE not available\n");
3209 return -ENODEV;
3210 }
3211
60a37709
AY		 3212 for (i = 0; i < 16; i++)
3213 kvm_s390_fac_list_mask[i] |=
3214 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3215
9d8d5786 3216 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 3217}
3218
3219static void __exit kvm_s390_exit(void)
3220{
3221 kvm_exit();
3222}
3223
3224module_init(kvm_s390_init);
3225module_exit(kvm_s390_exit);
566af940
CH		 3226
3227/*
3228 * Enable autoloading of the kvm module.
3229 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3230 * since x86 takes a different approach.
3231 */
3232#include <linux/miscdevice.h>
3233MODULE_ALIAS_MISCDEV(KVM_MINOR);
3234MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)