projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: take care of clock-comparator sign control
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
d809aa23 1// SPDX-License-Identifier: GPL-2.0
b0c632db 2/*
bb64da9a 3 * hosting IBM Z kernel virtual machines (s390x)
b0c632db 4 *
a37cb07a 5 * Copyright IBM Corp. 2008, 2018
b0c632db
HC		 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 11 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 12 */
13
14#include <linux/compiler.h>
15#include <linux/err.h>
16#include <linux/fs.h>
ca872302 17#include <linux/hrtimer.h>
b0c632db
HC		 18#include <linux/init.h>
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
b2d73b2a 21#include <linux/mman.h>
b0c632db 22#include <linux/module.h>
d3217967 23#include <linux/moduleparam.h>
a374e892 24#include <linux/random.h>
b0c632db 25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
41408c28 27#include <linux/vmalloc.h>
15c9705f 28#include <linux/bitmap.h>
174cd4b1 29#include <linux/sched/signal.h>
190df4a2 30#include <linux/string.h>
174cd4b1 31
cbb870c8 32#include <asm/asm-offsets.h>
b0c632db 33#include <asm/lowcore.h>
fd5ada04 34#include <asm/stp.h>
b0c632db 35#include <asm/pgtable.h>
1e133ab2 36#include <asm/gmap.h>
f5daba1d 37#include <asm/nmi.h>
a0616cde 38#include <asm/switch_to.h>
6d3da241 39#include <asm/isc.h>
1526bf9c 40#include <asm/sclp.h>
0a763c78 41#include <asm/cpacf.h>
221bb8a4 42#include <asm/timex.h>
8f2abe6a 43#include "kvm-s390.h"
b0c632db
HC		 44#include "gaccess.h"
45
ea2cdd27
DH		 46#define KMSG_COMPONENT "kvm-s390"
47#undef pr_fmt
48#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
49
5786fffa
CH		 50#define CREATE_TRACE_POINTS
51#include "trace.h"
ade38c31 52#include "trace-s390.h"
5786fffa 53
41408c28 54#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 55#define LOCAL_IRQS 32
56#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 58
b0c632db
HC		 59#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60
61struct kvm_stats_debugfs_item debugfs_entries[] = {
62 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 63 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 64 { "exit_validity", VCPU_STAT(exit_validity) },
65 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
66 { "exit_external_request", VCPU_STAT(exit_external_request) },
67 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 68 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 69 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 70 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
71 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 72 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 73 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 74 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 75 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
ce2e4f0b 76 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 77 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 78 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 79 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
80 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 81 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 82 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 83 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
84 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
85 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
86 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
89 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
a37cb07a
CB		 90 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
91 { "instruction_gs", VCPU_STAT(instruction_gs) },
92 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
93 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
94 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
69d0d3a3 95 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
a37cb07a 96 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
453423dc 97 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
a37cb07a
CB		 98 { "instruction_sck", VCPU_STAT(instruction_sck) },
99 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
453423dc
CB		 100 { "instruction_spx", VCPU_STAT(instruction_spx) },
101 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
102 { "instruction_stap", VCPU_STAT(instruction_stap) },
a37cb07a
CB		 103 { "instruction_iske", VCPU_STAT(instruction_iske) },
104 { "instruction_ri", VCPU_STAT(instruction_ri) },
105 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
106 { "instruction_sske", VCPU_STAT(instruction_sske) },
8a242234 107 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
b31288fa 108 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 109 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
110 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
a37cb07a
CB		 111 { "instruction_tb", VCPU_STAT(instruction_tb) },
112 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
bb25b9ba 113 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
a37cb07a 114 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
95ca2cb5 115 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 116 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 117 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 118 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 119 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 120 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 121 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
122 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 123 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 124 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
125 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 126 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 127 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
128 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
129 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 130 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
131 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
132 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
866c138c
CB		 133 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
134 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
135 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
136 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
137 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
138 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
a37cb07a 139 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
b0c632db
HC		 140 { NULL }
141};
142
8fa1696e
CW		 143struct kvm_s390_tod_clock_ext {
144 __u8 epoch_idx;
145 __u64 tod;
146 __u8 reserved[7];
147} __packed;
148
a411edf1
DH		 149/* allow nested virtualization in KVM (if enabled by user space) */
150static int nested;
151module_param(nested, int, S_IRUGO);
152MODULE_PARM_DESC(nested, "Nested virtualization support");
153
9d8d5786 154/* upper facilities limit for kvm */
f6c1d359 155unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM };
b0c632db 156
9d8d5786 157unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 158{
9d8d5786
MM		 159 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
160 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 161}
162
15c9705f
DH		 163/* available cpu features supported by kvm */
164static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 165/* available subfunctions indicated via query / "test bit" */
166static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 167
9d8d5786 168static struct gmap_notifier gmap_notifier;
a3508fbe 169static struct gmap_notifier vsie_gmap_notifier;
78f26131 170debug_info_t *kvm_s390_dbf;
9d8d5786 171
b0c632db 172/* Section: not file related */
13a34e06 173int kvm_arch_hardware_enable(void)
b0c632db
HC		 174{
175 /* every s390 is virtualization enabled ;-) */
10474ae8 176 return 0;
b0c632db
HC		 177}
178
414d3b07
MS		 179static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
180 unsigned long end);
2c70fe44 181
fdf03650
FZ		 182/*
183 * This callback is executed during stop_machine(). All CPUs are therefore
184 * temporarily stopped. In order not to change guest behavior, we have to
185 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
186 * so a CPU won't be stopped while calculating with the epoch.
187 */
188static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
189 void *v)
190{
191 struct kvm *kvm;
192 struct kvm_vcpu *vcpu;
193 int i;
194 unsigned long long *delta = v;
195
196 list_for_each_entry(kvm, &vm_list, vm_list) {
197 kvm->arch.epoch -= *delta;
198 kvm_for_each_vcpu(i, vcpu, kvm) {
199 vcpu->arch.sie_block->epoch -= *delta;
db0758b2
DH		 200 if (vcpu->arch.cputm_enabled)
201 vcpu->arch.cputm_start += *delta;
91473b48
DH		 202 if (vcpu->arch.vsie_block)
203 vcpu->arch.vsie_block->epoch -= *delta;
fdf03650
FZ		 204 }
205 }
206 return NOTIFY_OK;
207}
208
209static struct notifier_block kvm_clock_notifier = {
210 .notifier_call = kvm_clock_sync,
211};
212
b0c632db
HC		 213int kvm_arch_hardware_setup(void)
214{
2c70fe44 215 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 216 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 217 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
218 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 219 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
220 &kvm_clock_notifier);
b0c632db
HC		 221 return 0;
222}
223
224void kvm_arch_hardware_unsetup(void)
225{
b2d73b2a 226 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 227 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 228 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
229 &kvm_clock_notifier);
b0c632db
HC		 230}
231
22be5a13
DH		 232static void allow_cpu_feat(unsigned long nr)
233{
234 set_bit_inv(nr, kvm_s390_available_cpu_feat);
235}
236
0a763c78
DH		 237static inline int plo_test_bit(unsigned char nr)
238{
239 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 240 int cc;
0a763c78
DH		 241
242 asm volatile(
243 /* Parameter registers are ignored for "test bit" */
244 " plo 0,0,0,0(0)\n"
245 " ipm %0\n"
246 " srl %0,28\n"
247 : "=d" (cc)
248 : "d" (r0)
249 : "cc");
250 return cc == 0;
251}
252
22be5a13
DH		 253static void kvm_s390_cpu_feat_init(void)
254{
0a763c78
DH		 255 int i;
256
257 for (i = 0; i < 256; ++i) {
258 if (plo_test_bit(i))
259 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
260 }
261
262 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 263 ptff(kvm_s390_available_subfunc.ptff,
264 sizeof(kvm_s390_available_subfunc.ptff),
265 PTFF_QAF);
0a763c78
DH		 266
267 if (test_facility(17)) { /* MSA */
69c0e360
MS		 268 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
269 kvm_s390_available_subfunc.kmac);
270 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
271 kvm_s390_available_subfunc.kmc);
272 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
273 kvm_s390_available_subfunc.km);
274 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
275 kvm_s390_available_subfunc.kimd);
276 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
277 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 278 }
279 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 280 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
281 kvm_s390_available_subfunc.pckmo);
0a763c78 282 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 283 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
284 kvm_s390_available_subfunc.kmctr);
285 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
286 kvm_s390_available_subfunc.kmf);
287 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
288 kvm_s390_available_subfunc.kmo);
289 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
290 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 291 }
292 if (test_facility(57)) /* MSA5 */
985a9d20 293 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 294 kvm_s390_available_subfunc.ppno);
0a763c78 295
e000b8e0
JH		 296 if (test_facility(146)) /* MSA8 */
297 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
298 kvm_s390_available_subfunc.kma);
299
22be5a13
DH		 300 if (MACHINE_HAS_ESOP)
301 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 302 /*
303 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
304 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
305 */
306 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 307 !test_facility(3) || !nested)
a3508fbe
DH		 308 return;
309 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 310 if (sclp.has_64bscao)
311 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 312 if (sclp.has_siif)
313 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 314 if (sclp.has_gpere)
315 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 316 if (sclp.has_gsls)
317 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 318 if (sclp.has_ib)
319 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 320 if (sclp.has_cei)
321 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 322 if (sclp.has_ibs)
323 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 324 if (sclp.has_kss)
325 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 326 /*
327 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
328 * all skey handling functions read/set the skey from the PGSTE
329 * instead of the real storage key.
330 *
331 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
332 * pages being detected as preserved although they are resident.
333 *
334 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
335 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
336 *
337 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
338 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
339 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
340 *
341 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
342 * cannot easily shadow the SCA because of the ipte lock.
343 */
22be5a13
DH		 344}
345
b0c632db
HC		 346int kvm_arch_init(void *opaque)
347{
78f26131
CB		 348 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
349 if (!kvm_s390_dbf)
350 return -ENOMEM;
351
352 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
353 debug_unregister(kvm_s390_dbf);
354 return -ENOMEM;
355 }
356
22be5a13
DH		 357 kvm_s390_cpu_feat_init();
358
84877d93
CH		 359 /* Register floating interrupt controller interface. */
360 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 361}
362
78f26131
CB		 363void kvm_arch_exit(void)
364{
365 debug_unregister(kvm_s390_dbf);
366}
367
b0c632db
HC		 368/* Section: device related */
369long kvm_arch_dev_ioctl(struct file *filp,
370 unsigned int ioctl, unsigned long arg)
371{
372 if (ioctl == KVM_S390_ENABLE_SIE)
373 return s390_enable_sie();
374 return -EINVAL;
375}
376
784aa3d7 377int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 378{
d7b0b5eb
CO		 379 int r;
380
2bd0ac4e 381 switch (ext) {
d7b0b5eb 382 case KVM_CAP_S390_PSW:
b6cf8788 383 case KVM_CAP_S390_GMAP:
52e16b18 384 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 385#ifdef CONFIG_KVM_S390_UCONTROL
386 case KVM_CAP_S390_UCONTROL:
387#endif
3c038e6b 388 case KVM_CAP_ASYNC_PF:
60b413c9 389 case KVM_CAP_SYNC_REGS:
14eebd91 390 case KVM_CAP_ONE_REG:
d6712df9 391 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 392 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 393 case KVM_CAP_IOEVENTFD:
c05c4186 394 case KVM_CAP_DEVICE_CTRL:
d938dc55 395 case KVM_CAP_ENABLE_CAP_VM:
78599d90 396 case KVM_CAP_S390_IRQCHIP:
f2061656 397 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 398 case KVM_CAP_MP_STATE:
460df4c1 399 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 400 case KVM_CAP_S390_INJECT_IRQ:
2444b352 401 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 402 case KVM_CAP_S390_USER_STSI:
30ee2a98 403 case KVM_CAP_S390_SKEYS:
816c7667 404 case KVM_CAP_S390_IRQ_STATE:
6502a34c 405 case KVM_CAP_S390_USER_INSTR0:
4036e387 406 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 407 case KVM_CAP_S390_AIS:
da9a1446 408 case KVM_CAP_S390_AIS_MIGRATION:
d7b0b5eb
CO		 409 r = 1;
410 break;
41408c28
TH		 411 case KVM_CAP_S390_MEM_OP:
412 r = MEM_OP_MAX_SIZE;
413 break;
e726b1bd
CB		 414 case KVM_CAP_NR_VCPUS:
415 case KVM_CAP_MAX_VCPUS:
76a6dd72 416 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 417 if (!kvm_s390_use_sca_entries())
418 r = KVM_MAX_VCPUS;
419 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 420 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 421 break;
e1e2e605
NW		 422 case KVM_CAP_NR_MEMSLOTS:
423 r = KVM_USER_MEM_SLOTS;
424 break;
1526bf9c 425 case KVM_CAP_S390_COW:
abf09bed 426 r = MACHINE_HAS_ESOP;
1526bf9c 427 break;
68c55750
EF		 428 case KVM_CAP_S390_VECTOR_REGISTERS:
429 r = MACHINE_HAS_VX;
430 break;
c6e5f166
FZ		 431 case KVM_CAP_S390_RI:
432 r = test_facility(64);
433 break;
4e0b1ab7
FZ		 434 case KVM_CAP_S390_GS:
435 r = test_facility(133);
436 break;
35b3fde6
CB		 437 case KVM_CAP_S390_BPB:
438 r = test_facility(82);
439 break;
2bd0ac4e 440 default:
d7b0b5eb 441 r = 0;
2bd0ac4e 442 }
d7b0b5eb 443 return r;
b0c632db
HC		 444}
445
15f36ebd
JH		 446static void kvm_s390_sync_dirty_log(struct kvm *kvm,
447 struct kvm_memory_slot *memslot)
448{
449 gfn_t cur_gfn, last_gfn;
450 unsigned long address;
451 struct gmap *gmap = kvm->arch.gmap;
452
15f36ebd
JH		 453 /* Loop over all guest pages */
454 last_gfn = memslot->base_gfn + memslot->npages;
455 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
456 address = gfn_to_hva_memslot(memslot, cur_gfn);
457
1e133ab2 458 if (test_and_clear_guest_dirty(gmap->mm, address))
15f36ebd 459 mark_page_dirty(kvm, cur_gfn);
1763f8d0
CB		 460 if (fatal_signal_pending(current))
461 return;
70c88a00 462 cond_resched();
15f36ebd 463 }
15f36ebd
JH		 464}
465
b0c632db 466/* Section: vm related */
a6e2f683
ED		 467static void sca_del_vcpu(struct kvm_vcpu *vcpu);
468
b0c632db
HC		 469/*
470 * Get (and clear) the dirty memory log for a memory slot.
471 */
472int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
473 struct kvm_dirty_log *log)
474{
15f36ebd
JH		 475 int r;
476 unsigned long n;
9f6b8029 477 struct kvm_memslots *slots;
15f36ebd
JH		 478 struct kvm_memory_slot *memslot;
479 int is_dirty = 0;
480
e1e8a962
JF		 481 if (kvm_is_ucontrol(kvm))
482 return -EINVAL;
483
15f36ebd
JH		 484 mutex_lock(&kvm->slots_lock);
485
486 r = -EINVAL;
487 if (log->slot >= KVM_USER_MEM_SLOTS)
488 goto out;
489
9f6b8029
PB		 490 slots = kvm_memslots(kvm);
491 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 492 r = -ENOENT;
493 if (!memslot->dirty_bitmap)
494 goto out;
495
496 kvm_s390_sync_dirty_log(kvm, memslot);
497 r = kvm_get_dirty_log(kvm, log, &is_dirty);
498 if (r)
499 goto out;
500
501 /* Clear the dirty log */
502 if (is_dirty) {
503 n = kvm_dirty_bitmap_bytes(memslot);
504 memset(memslot->dirty_bitmap, 0, n);
505 }
506 r = 0;
507out:
508 mutex_unlock(&kvm->slots_lock);
509 return r;
b0c632db
HC		 510}
511
6502a34c
DH		 512static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
513{
514 unsigned int i;
515 struct kvm_vcpu *vcpu;
516
517 kvm_for_each_vcpu(i, vcpu, kvm) {
518 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
519 }
520}
521
d938dc55
CH		 522static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
523{
524 int r;
525
526 if (cap->flags)
527 return -EINVAL;
528
529 switch (cap->cap) {
84223598 530 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 531 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 532 kvm->arch.use_irqchip = 1;
533 r = 0;
534 break;
2444b352 535 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 536 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 537 kvm->arch.user_sigp = 1;
538 r = 0;
539 break;
68c55750 540 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 541 mutex_lock(&kvm->lock);
a03825bb 542 if (kvm->created_vcpus) {
5967c17b
DH		 543 r = -EBUSY;
544 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 545 set_kvm_facility(kvm->arch.model.fac_mask, 129);
546 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 547 if (test_facility(134)) {
548 set_kvm_facility(kvm->arch.model.fac_mask, 134);
549 set_kvm_facility(kvm->arch.model.fac_list, 134);
550 }
53743aa7
MS		 551 if (test_facility(135)) {
552 set_kvm_facility(kvm->arch.model.fac_mask, 135);
553 set_kvm_facility(kvm->arch.model.fac_list, 135);
554 }
18280d8b
MM		 555 r = 0;
556 } else
557 r = -EINVAL;
5967c17b 558 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 559 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
560 r ? "(not available)" : "(success)");
68c55750 561 break;
c6e5f166
FZ		 562 case KVM_CAP_S390_RI:
563 r = -EINVAL;
564 mutex_lock(&kvm->lock);
a03825bb 565 if (kvm->created_vcpus) {
c6e5f166
FZ		 566 r = -EBUSY;
567 } else if (test_facility(64)) {
c54f0d6a
DH		 568 set_kvm_facility(kvm->arch.model.fac_mask, 64);
569 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 570 r = 0;
571 }
572 mutex_unlock(&kvm->lock);
573 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
574 r ? "(not available)" : "(success)");
575 break;
47a4693e
YMZ		 576 case KVM_CAP_S390_AIS:
577 mutex_lock(&kvm->lock);
578 if (kvm->created_vcpus) {
579 r = -EBUSY;
580 } else {
581 set_kvm_facility(kvm->arch.model.fac_mask, 72);
582 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 583 r = 0;
584 }
585 mutex_unlock(&kvm->lock);
586 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
587 r ? "(not available)" : "(success)");
588 break;
4e0b1ab7
FZ		 589 case KVM_CAP_S390_GS:
590 r = -EINVAL;
591 mutex_lock(&kvm->lock);
241e3ec0 592 if (kvm->created_vcpus) {
4e0b1ab7
FZ		 593 r = -EBUSY;
594 } else if (test_facility(133)) {
595 set_kvm_facility(kvm->arch.model.fac_mask, 133);
596 set_kvm_facility(kvm->arch.model.fac_list, 133);
597 r = 0;
598 }
599 mutex_unlock(&kvm->lock);
600 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
601 r ? "(not available)" : "(success)");
602 break;
e44fc8c9 603 case KVM_CAP_S390_USER_STSI:
c92ea7b9 604 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 605 kvm->arch.user_stsi = 1;
606 r = 0;
607 break;
6502a34c
DH		 608 case KVM_CAP_S390_USER_INSTR0:
609 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
610 kvm->arch.user_instr0 = 1;
611 icpt_operexc_on_all_vcpus(kvm);
612 r = 0;
613 break;
d938dc55
CH		 614 default:
615 r = -EINVAL;
616 break;
617 }
618 return r;
619}
620
8c0a7ce6
DD		 621static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
622{
623 int ret;
624
625 switch (attr->attr) {
626 case KVM_S390_VM_MEM_LIMIT_SIZE:
627 ret = 0;
c92ea7b9 628 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 629 kvm->arch.mem_limit);
630 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 631 ret = -EFAULT;
632 break;
633 default:
634 ret = -ENXIO;
635 break;
636 }
637 return ret;
638}
639
640static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 641{
642 int ret;
643 unsigned int idx;
644 switch (attr->attr) {
645 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 646 ret = -ENXIO;
c24cc9c8 647 if (!sclp.has_cmma)
e6db1d61
DD		 648 break;
649
4f718eab 650 ret = -EBUSY;
c92ea7b9 651 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 652 mutex_lock(&kvm->lock);
a03825bb 653 if (!kvm->created_vcpus) {
4f718eab
DD		 654 kvm->arch.use_cmma = 1;
655 ret = 0;
656 }
657 mutex_unlock(&kvm->lock);
658 break;
659 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 660 ret = -ENXIO;
661 if (!sclp.has_cmma)
662 break;
c3489155
DD		 663 ret = -EINVAL;
664 if (!kvm->arch.use_cmma)
665 break;
666
c92ea7b9 667 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 668 mutex_lock(&kvm->lock);
669 idx = srcu_read_lock(&kvm->srcu);
a13cff31 670 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 671 srcu_read_unlock(&kvm->srcu, idx);
672 mutex_unlock(&kvm->lock);
673 ret = 0;
674 break;
8c0a7ce6
DD		 675 case KVM_S390_VM_MEM_LIMIT_SIZE: {
676 unsigned long new_limit;
677
678 if (kvm_is_ucontrol(kvm))
679 return -EINVAL;
680
681 if (get_user(new_limit, (u64 __user *)attr->addr))
682 return -EFAULT;
683
a3a92c31
DD		 684 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
685 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 686 return -E2BIG;
687
a3a92c31
DD		 688 if (!new_limit)
689 return -EINVAL;
690
6ea427bb 691 /* gmap_create takes last usable address */
a3a92c31
DD		 692 if (new_limit != KVM_S390_NO_MEM_LIMIT)
693 new_limit -= 1;
694
8c0a7ce6
DD		 695 ret = -EBUSY;
696 mutex_lock(&kvm->lock);
a03825bb 697 if (!kvm->created_vcpus) {
6ea427bb
MS		 698 /* gmap_create will round the limit up */
699 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 700
701 if (!new) {
702 ret = -ENOMEM;
703 } else {
6ea427bb 704 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 705 new->private = kvm;
706 kvm->arch.gmap = new;
707 ret = 0;
708 }
709 }
710 mutex_unlock(&kvm->lock);
a3a92c31
DD		 711 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
712 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
713 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 714 break;
715 }
4f718eab
DD		 716 default:
717 ret = -ENXIO;
718 break;
719 }
720 return ret;
721}
722
a374e892
TK		 723static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
724
725static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
726{
727 struct kvm_vcpu *vcpu;
728 int i;
729
9d8d5786 730 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 731 return -EINVAL;
732
733 mutex_lock(&kvm->lock);
734 switch (attr->attr) {
735 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
736 get_random_bytes(
737 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
738 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
739 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 740 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 741 break;
742 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
743 get_random_bytes(
744 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
745 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
746 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 747 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 748 break;
749 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
750 kvm->arch.crypto.aes_kw = 0;
751 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
752 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 753 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 754 break;
755 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
756 kvm->arch.crypto.dea_kw = 0;
757 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
758 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 759 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892
TK		 760 break;
761 default:
762 mutex_unlock(&kvm->lock);
763 return -ENXIO;
764 }
765
766 kvm_for_each_vcpu(i, vcpu, kvm) {
767 kvm_s390_vcpu_crypto_setup(vcpu);
768 exit_sie(vcpu);
769 }
770 mutex_unlock(&kvm->lock);
771 return 0;
772}
773
190df4a2
CI		 774static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
775{
776 int cx;
777 struct kvm_vcpu *vcpu;
778
779 kvm_for_each_vcpu(cx, vcpu, kvm)
780 kvm_s390_sync_request(req, vcpu);
781}
782
783/*
784 * Must be called with kvm->srcu held to avoid races on memslots, and with
1de1ea7e 785 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
190df4a2
CI		 786 */
787static int kvm_s390_vm_start_migration(struct kvm *kvm)
788{
789 struct kvm_s390_migration_state *mgs;
790 struct kvm_memory_slot *ms;
791 /* should be the only one */
792 struct kvm_memslots *slots;
793 unsigned long ram_pages;
794 int slotnr;
795
796 /* migration mode already enabled */
797 if (kvm->arch.migration_state)
798 return 0;
799
800 slots = kvm_memslots(kvm);
801 if (!slots || !slots->used_slots)
802 return -EINVAL;
803
804 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
805 if (!mgs)
806 return -ENOMEM;
807 kvm->arch.migration_state = mgs;
808
809 if (kvm->arch.use_cmma) {
810 /*
32aa144f
CB		 811 * Get the first slot. They are reverse sorted by base_gfn, so
812 * the first slot is also the one at the end of the address
813 * space. We have verified above that at least one slot is
814 * present.
190df4a2 815 */
32aa144f 816 ms = slots->memslots;
190df4a2
CI		 817 /* round up so we only use full longs */
818 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
819 /* allocate enough bytes to store all the bits */
820 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
821 if (!mgs->pgste_bitmap) {
822 kfree(mgs);
823 kvm->arch.migration_state = NULL;
824 return -ENOMEM;
825 }
826
827 mgs->bitmap_size = ram_pages;
828 atomic64_set(&mgs->dirty_pages, ram_pages);
829 /* mark all the pages in active slots as dirty */
830 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
831 ms = slots->memslots + slotnr;
832 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
833 }
834
835 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
836 }
837 return 0;
838}
839
840/*
1de1ea7e 841 * Must be called with kvm->slots_lock to avoid races with ourselves and
190df4a2
CI		 842 * kvm_s390_vm_start_migration.
843 */
844static int kvm_s390_vm_stop_migration(struct kvm *kvm)
845{
846 struct kvm_s390_migration_state *mgs;
847
848 /* migration mode already disabled */
849 if (!kvm->arch.migration_state)
850 return 0;
851 mgs = kvm->arch.migration_state;
852 kvm->arch.migration_state = NULL;
853
854 if (kvm->arch.use_cmma) {
855 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1de1ea7e
CB		 856 /* We have to wait for the essa emulation to finish */
857 synchronize_srcu(&kvm->srcu);
190df4a2
CI		 858 vfree(mgs->pgste_bitmap);
859 }
860 kfree(mgs);
861 return 0;
862}
863
864static int kvm_s390_vm_set_migration(struct kvm *kvm,
865 struct kvm_device_attr *attr)
866{
1de1ea7e 867 int res = -ENXIO;
190df4a2 868
1de1ea7e 869 mutex_lock(&kvm->slots_lock);
190df4a2
CI		 870 switch (attr->attr) {
871 case KVM_S390_VM_MIGRATION_START:
190df4a2 872 res = kvm_s390_vm_start_migration(kvm);
190df4a2
CI		 873 break;
874 case KVM_S390_VM_MIGRATION_STOP:
875 res = kvm_s390_vm_stop_migration(kvm);
876 break;
877 default:
878 break;
879 }
1de1ea7e 880 mutex_unlock(&kvm->slots_lock);
190df4a2
CI		 881
882 return res;
883}
884
885static int kvm_s390_vm_get_migration(struct kvm *kvm,
886 struct kvm_device_attr *attr)
887{
888 u64 mig = (kvm->arch.migration_state != NULL);
889
890 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
891 return -ENXIO;
892
893 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
894 return -EFAULT;
895 return 0;
896}
897
8fa1696e
CW		 898static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
899{
900 struct kvm_s390_vm_tod_clock gtod;
901
902 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
903 return -EFAULT;
904
905 if (test_kvm_facility(kvm, 139))
906 kvm_s390_set_tod_clock_ext(kvm, &gtod);
907 else if (gtod.epoch_idx == 0)
908 kvm_s390_set_tod_clock(kvm, gtod.tod);
909 else
910 return -EINVAL;
911
912 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
913 gtod.epoch_idx, gtod.tod);
914
915 return 0;
916}
917
72f25020
JH		 918static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
919{
920 u8 gtod_high;
921
922 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
923 sizeof(gtod_high)))
924 return -EFAULT;
925
926 if (gtod_high != 0)
927 return -EINVAL;
58c383c6 928 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 929
930 return 0;
931}
932
933static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
934{
5a3d883a 935 u64 gtod;
72f25020
JH		 936
937 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
938 return -EFAULT;
939
25ed1675 940 kvm_s390_set_tod_clock(kvm, gtod);
58c383c6 941 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
72f25020
JH		 942 return 0;
943}
944
945static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
946{
947 int ret;
948
949 if (attr->flags)
950 return -EINVAL;
951
952 switch (attr->attr) {
8fa1696e
CW		 953 case KVM_S390_VM_TOD_EXT:
954 ret = kvm_s390_set_tod_ext(kvm, attr);
955 break;
72f25020
JH		 956 case KVM_S390_VM_TOD_HIGH:
957 ret = kvm_s390_set_tod_high(kvm, attr);
958 break;
959 case KVM_S390_VM_TOD_LOW:
960 ret = kvm_s390_set_tod_low(kvm, attr);
961 break;
962 default:
963 ret = -ENXIO;
964 break;
965 }
966 return ret;
967}
968
8fa1696e
CW		 969static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
970 struct kvm_s390_vm_tod_clock *gtod)
971{
972 struct kvm_s390_tod_clock_ext htod;
973
974 preempt_disable();
975
976 get_tod_clock_ext((char *)&htod);
977
978 gtod->tod = htod.tod + kvm->arch.epoch;
979 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
980
981 if (gtod->tod < htod.tod)
982 gtod->epoch_idx += 1;
983
984 preempt_enable();
985}
986
987static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
988{
989 struct kvm_s390_vm_tod_clock gtod;
990
991 memset(&gtod, 0, sizeof(gtod));
992
993 if (test_kvm_facility(kvm, 139))
994 kvm_s390_get_tod_clock_ext(kvm, &gtod);
995 else
996 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
997
998 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
999 return -EFAULT;
1000
1001 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1002 gtod.epoch_idx, gtod.tod);
1003 return 0;
1004}
1005
72f25020
JH		 1006static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1007{
1008 u8 gtod_high = 0;
1009
1010 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1011 sizeof(gtod_high)))
1012 return -EFAULT;
58c383c6 1013 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1014
1015 return 0;
1016}
1017
1018static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1019{
5a3d883a 1020 u64 gtod;
72f25020 1021
60417fcc 1022 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1023 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1024 return -EFAULT;
58c383c6 1025 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1026
1027 return 0;
1028}
1029
1030static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1031{
1032 int ret;
1033
1034 if (attr->flags)
1035 return -EINVAL;
1036
1037 switch (attr->attr) {
8fa1696e
CW		 1038 case KVM_S390_VM_TOD_EXT:
1039 ret = kvm_s390_get_tod_ext(kvm, attr);
1040 break;
72f25020
JH		 1041 case KVM_S390_VM_TOD_HIGH:
1042 ret = kvm_s390_get_tod_high(kvm, attr);
1043 break;
1044 case KVM_S390_VM_TOD_LOW:
1045 ret = kvm_s390_get_tod_low(kvm, attr);
1046 break;
1047 default:
1048 ret = -ENXIO;
1049 break;
1050 }
1051 return ret;
1052}
1053
658b6eda
MM		 1054static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1055{
1056 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1057 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1058 int ret = 0;
1059
1060 mutex_lock(&kvm->lock);
a03825bb 1061 if (kvm->created_vcpus) {
658b6eda
MM		 1062 ret = -EBUSY;
1063 goto out;
1064 }
1065 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1066 if (!proc) {
1067 ret = -ENOMEM;
1068 goto out;
1069 }
1070 if (!copy_from_user(proc, (void __user *)attr->addr,
1071 sizeof(*proc))) {
9bb0ec09 1072 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1073 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1074 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1075 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1076 if (proc->ibc > unblocked_ibc)
1077 kvm->arch.model.ibc = unblocked_ibc;
1078 else if (proc->ibc < lowest_ibc)
1079 kvm->arch.model.ibc = lowest_ibc;
1080 else
1081 kvm->arch.model.ibc = proc->ibc;
1082 }
c54f0d6a 1083 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1084 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1085 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1086 kvm->arch.model.ibc,
1087 kvm->arch.model.cpuid);
1088 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1089 kvm->arch.model.fac_list[0],
1090 kvm->arch.model.fac_list[1],
1091 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1092 } else
1093 ret = -EFAULT;
1094 kfree(proc);
1095out:
1096 mutex_unlock(&kvm->lock);
1097 return ret;
1098}
1099
15c9705f
DH		 1100static int kvm_s390_set_processor_feat(struct kvm *kvm,
1101 struct kvm_device_attr *attr)
1102{
1103 struct kvm_s390_vm_cpu_feat data;
15c9705f
DH		 1104
1105 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1106 return -EFAULT;
1107 if (!bitmap_subset((unsigned long *) data.feat,
1108 kvm_s390_available_cpu_feat,
1109 KVM_S390_VM_CPU_FEAT_NR_BITS))
1110 return -EINVAL;
1111
1112 mutex_lock(&kvm->lock);
2f8311c9
CB		 1113 if (kvm->created_vcpus) {
1114 mutex_unlock(&kvm->lock);
1115 return -EBUSY;
15c9705f 1116 }
2f8311c9
CB		 1117 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1118 KVM_S390_VM_CPU_FEAT_NR_BITS);
15c9705f 1119 mutex_unlock(&kvm->lock);
2f8311c9
CB		 1120 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1121 data.feat[0],
1122 data.feat[1],
1123 data.feat[2]);
1124 return 0;
15c9705f
DH		 1125}
1126
0a763c78
DH		 1127static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1128 struct kvm_device_attr *attr)
1129{
1130 /*
1131 * Once supported by kernel + hw, we have to store the subfunctions
1132 * in kvm->arch and remember that user space configured them.
1133 */
1134 return -ENXIO;
1135}
1136
658b6eda
MM		 1137static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1138{
1139 int ret = -ENXIO;
1140
1141 switch (attr->attr) {
1142 case KVM_S390_VM_CPU_PROCESSOR:
1143 ret = kvm_s390_set_processor(kvm, attr);
1144 break;
15c9705f
DH		 1145 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1146 ret = kvm_s390_set_processor_feat(kvm, attr);
1147 break;
0a763c78
DH		 1148 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1149 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1150 break;
658b6eda
MM		 1151 }
1152 return ret;
1153}
1154
1155static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1156{
1157 struct kvm_s390_vm_cpu_processor *proc;
1158 int ret = 0;
1159
1160 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1161 if (!proc) {
1162 ret = -ENOMEM;
1163 goto out;
1164 }
9bb0ec09 1165 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1166 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1167 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1168 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1169 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1170 kvm->arch.model.ibc,
1171 kvm->arch.model.cpuid);
1172 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1173 kvm->arch.model.fac_list[0],
1174 kvm->arch.model.fac_list[1],
1175 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1176 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1177 ret = -EFAULT;
1178 kfree(proc);
1179out:
1180 return ret;
1181}
1182
1183static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1184{
1185 struct kvm_s390_vm_cpu_machine *mach;
1186 int ret = 0;
1187
1188 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1189 if (!mach) {
1190 ret = -ENOMEM;
1191 goto out;
1192 }
1193 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1194 mach->ibc = sclp.ibc;
c54f0d6a 1195 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1196 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1197 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1198 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1199 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1200 kvm->arch.model.ibc,
1201 kvm->arch.model.cpuid);
1202 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1203 mach->fac_mask[0],
1204 mach->fac_mask[1],
1205 mach->fac_mask[2]);
1206 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1207 mach->fac_list[0],
1208 mach->fac_list[1],
1209 mach->fac_list[2]);
658b6eda
MM		 1210 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1211 ret = -EFAULT;
1212 kfree(mach);
1213out:
1214 return ret;
1215}
1216
15c9705f
DH		 1217static int kvm_s390_get_processor_feat(struct kvm *kvm,
1218 struct kvm_device_attr *attr)
1219{
1220 struct kvm_s390_vm_cpu_feat data;
1221
1222 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1223 KVM_S390_VM_CPU_FEAT_NR_BITS);
1224 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1225 return -EFAULT;
2f8311c9
CB		 1226 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1227 data.feat[0],
1228 data.feat[1],
1229 data.feat[2]);
15c9705f
DH		 1230 return 0;
1231}
1232
1233static int kvm_s390_get_machine_feat(struct kvm *kvm,
1234 struct kvm_device_attr *attr)
1235{
1236 struct kvm_s390_vm_cpu_feat data;
1237
1238 bitmap_copy((unsigned long *) data.feat,
1239 kvm_s390_available_cpu_feat,
1240 KVM_S390_VM_CPU_FEAT_NR_BITS);
1241 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1242 return -EFAULT;
2f8311c9
CB		 1243 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1244 data.feat[0],
1245 data.feat[1],
1246 data.feat[2]);
15c9705f
DH		 1247 return 0;
1248}
1249
0a763c78
DH		 1250static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1251 struct kvm_device_attr *attr)
1252{
1253 /*
1254 * Once we can actually configure subfunctions (kernel + hw support),
1255 * we have to check if they were already set by user space, if so copy
1256 * them from kvm->arch.
1257 */
1258 return -ENXIO;
1259}
1260
1261static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1262 struct kvm_device_attr *attr)
1263{
1264 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1265 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1266 return -EFAULT;
1267 return 0;
1268}
658b6eda
MM		 1269static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1270{
1271 int ret = -ENXIO;
1272
1273 switch (attr->attr) {
1274 case KVM_S390_VM_CPU_PROCESSOR:
1275 ret = kvm_s390_get_processor(kvm, attr);
1276 break;
1277 case KVM_S390_VM_CPU_MACHINE:
1278 ret = kvm_s390_get_machine(kvm, attr);
1279 break;
15c9705f
DH		 1280 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1281 ret = kvm_s390_get_processor_feat(kvm, attr);
1282 break;
1283 case KVM_S390_VM_CPU_MACHINE_FEAT:
1284 ret = kvm_s390_get_machine_feat(kvm, attr);
1285 break;
0a763c78
DH		 1286 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1287 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1288 break;
1289 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1290 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1291 break;
658b6eda
MM		 1292 }
1293 return ret;
1294}
1295
f2061656
DD		 1296static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1297{
1298 int ret;
1299
1300 switch (attr->group) {
4f718eab 1301 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1302 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1303 break;
72f25020
JH		 1304 case KVM_S390_VM_TOD:
1305 ret = kvm_s390_set_tod(kvm, attr);
1306 break;
658b6eda
MM		 1307 case KVM_S390_VM_CPU_MODEL:
1308 ret = kvm_s390_set_cpu_model(kvm, attr);
1309 break;
a374e892
TK		 1310 case KVM_S390_VM_CRYPTO:
1311 ret = kvm_s390_vm_set_crypto(kvm, attr);
1312 break;
190df4a2
CI		 1313 case KVM_S390_VM_MIGRATION:
1314 ret = kvm_s390_vm_set_migration(kvm, attr);
1315 break;
f2061656
DD		 1316 default:
1317 ret = -ENXIO;
1318 break;
1319 }
1320
1321 return ret;
1322}
1323
1324static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1325{
8c0a7ce6
DD		 1326 int ret;
1327
1328 switch (attr->group) {
1329 case KVM_S390_VM_MEM_CTRL:
1330 ret = kvm_s390_get_mem_control(kvm, attr);
1331 break;
72f25020
JH		 1332 case KVM_S390_VM_TOD:
1333 ret = kvm_s390_get_tod(kvm, attr);
1334 break;
658b6eda
MM		 1335 case KVM_S390_VM_CPU_MODEL:
1336 ret = kvm_s390_get_cpu_model(kvm, attr);
1337 break;
190df4a2
CI		 1338 case KVM_S390_VM_MIGRATION:
1339 ret = kvm_s390_vm_get_migration(kvm, attr);
1340 break;
8c0a7ce6
DD		 1341 default:
1342 ret = -ENXIO;
1343 break;
1344 }
1345
1346 return ret;
f2061656
DD		 1347}
1348
1349static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1350{
1351 int ret;
1352
1353 switch (attr->group) {
4f718eab
DD		 1354 case KVM_S390_VM_MEM_CTRL:
1355 switch (attr->attr) {
1356 case KVM_S390_VM_MEM_ENABLE_CMMA:
1357 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1358 ret = sclp.has_cmma ? 0 : -ENXIO;
1359 break;
8c0a7ce6 1360 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1361 ret = 0;
1362 break;
1363 default:
1364 ret = -ENXIO;
1365 break;
1366 }
1367 break;
72f25020
JH		 1368 case KVM_S390_VM_TOD:
1369 switch (attr->attr) {
1370 case KVM_S390_VM_TOD_LOW:
1371 case KVM_S390_VM_TOD_HIGH:
1372 ret = 0;
1373 break;
1374 default:
1375 ret = -ENXIO;
1376 break;
1377 }
1378 break;
658b6eda
MM		 1379 case KVM_S390_VM_CPU_MODEL:
1380 switch (attr->attr) {
1381 case KVM_S390_VM_CPU_PROCESSOR:
1382 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1383 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1384 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1385 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
658b6eda
MM		 1386 ret = 0;
1387 break;
0a763c78
DH		 1388 /* configuring subfunctions is not supported yet */
1389 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1390 default:
1391 ret = -ENXIO;
1392 break;
1393 }
1394 break;
a374e892
TK		 1395 case KVM_S390_VM_CRYPTO:
1396 switch (attr->attr) {
1397 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1398 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1399 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1400 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1401 ret = 0;
1402 break;
1403 default:
1404 ret = -ENXIO;
1405 break;
1406 }
1407 break;
190df4a2
CI		 1408 case KVM_S390_VM_MIGRATION:
1409 ret = 0;
1410 break;
f2061656
DD		 1411 default:
1412 ret = -ENXIO;
1413 break;
1414 }
1415
1416 return ret;
1417}
1418
30ee2a98
JH		 1419static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1420{
1421 uint8_t *keys;
1422 uint64_t hva;
4f899147 1423 int srcu_idx, i, r = 0;
30ee2a98
JH		 1424
1425 if (args->flags != 0)
1426 return -EINVAL;
1427
1428 /* Is this guest using storage keys? */
1429 if (!mm_use_skey(current->mm))
1430 return KVM_S390_GET_SKEYS_NONE;
1431
1432 /* Enforce sane limit on memory allocation */
1433 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1434 return -EINVAL;
1435
752ade68 1436 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1437 if (!keys)
1438 return -ENOMEM;
1439
d3ed1cee 1440 down_read(&current->mm->mmap_sem);
4f899147 1441 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1442 for (i = 0; i < args->count; i++) {
1443 hva = gfn_to_hva(kvm, args->start_gfn + i);
1444 if (kvm_is_error_hva(hva)) {
1445 r = -EFAULT;
d3ed1cee 1446 break;
30ee2a98
JH		 1447 }
1448
154c8c19
DH		 1449 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1450 if (r)
d3ed1cee 1451 break;
30ee2a98 1452 }
4f899147 1453 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1454 up_read(&current->mm->mmap_sem);
1455
1456 if (!r) {
1457 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1458 sizeof(uint8_t) * args->count);
1459 if (r)
1460 r = -EFAULT;
30ee2a98
JH		 1461 }
1462
30ee2a98
JH		 1463 kvfree(keys);
1464 return r;
1465}
1466
1467static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1468{
1469 uint8_t *keys;
1470 uint64_t hva;
4f899147 1471 int srcu_idx, i, r = 0;
30ee2a98
JH		 1472
1473 if (args->flags != 0)
1474 return -EINVAL;
1475
1476 /* Enforce sane limit on memory allocation */
1477 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1478 return -EINVAL;
1479
752ade68 1480 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1481 if (!keys)
1482 return -ENOMEM;
1483
1484 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1485 sizeof(uint8_t) * args->count);
1486 if (r) {
1487 r = -EFAULT;
1488 goto out;
1489 }
1490
1491 /* Enable storage key handling for the guest */
14d4a425
DD		 1492 r = s390_enable_skey();
1493 if (r)
1494 goto out;
30ee2a98 1495
d3ed1cee 1496 down_read(&current->mm->mmap_sem);
4f899147 1497 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1498 for (i = 0; i < args->count; i++) {
1499 hva = gfn_to_hva(kvm, args->start_gfn + i);
1500 if (kvm_is_error_hva(hva)) {
1501 r = -EFAULT;
d3ed1cee 1502 break;
30ee2a98
JH		 1503 }
1504
1505 /* Lowest order bit is reserved */
1506 if (keys[i] & 0x01) {
1507 r = -EINVAL;
d3ed1cee 1508 break;
30ee2a98
JH		 1509 }
1510
fe69eabf 1511 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
30ee2a98 1512 if (r)
d3ed1cee 1513 break;
30ee2a98 1514 }
4f899147 1515 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1516 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1517out:
1518 kvfree(keys);
1519 return r;
1520}
1521
4036e387
CI		 1522/*
1523 * Base address and length must be sent at the start of each block, therefore
1524 * it's cheaper to send some clean data, as long as it's less than the size of
1525 * two longs.
1526 */
1527#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1528/* for consistency */
1529#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1530
1531/*
1532 * This function searches for the next page with dirty CMMA attributes, and
1533 * saves the attributes in the buffer up to either the end of the buffer or
1534 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1535 * no trailing clean bytes are saved.
1536 * In case no dirty bits were found, or if CMMA was not enabled or used, the
1537 * output buffer will indicate 0 as length.
1538 */
1539static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1540 struct kvm_s390_cmma_log *args)
1541{
1542 struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1543 unsigned long bufsize, hva, pgstev, i, next, cur;
1544 int srcu_idx, peek, r = 0, rr;
1545 u8 *res;
1546
1547 cur = args->start_gfn;
1548 i = next = pgstev = 0;
1549
1550 if (unlikely(!kvm->arch.use_cmma))
1551 return -ENXIO;
1552 /* Invalid/unsupported flags were specified */
1553 if (args->flags & ~KVM_S390_CMMA_PEEK)
1554 return -EINVAL;
1555 /* Migration mode query, and we are not doing a migration */
1556 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1557 if (!peek && !s)
1558 return -EINVAL;
1559 /* CMMA is disabled or was not used, or the buffer has length zero */
1560 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1561 if (!bufsize || !kvm->mm->context.use_cmma) {
1562 memset(args, 0, sizeof(*args));
1563 return 0;
1564 }
1565
1566 if (!peek) {
1567 /* We are not peeking, and there are no dirty pages */
1568 if (!atomic64_read(&s->dirty_pages)) {
1569 memset(args, 0, sizeof(*args));
1570 return 0;
1571 }
1572 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1573 args->start_gfn);
1574 if (cur >= s->bitmap_size) /* nothing found, loop back */
1575 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1576 if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1577 memset(args, 0, sizeof(*args));
1578 return 0;
1579 }
1580 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1581 }
1582
1583 res = vmalloc(bufsize);
1584 if (!res)
1585 return -ENOMEM;
1586
1587 args->start_gfn = cur;
1588
1589 down_read(&kvm->mm->mmap_sem);
1590 srcu_idx = srcu_read_lock(&kvm->srcu);
1591 while (i < bufsize) {
1592 hva = gfn_to_hva(kvm, cur);
1593 if (kvm_is_error_hva(hva)) {
1594 r = -EFAULT;
1595 break;
1596 }
1597 /* decrement only if we actually flipped the bit to 0 */
1598 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1599 atomic64_dec(&s->dirty_pages);
1600 r = get_pgste(kvm->mm, hva, &pgstev);
1601 if (r < 0)
1602 pgstev = 0;
1603 /* save the value */
1bab1c02 1604 res[i++] = (pgstev >> 24) & 0x43;
4036e387
CI		 1605 /*
1606 * if the next bit is too far away, stop.
1607 * if we reached the previous "next", find the next one
1608 */
1609 if (!peek) {
1610 if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1611 break;
1612 if (cur == next)
1613 next = find_next_bit(s->pgste_bitmap,
1614 s->bitmap_size, cur + 1);
1615 /* reached the end of the bitmap or of the buffer, stop */
1616 if ((next >= s->bitmap_size) ||
1617 (next >= args->start_gfn + bufsize))
1618 break;
1619 }
1620 cur++;
1621 }
1622 srcu_read_unlock(&kvm->srcu, srcu_idx);
1623 up_read(&kvm->mm->mmap_sem);
1624 args->count = i;
1625 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1626
1627 rr = copy_to_user((void __user *)args->values, res, args->count);
1628 if (rr)
1629 r = -EFAULT;
1630
1631 vfree(res);
1632 return r;
1633}
1634
1635/*
1636 * This function sets the CMMA attributes for the given pages. If the input
1637 * buffer has zero length, no action is taken, otherwise the attributes are
1638 * set and the mm->context.use_cmma flag is set.
1639 */
1640static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1641 const struct kvm_s390_cmma_log *args)
1642{
1643 unsigned long hva, mask, pgstev, i;
1644 uint8_t *bits;
1645 int srcu_idx, r = 0;
1646
1647 mask = args->mask;
1648
1649 if (!kvm->arch.use_cmma)
1650 return -ENXIO;
1651 /* invalid/unsupported flags */
1652 if (args->flags != 0)
1653 return -EINVAL;
1654 /* Enforce sane limit on memory allocation */
1655 if (args->count > KVM_S390_CMMA_SIZE_MAX)
1656 return -EINVAL;
1657 /* Nothing to do */
1658 if (args->count == 0)
1659 return 0;
1660
1661 bits = vmalloc(sizeof(*bits) * args->count);
1662 if (!bits)
1663 return -ENOMEM;
1664
1665 r = copy_from_user(bits, (void __user *)args->values, args->count);
1666 if (r) {
1667 r = -EFAULT;
1668 goto out;
1669 }
1670
1671 down_read(&kvm->mm->mmap_sem);
1672 srcu_idx = srcu_read_lock(&kvm->srcu);
1673 for (i = 0; i < args->count; i++) {
1674 hva = gfn_to_hva(kvm, args->start_gfn + i);
1675 if (kvm_is_error_hva(hva)) {
1676 r = -EFAULT;
1677 break;
1678 }
1679
1680 pgstev = bits[i];
1681 pgstev = pgstev << 24;
1bab1c02 1682 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 1683 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1684 }
1685 srcu_read_unlock(&kvm->srcu, srcu_idx);
1686 up_read(&kvm->mm->mmap_sem);
1687
1688 if (!kvm->mm->context.use_cmma) {
1689 down_write(&kvm->mm->mmap_sem);
1690 kvm->mm->context.use_cmma = 1;
1691 up_write(&kvm->mm->mmap_sem);
1692 }
1693out:
1694 vfree(bits);
1695 return r;
1696}
1697
b0c632db
HC		 1698long kvm_arch_vm_ioctl(struct file *filp,
1699 unsigned int ioctl, unsigned long arg)
1700{
1701 struct kvm *kvm = filp->private_data;
1702 void __user *argp = (void __user *)arg;
f2061656 1703 struct kvm_device_attr attr;
b0c632db
HC		 1704 int r;
1705
1706 switch (ioctl) {
ba5c1e9b
CO		 1707 case KVM_S390_INTERRUPT: {
1708 struct kvm_s390_interrupt s390int;
1709
1710 r = -EFAULT;
1711 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1712 break;
1713 r = kvm_s390_inject_vm(kvm, &s390int);
1714 break;
1715 }
d938dc55
CH		 1716 case KVM_ENABLE_CAP: {
1717 struct kvm_enable_cap cap;
1718 r = -EFAULT;
1719 if (copy_from_user(&cap, argp, sizeof(cap)))
1720 break;
1721 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1722 break;
1723 }
84223598
CH		 1724 case KVM_CREATE_IRQCHIP: {
1725 struct kvm_irq_routing_entry routing;
1726
1727 r = -EINVAL;
1728 if (kvm->arch.use_irqchip) {
1729 /* Set up dummy routing. */
1730 memset(&routing, 0, sizeof(routing));
152b2839 1731 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 1732 }
1733 break;
1734 }
f2061656
DD		 1735 case KVM_SET_DEVICE_ATTR: {
1736 r = -EFAULT;
1737 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1738 break;
1739 r = kvm_s390_vm_set_attr(kvm, &attr);
1740 break;
1741 }
1742 case KVM_GET_DEVICE_ATTR: {
1743 r = -EFAULT;
1744 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1745 break;
1746 r = kvm_s390_vm_get_attr(kvm, &attr);
1747 break;
1748 }
1749 case KVM_HAS_DEVICE_ATTR: {
1750 r = -EFAULT;
1751 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1752 break;
1753 r = kvm_s390_vm_has_attr(kvm, &attr);
1754 break;
1755 }
30ee2a98
JH		 1756 case KVM_S390_GET_SKEYS: {
1757 struct kvm_s390_skeys args;
1758
1759 r = -EFAULT;
1760 if (copy_from_user(&args, argp,
1761 sizeof(struct kvm_s390_skeys)))
1762 break;
1763 r = kvm_s390_get_skeys(kvm, &args);
1764 break;
1765 }
1766 case KVM_S390_SET_SKEYS: {
1767 struct kvm_s390_skeys args;
1768
1769 r = -EFAULT;
1770 if (copy_from_user(&args, argp,
1771 sizeof(struct kvm_s390_skeys)))
1772 break;
1773 r = kvm_s390_set_skeys(kvm, &args);
1774 break;
1775 }
4036e387
CI		 1776 case KVM_S390_GET_CMMA_BITS: {
1777 struct kvm_s390_cmma_log args;
1778
1779 r = -EFAULT;
1780 if (copy_from_user(&args, argp, sizeof(args)))
1781 break;
1de1ea7e 1782 mutex_lock(&kvm->slots_lock);
4036e387 1783 r = kvm_s390_get_cmma_bits(kvm, &args);
1de1ea7e 1784 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 1785 if (!r) {
1786 r = copy_to_user(argp, &args, sizeof(args));
1787 if (r)
1788 r = -EFAULT;
1789 }
1790 break;
1791 }
1792 case KVM_S390_SET_CMMA_BITS: {
1793 struct kvm_s390_cmma_log args;
1794
1795 r = -EFAULT;
1796 if (copy_from_user(&args, argp, sizeof(args)))
1797 break;
1de1ea7e 1798 mutex_lock(&kvm->slots_lock);
4036e387 1799 r = kvm_s390_set_cmma_bits(kvm, &args);
1de1ea7e 1800 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 1801 break;
1802 }
b0c632db 1803 default:
367e1319 1804 r = -ENOTTY;
b0c632db
HC		 1805 }
1806
1807 return r;
1808}
1809
45c9b47c
TK		 1810static int kvm_s390_query_ap_config(u8 *config)
1811{
1812 u32 fcn_code = 0x04000000UL;
86044c8c 1813 u32 cc = 0;
45c9b47c 1814
86044c8c 1815 memset(config, 0, 128);
45c9b47c
TK		 1816 asm volatile(
1817 "lgr 0,%1\n"
1818 "lgr 2,%2\n"
1819 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 1820 "0: ipm %0\n"
45c9b47c 1821 "srl %0,28\n"
86044c8c
CB		 1822 "1:\n"
1823 EX_TABLE(0b, 1b)
1824 : "+r" (cc)
45c9b47c
TK		 1825 : "r" (fcn_code), "r" (config)
1826 : "cc", "0", "2", "memory"
1827 );
1828
1829 return cc;
1830}
1831
1832static int kvm_s390_apxa_installed(void)
1833{
1834 u8 config[128];
1835 int cc;
1836
a6aacc3f 1837 if (test_facility(12)) {
45c9b47c
TK		 1838 cc = kvm_s390_query_ap_config(config);
1839
1840 if (cc)
1841 pr_err("PQAP(QCI) failed with cc=%d", cc);
1842 else
1843 return config[0] & 0x40;
1844 }
1845
1846 return 0;
1847}
1848
1849static void kvm_s390_set_crycb_format(struct kvm *kvm)
1850{
1851 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1852
1853 if (kvm_s390_apxa_installed())
1854 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1855 else
1856 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1857}
1858
9bb0ec09 1859static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 1860{
9bb0ec09
DH		 1861 struct cpuid cpuid;
1862
1863 get_cpu_id(&cpuid);
1864 cpuid.version = 0xff;
1865 return *((u64 *) &cpuid);
9d8d5786
MM		 1866}
1867
c54f0d6a 1868static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 1869{
9d8d5786 1870 if (!test_kvm_facility(kvm, 76))
c54f0d6a 1871 return;
5102ee87 1872
c54f0d6a 1873 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 1874 kvm_s390_set_crycb_format(kvm);
5102ee87 1875
ed6f76b4
TK		 1876 /* Enable AES/DEA protected key functions by default */
1877 kvm->arch.crypto.aes_kw = 1;
1878 kvm->arch.crypto.dea_kw = 1;
1879 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1880 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1881 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1882 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 1883}
1884
7d43bafc
ED		 1885static void sca_dispose(struct kvm *kvm)
1886{
1887 if (kvm->arch.use_esca)
5e044315 1888 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 1889 else
1890 free_page((unsigned long)(kvm->arch.sca));
1891 kvm->arch.sca = NULL;
1892}
1893
e08b9637 1894int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1895{
76a6dd72 1896 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 1897 int i, rc;
b0c632db 1898 char debug_name[16];
f6c137ff 1899 static unsigned long sca_offset;
b0c632db 1900
e08b9637
CO		 1901 rc = -EINVAL;
1902#ifdef CONFIG_KVM_S390_UCONTROL
1903 if (type & ~KVM_VM_S390_UCONTROL)
1904 goto out_err;
1905 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1906 goto out_err;
1907#else
1908 if (type)
1909 goto out_err;
1910#endif
1911
b0c632db
HC		 1912 rc = s390_enable_sie();
1913 if (rc)
d89f5eff 1914 goto out_err;
b0c632db 1915
b290411a
CO		 1916 rc = -ENOMEM;
1917
7d43bafc 1918 kvm->arch.use_esca = 0; /* start with basic SCA */
76a6dd72
DH		 1919 if (!sclp.has_64bscao)
1920 alloc_flags |= GFP_DMA;
5e044315 1921 rwlock_init(&kvm->arch.sca_lock);
76a6dd72 1922 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 1923 if (!kvm->arch.sca)
d89f5eff 1924 goto out_err;
f6c137ff 1925 spin_lock(&kvm_lock);
c5c2c393 1926 sca_offset += 16;
bc784cce 1927 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 1928 sca_offset = 0;
bc784cce
ED		 1929 kvm->arch.sca = (struct bsca_block *)
1930 ((char *) kvm->arch.sca + sca_offset);
f6c137ff 1931 spin_unlock(&kvm_lock);
b0c632db
HC		 1932
1933 sprintf(debug_name, "kvm-%u", current->pid);
1934
1cb9cf72 1935 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 1936 if (!kvm->arch.dbf)
40f5b735 1937 goto out_err;
b0c632db 1938
19114beb 1939 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
c54f0d6a
DH		 1940 kvm->arch.sie_page2 =
1941 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1942 if (!kvm->arch.sie_page2)
40f5b735 1943 goto out_err;
9d8d5786 1944
fb5bf93f 1945 /* Populate the facility mask initially. */
c54f0d6a 1946 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
04478197 1947 sizeof(S390_lowcore.stfle_fac_list));
9d8d5786
MM		 1948 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1949 if (i < kvm_s390_fac_list_mask_size())
c54f0d6a 1950 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1951 else
c54f0d6a 1952 kvm->arch.model.fac_mask[i] = 0UL;
9d8d5786
MM		 1953 }
1954
981467c9 1955 /* Populate the facility list initially. */
c54f0d6a
DH		 1956 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1957 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
981467c9
MM		 1958 S390_ARCH_FAC_LIST_SIZE_BYTE);
1959
1935222d
DH		 1960 /* we are always in czam mode - even on pre z14 machines */
1961 set_kvm_facility(kvm->arch.model.fac_mask, 138);
1962 set_kvm_facility(kvm->arch.model.fac_list, 138);
1963 /* we emulate STHYI in kvm */
95ca2cb5
JF		 1964 set_kvm_facility(kvm->arch.model.fac_mask, 74);
1965 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 1966 if (MACHINE_HAS_TLB_GUEST) {
1967 set_kvm_facility(kvm->arch.model.fac_mask, 147);
1968 set_kvm_facility(kvm->arch.model.fac_list, 147);
1969 }
95ca2cb5 1970
9bb0ec09 1971 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 1972 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 1973
c54f0d6a 1974 kvm_s390_crypto_init(kvm);
5102ee87 1975
51978393
FL		 1976 mutex_init(&kvm->arch.float_int.ais_lock);
1977 kvm->arch.float_int.simm = 0;
1978 kvm->arch.float_int.nimm = 0;
ba5c1e9b 1979 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1980 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1981 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1982 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1983 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1984
b0c632db 1985 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 1986 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 1987
e08b9637
CO		 1988 if (type & KVM_VM_S390_UCONTROL) {
1989 kvm->arch.gmap = NULL;
a3a92c31 1990 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 1991 } else {
32e6b236 1992 if (sclp.hamax == U64_MAX)
ee71d16d 1993 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 1994 else
ee71d16d 1995 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 1996 sclp.hamax + 1);
6ea427bb 1997 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 1998 if (!kvm->arch.gmap)
40f5b735 1999 goto out_err;
2c70fe44 2000 kvm->arch.gmap->private = kvm;
24eb3a82 2001 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 2002 }
fa6b7fe9
CH		 2003
2004 kvm->arch.css_support = 0;
84223598 2005 kvm->arch.use_irqchip = 0;
72f25020 2006 kvm->arch.epoch = 0;
fa6b7fe9 2007
8ad35755 2008 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 2009 kvm_s390_vsie_init(kvm);
d7c5cb01 2010 kvm_s390_gisa_init(kvm);
8335713a 2011 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 2012
d89f5eff 2013 return 0;
40f5b735 2014out_err:
c54f0d6a 2015 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 2016 debug_unregister(kvm->arch.dbf);
7d43bafc 2017 sca_dispose(kvm);
78f26131 2018 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 2019 return rc;
b0c632db
HC		 2020}
2021
235539b4
LC		 2022bool kvm_arch_has_vcpu_debugfs(void)
2023{
2024 return false;
2025}
2026
2027int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2028{
2029 return 0;
2030}
2031
d329c035
CB		 2032void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2033{
2034 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2035 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2036 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2037 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2038 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2039 sca_del_vcpu(vcpu);
27e0393f
CO		 2040
2041 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2042 gmap_remove(vcpu->arch.gmap);
27e0393f 2043
e6db1d61 2044 if (vcpu->kvm->arch.use_cmma)
b31605c1 2045 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 2046 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 2047
6692cef3 2048 kvm_vcpu_uninit(vcpu);
b110feaf 2049 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 2050}
2051
2052static void kvm_free_vcpus(struct kvm *kvm)
2053{
2054 unsigned int i;
988a2cae 2055 struct kvm_vcpu *vcpu;
d329c035 2056
988a2cae
GN		 2057 kvm_for_each_vcpu(i, vcpu, kvm)
2058 kvm_arch_vcpu_destroy(vcpu);
2059
2060 mutex_lock(&kvm->lock);
2061 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2062 kvm->vcpus[i] = NULL;
2063
2064 atomic_set(&kvm->online_vcpus, 0);
2065 mutex_unlock(&kvm->lock);
d329c035
CB		 2066}
2067
b0c632db
HC		 2068void kvm_arch_destroy_vm(struct kvm *kvm)
2069{
d329c035 2070 kvm_free_vcpus(kvm);
7d43bafc 2071 sca_dispose(kvm);
d329c035 2072 debug_unregister(kvm->arch.dbf);
d7c5cb01 2073 kvm_s390_gisa_destroy(kvm);
c54f0d6a 2074 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2075 if (!kvm_is_ucontrol(kvm))
6ea427bb 2076 gmap_remove(kvm->arch.gmap);
841b91c5 2077 kvm_s390_destroy_adapters(kvm);
67335e63 2078 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2079 kvm_s390_vsie_destroy(kvm);
190df4a2
CI		 2080 if (kvm->arch.migration_state) {
2081 vfree(kvm->arch.migration_state->pgste_bitmap);
2082 kfree(kvm->arch.migration_state);
2083 }
8335713a 2084 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2085}
2086
2087/* Section: vcpu related */
dafd032a
DD		 2088static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2089{
6ea427bb 2090 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2091 if (!vcpu->arch.gmap)
2092 return -ENOMEM;
2093 vcpu->arch.gmap->private = vcpu->kvm;
2094
2095 return 0;
2096}
2097
a6e2f683
ED		 2098static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2099{
a6940674
DH		 2100 if (!kvm_s390_use_sca_entries())
2101 return;
5e044315 2102 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2103 if (vcpu->kvm->arch.use_esca) {
2104 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2105
7d43bafc 2106 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2107 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2108 } else {
2109 struct bsca_block *sca = vcpu->kvm->arch.sca;
2110
2111 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2112 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2113 }
5e044315 2114 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2115}
2116
eaa78f34 2117static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2118{
a6940674
DH		 2119 if (!kvm_s390_use_sca_entries()) {
2120 struct bsca_block *sca = vcpu->kvm->arch.sca;
2121
2122 /* we still need the basic sca for the ipte control */
2123 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2124 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2125 }
eaa78f34
DH		 2126 read_lock(&vcpu->kvm->arch.sca_lock);
2127 if (vcpu->kvm->arch.use_esca) {
2128 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2129
eaa78f34 2130 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2131 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2132 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2133 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2134 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2135 } else {
eaa78f34 2136 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2137
eaa78f34 2138 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2139 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2140 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2141 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2142 }
eaa78f34 2143 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2144}
2145
2146/* Basic SCA to Extended SCA data copy routines */
2147static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2148{
2149 d->sda = s->sda;
2150 d->sigp_ctrl.c = s->sigp_ctrl.c;
2151 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2152}
2153
2154static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2155{
2156 int i;
2157
2158 d->ipte_control = s->ipte_control;
2159 d->mcn[0] = s->mcn;
2160 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2161 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2162}
2163
2164static int sca_switch_to_extended(struct kvm *kvm)
2165{
2166 struct bsca_block *old_sca = kvm->arch.sca;
2167 struct esca_block *new_sca;
2168 struct kvm_vcpu *vcpu;
2169 unsigned int vcpu_idx;
2170 u32 scaol, scaoh;
2171
2172 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2173 if (!new_sca)
2174 return -ENOMEM;
2175
2176 scaoh = (u32)((u64)(new_sca) >> 32);
2177 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2178
2179 kvm_s390_vcpu_block_all(kvm);
2180 write_lock(&kvm->arch.sca_lock);
2181
2182 sca_copy_b_to_e(new_sca, old_sca);
2183
2184 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2185 vcpu->arch.sie_block->scaoh = scaoh;
2186 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 2187 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 2188 }
2189 kvm->arch.sca = new_sca;
2190 kvm->arch.use_esca = 1;
2191
2192 write_unlock(&kvm->arch.sca_lock);
2193 kvm_s390_vcpu_unblock_all(kvm);
2194
2195 free_page((unsigned long)old_sca);
2196
8335713a
CB		 2197 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2198 old_sca, kvm->arch.sca);
5e044315 2199 return 0;
a6e2f683
ED		 2200}
2201
2202static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2203{
5e044315
ED		 2204 int rc;
2205
a6940674
DH		 2206 if (!kvm_s390_use_sca_entries()) {
2207 if (id < KVM_MAX_VCPUS)
2208 return true;
2209 return false;
2210 }
5e044315
ED		 2211 if (id < KVM_S390_BSCA_CPU_SLOTS)
2212 return true;
76a6dd72 2213 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 2214 return false;
2215
2216 mutex_lock(&kvm->lock);
2217 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2218 mutex_unlock(&kvm->lock);
2219
2220 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 2221}
2222
b0c632db
HC		 2223int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2224{
3c038e6b
DD		 2225 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2226 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 2227 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2228 KVM_SYNC_GPRS |
9eed0735 2229 KVM_SYNC_ACRS |
b028ee3e
DH		 2230 KVM_SYNC_CRS |
2231 KVM_SYNC_ARCH0 |
2232 KVM_SYNC_PFAULT;
75a4615c 2233 kvm_s390_set_prefix(vcpu, 0);
c6e5f166
FZ		 2234 if (test_kvm_facility(vcpu->kvm, 64))
2235 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
35b3fde6
CB		 2236 if (test_kvm_facility(vcpu->kvm, 82))
2237 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
4e0b1ab7
FZ		 2238 if (test_kvm_facility(vcpu->kvm, 133))
2239 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
f6aa6dc4
DH		 2240 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2241 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2242 */
2243 if (MACHINE_HAS_VX)
68c55750 2244 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
6fd8e67d
DH		 2245 else
2246 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
dafd032a
DD		 2247
2248 if (kvm_is_ucontrol(vcpu->kvm))
2249 return __kvm_ucontrol_vcpu_init(vcpu);
2250
b0c632db
HC		 2251 return 0;
2252}
2253
db0758b2
DH		 2254/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2255static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2256{
2257 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 2258 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 2259 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 2260 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2261}
2262
2263/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2264static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2265{
2266 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 2267 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2268 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2269 vcpu->arch.cputm_start = 0;
9c23a131 2270 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2271}
2272
2273/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2274static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2275{
2276 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2277 vcpu->arch.cputm_enabled = true;
2278 __start_cpu_timer_accounting(vcpu);
2279}
2280
2281/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2282static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2283{
2284 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2285 __stop_cpu_timer_accounting(vcpu);
2286 vcpu->arch.cputm_enabled = false;
2287}
2288
2289static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2290{
2291 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2292 __enable_cpu_timer_accounting(vcpu);
2293 preempt_enable();
2294}
2295
2296static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2297{
2298 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2299 __disable_cpu_timer_accounting(vcpu);
2300 preempt_enable();
2301}
2302
4287f247
DH		 2303/* set the cpu timer - may only be called from the VCPU thread itself */
2304void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2305{
db0758b2 2306 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 2307 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2308 if (vcpu->arch.cputm_enabled)
2309 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 2310 vcpu->arch.sie_block->cputm = cputm;
9c23a131 2311 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 2312 preempt_enable();
4287f247
DH		 2313}
2314
db0758b2 2315/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 2316__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2317{
9c23a131 2318 unsigned int seq;
db0758b2 2319 __u64 value;
db0758b2
DH		 2320
2321 if (unlikely(!vcpu->arch.cputm_enabled))
2322 return vcpu->arch.sie_block->cputm;
2323
9c23a131
DH		 2324 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2325 do {
2326 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2327 /*
2328 * If the writer would ever execute a read in the critical
2329 * section, e.g. in irq context, we have a deadlock.
2330 */
2331 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2332 value = vcpu->arch.sie_block->cputm;
2333 /* if cputm_start is 0, accounting is being started/stopped */
2334 if (likely(vcpu->arch.cputm_start))
2335 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2336 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2337 preempt_enable();
db0758b2 2338 return value;
4287f247
DH		 2339}
2340
b0c632db
HC		 2341void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2342{
9977e886 2343
37d9df98 2344 gmap_enable(vcpu->arch.enabled_gmap);
ef8f4f49 2345 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
5ebda316 2346 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2347 __start_cpu_timer_accounting(vcpu);
01a745ac 2348 vcpu->cpu = cpu;
b0c632db
HC		 2349}
2350
2351void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2352{
01a745ac 2353 vcpu->cpu = -1;
5ebda316 2354 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2355 __stop_cpu_timer_accounting(vcpu);
9daecfc6 2356 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
37d9df98
DH		 2357 vcpu->arch.enabled_gmap = gmap_get_enabled();
2358 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 2359
b0c632db
HC		 2360}
2361
2362static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2363{
2364 /* this equals initial cpu reset in pop, but we don't switch to ESA */
2365 vcpu->arch.sie_block->gpsw.mask = 0UL;
2366 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 2367 kvm_s390_set_prefix(vcpu, 0);
4287f247 2368 kvm_s390_set_cpu_timer(vcpu, 0);
b0c632db
HC		 2369 vcpu->arch.sie_block->ckc = 0UL;
2370 vcpu->arch.sie_block->todpr = 0;
2371 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2372 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
2373 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
9abc2a08
DH		 2374 /* make sure the new fpc will be lazily loaded */
2375 save_fpu_regs();
2376 current->thread.fpu.fpc = 0;
b0c632db 2377 vcpu->arch.sie_block->gbea = 1;
672550fb 2378 vcpu->arch.sie_block->pp = 0;
35b3fde6 2379 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3c038e6b
DD		 2380 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2381 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 2382 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2383 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 2384 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 2385}
2386
31928aa5 2387void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 2388{
72f25020 2389 mutex_lock(&vcpu->kvm->lock);
fdf03650 2390 preempt_disable();
72f25020 2391 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
fdf03650 2392 preempt_enable();
72f25020 2393 mutex_unlock(&vcpu->kvm->lock);
25508824 2394 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 2395 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 2396 sca_add_vcpu(vcpu);
25508824 2397 }
6502a34c
DH		 2398 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2399 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 2400 /* make vcpu_load load the right gmap on the first trigger */
2401 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 2402}
2403
5102ee87
TK		 2404static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2405{
9d8d5786 2406 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 2407 return;
2408
a374e892
TK		 2409 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2410
2411 if (vcpu->kvm->arch.crypto.aes_kw)
2412 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2413 if (vcpu->kvm->arch.crypto.dea_kw)
2414 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2415
5102ee87
TK		 2416 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2417}
2418
b31605c1
DD		 2419void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2420{
2421 free_page(vcpu->arch.sie_block->cbrlo);
2422 vcpu->arch.sie_block->cbrlo = 0;
2423}
2424
2425int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2426{
2427 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2428 if (!vcpu->arch.sie_block->cbrlo)
2429 return -ENOMEM;
2430
0c9d8683 2431 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI;
b31605c1
DD		 2432 return 0;
2433}
2434
91520f1a
MM		 2435static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2436{
2437 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2438
91520f1a 2439 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 2440 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 2441 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 2442}
2443
b0c632db
HC		 2444int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2445{
b31605c1 2446 int rc = 0;
b31288fa 2447
9e6dabef
CH		 2448 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2449 CPUSTAT_SM |
a4a4f191
GH		 2450 CPUSTAT_STOPPED);
2451
53df84f8 2452 if (test_kvm_facility(vcpu->kvm, 78))
ef8f4f49 2453 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
53df84f8 2454 else if (test_kvm_facility(vcpu->kvm, 8))
ef8f4f49 2455 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
a4a4f191 2456
91520f1a
MM		 2457 kvm_s390_vcpu_setup_model(vcpu);
2458
bdab09f3
DH		 2459 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2460 if (MACHINE_HAS_ESOP)
0c9d8683 2461 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 2462 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 2463 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 2464 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 2465 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 2466
873b425e 2467 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
0c9d8683 2468 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 2469 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 2470 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2471 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 2472 if (sclp.has_cei)
0c9d8683 2473 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 2474 if (sclp.has_ib)
0c9d8683 2475 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 2476 if (sclp.has_siif)
0c9d8683 2477 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 2478 if (sclp.has_sigpif)
0c9d8683 2479 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 2480 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 2481 vcpu->arch.sie_block->eca |= ECA_VX;
2482 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 2483 }
8fa1696e
CW		 2484 if (test_kvm_facility(vcpu->kvm, 139))
2485 vcpu->arch.sie_block->ecd |= ECD_MEF;
2486
d7c5cb01
MM		 2487 if (vcpu->arch.sie_block->gd) {
2488 vcpu->arch.sie_block->eca |= ECA_AIV;
2489 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2490 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2491 }
4e0b1ab7
FZ		 2492 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2493 | SDNXC;
c6e5f166 2494 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 2495
2496 if (sclp.has_kss)
ef8f4f49 2497 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
730cd632
FA		 2498 else
2499 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 2500
e6db1d61 2501 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 2502 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2503 if (rc)
2504 return rc;
b31288fa 2505 }
0ac96caf 2506 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 2507 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2508
5102ee87
TK		 2509 kvm_s390_vcpu_crypto_setup(vcpu);
2510
b31605c1 2511 return rc;
b0c632db
HC		 2512}
2513
2514struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2515 unsigned int id)
2516{
4d47555a 2517 struct kvm_vcpu *vcpu;
7feb6bb8 2518 struct sie_page *sie_page;
4d47555a
CO		 2519 int rc = -EINVAL;
2520
4215825e 2521 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
4d47555a
CO		 2522 goto out;
2523
2524 rc = -ENOMEM;
b0c632db 2525
b110feaf 2526 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 2527 if (!vcpu)
4d47555a 2528 goto out;
b0c632db 2529
da72ca4d 2530 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 2531 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2532 if (!sie_page)
b0c632db
HC		 2533 goto out_free_cpu;
2534
7feb6bb8
MM		 2535 vcpu->arch.sie_block = &sie_page->sie_block;
2536 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2537
efed1104
DH		 2538 /* the real guest size will always be smaller than msl */
2539 vcpu->arch.sie_block->mso = 0;
2540 vcpu->arch.sie_block->msl = sclp.hamax;
2541
b0c632db 2542 vcpu->arch.sie_block->icpua = id;
ba5c1e9b 2543 spin_lock_init(&vcpu->arch.local_int.lock);
d7c5cb01 2544 vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
4b9f9525
MM		 2545 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2546 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
9c23a131 2547 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 2548
b0c632db
HC		 2549 rc = kvm_vcpu_init(vcpu, kvm, id);
2550 if (rc)
9abc2a08 2551 goto out_free_sie_block;
8335713a 2552 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
b0c632db 2553 vcpu->arch.sie_block);
ade38c31 2554 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 2555
b0c632db 2556 return vcpu;
7b06bf2f
WY		 2557out_free_sie_block:
2558 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 2559out_free_cpu:
b110feaf 2560 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 2561out:
b0c632db
HC		 2562 return ERR_PTR(rc);
2563}
2564
b0c632db
HC		 2565int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2566{
9a022067 2567 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 2568}
2569
199b5763
LM		 2570bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2571{
0546c63d 2572 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 2573}
2574
27406cd5 2575void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 2576{
805de8f4 2577 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 2578 exit_sie(vcpu);
49b99e1e
CB		 2579}
2580
27406cd5 2581void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 2582{
805de8f4 2583 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 2584}
2585
8e236546
CB		 2586static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2587{
805de8f4 2588 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 2589 exit_sie(vcpu);
8e236546
CB		 2590}
2591
2592static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2593{
9bf9fde2 2594 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 2595}
2596
49b99e1e
CB		 2597/*
2598 * Kick a guest cpu out of SIE and wait until SIE is not running.
2599 * If the CPU is not running (e.g. waiting as idle) the function will
2600 * return immediately. */
2601void exit_sie(struct kvm_vcpu *vcpu)
2602{
ef8f4f49 2603 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
49b99e1e
CB		 2604 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2605 cpu_relax();
2606}
2607
8e236546
CB		 2608/* Kick a guest cpu out of SIE to process a request synchronously */
2609void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 2610{
8e236546
CB		 2611 kvm_make_request(req, vcpu);
2612 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 2613}
2614
414d3b07
MS		 2615static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2616 unsigned long end)
2c70fe44 2617{
2c70fe44
CB		 2618 struct kvm *kvm = gmap->private;
2619 struct kvm_vcpu *vcpu;
414d3b07
MS		 2620 unsigned long prefix;
2621 int i;
2c70fe44 2622
65d0b0d4
DH		 2623 if (gmap_is_shadow(gmap))
2624 return;
414d3b07
MS		 2625 if (start >= 1UL << 31)
2626 /* We are only interested in prefix pages */
2627 return;
2c70fe44
CB		 2628 kvm_for_each_vcpu(i, vcpu, kvm) {
2629 /* match against both prefix pages */
414d3b07
MS		 2630 prefix = kvm_s390_get_prefix(vcpu);
2631 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2632 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2633 start, end);
8e236546 2634 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 2635 }
2636 }
2637}
2638
b6d33834
CD		 2639int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2640{
2641 /* kvm common code refers to this, but never calls it */
2642 BUG();
2643 return 0;
2644}
2645
14eebd91
CO		 2646static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2647 struct kvm_one_reg *reg)
2648{
2649 int r = -EINVAL;
2650
2651 switch (reg->id) {
29b7c71b
CO		 2652 case KVM_REG_S390_TODPR:
2653 r = put_user(vcpu->arch.sie_block->todpr,
2654 (u32 __user *)reg->addr);
2655 break;
2656 case KVM_REG_S390_EPOCHDIFF:
2657 r = put_user(vcpu->arch.sie_block->epoch,
2658 (u64 __user *)reg->addr);
2659 break;
46a6dd1c 2660 case KVM_REG_S390_CPU_TIMER:
4287f247 2661 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 2662 (u64 __user *)reg->addr);
2663 break;
2664 case KVM_REG_S390_CLOCK_COMP:
2665 r = put_user(vcpu->arch.sie_block->ckc,
2666 (u64 __user *)reg->addr);
2667 break;
536336c2
DD		 2668 case KVM_REG_S390_PFTOKEN:
2669 r = put_user(vcpu->arch.pfault_token,
2670 (u64 __user *)reg->addr);
2671 break;
2672 case KVM_REG_S390_PFCOMPARE:
2673 r = put_user(vcpu->arch.pfault_compare,
2674 (u64 __user *)reg->addr);
2675 break;
2676 case KVM_REG_S390_PFSELECT:
2677 r = put_user(vcpu->arch.pfault_select,
2678 (u64 __user *)reg->addr);
2679 break;
672550fb
CB		 2680 case KVM_REG_S390_PP:
2681 r = put_user(vcpu->arch.sie_block->pp,
2682 (u64 __user *)reg->addr);
2683 break;
afa45ff5
CB		 2684 case KVM_REG_S390_GBEA:
2685 r = put_user(vcpu->arch.sie_block->gbea,
2686 (u64 __user *)reg->addr);
2687 break;
14eebd91
CO		 2688 default:
2689 break;
2690 }
2691
2692 return r;
2693}
2694
2695static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2696 struct kvm_one_reg *reg)
2697{
2698 int r = -EINVAL;
4287f247 2699 __u64 val;
14eebd91
CO		 2700
2701 switch (reg->id) {
29b7c71b
CO		 2702 case KVM_REG_S390_TODPR:
2703 r = get_user(vcpu->arch.sie_block->todpr,
2704 (u32 __user *)reg->addr);
2705 break;
2706 case KVM_REG_S390_EPOCHDIFF:
2707 r = get_user(vcpu->arch.sie_block->epoch,
2708 (u64 __user *)reg->addr);
2709 break;
46a6dd1c 2710 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 2711 r = get_user(val, (u64 __user *)reg->addr);
2712 if (!r)
2713 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 2714 break;
2715 case KVM_REG_S390_CLOCK_COMP:
2716 r = get_user(vcpu->arch.sie_block->ckc,
2717 (u64 __user *)reg->addr);
2718 break;
536336c2
DD		 2719 case KVM_REG_S390_PFTOKEN:
2720 r = get_user(vcpu->arch.pfault_token,
2721 (u64 __user *)reg->addr);
9fbd8082
DH		 2722 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2723 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 2724 break;
2725 case KVM_REG_S390_PFCOMPARE:
2726 r = get_user(vcpu->arch.pfault_compare,
2727 (u64 __user *)reg->addr);
2728 break;
2729 case KVM_REG_S390_PFSELECT:
2730 r = get_user(vcpu->arch.pfault_select,
2731 (u64 __user *)reg->addr);
2732 break;
672550fb
CB		 2733 case KVM_REG_S390_PP:
2734 r = get_user(vcpu->arch.sie_block->pp,
2735 (u64 __user *)reg->addr);
2736 break;
afa45ff5
CB		 2737 case KVM_REG_S390_GBEA:
2738 r = get_user(vcpu->arch.sie_block->gbea,
2739 (u64 __user *)reg->addr);
2740 break;
14eebd91
CO		 2741 default:
2742 break;
2743 }
2744
2745 return r;
2746}
b6d33834 2747
b0c632db
HC		 2748static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2749{
b0c632db 2750 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 2751 return 0;
2752}
2753
2754int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2755{
875656fe 2756 vcpu_load(vcpu);
5a32c1af 2757 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
875656fe 2758 vcpu_put(vcpu);
b0c632db
HC		 2759 return 0;
2760}
2761
2762int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2763{
1fc9b76b 2764 vcpu_load(vcpu);
5a32c1af 2765 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1fc9b76b 2766 vcpu_put(vcpu);
b0c632db
HC		 2767 return 0;
2768}
2769
2770int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2771 struct kvm_sregs *sregs)
2772{
b4ef9d4e
CD		 2773 vcpu_load(vcpu);
2774
59674c1a 2775 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 2776 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b4ef9d4e
CD		 2777
2778 vcpu_put(vcpu);
b0c632db
HC		 2779 return 0;
2780}
2781
2782int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2783 struct kvm_sregs *sregs)
2784{
bcdec41c
CD		 2785 vcpu_load(vcpu);
2786
59674c1a 2787 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 2788 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
bcdec41c
CD		 2789
2790 vcpu_put(vcpu);
b0c632db
HC		 2791 return 0;
2792}
2793
2794int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2795{
6a96bc7f
CD		 2796 int ret = 0;
2797
2798 vcpu_load(vcpu);
2799
2800 if (test_fp_ctl(fpu->fpc)) {
2801 ret = -EINVAL;
2802 goto out;
2803 }
e1788bb9 2804 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 2805 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2806 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2807 (freg_t *) fpu->fprs);
9abc2a08 2808 else
a7d4b8f2 2809 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
6a96bc7f
CD		 2810
2811out:
2812 vcpu_put(vcpu);
2813 return ret;
b0c632db
HC		 2814}
2815
2816int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2817{
1393123e
CD		 2818 vcpu_load(vcpu);
2819
9abc2a08
DH		 2820 /* make sure we have the latest values */
2821 save_fpu_regs();
2822 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 2823 convert_vx_to_fp((freg_t *) fpu->fprs,
2824 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 2825 else
a7d4b8f2 2826 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 2827 fpu->fpc = vcpu->run->s.regs.fpc;
1393123e
CD		 2828
2829 vcpu_put(vcpu);
b0c632db
HC		 2830 return 0;
2831}
2832
2833static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2834{
2835 int rc = 0;
2836
7a42fdc2 2837 if (!is_vcpu_stopped(vcpu))
b0c632db 2838 rc = -EBUSY;
d7b0b5eb
CO		 2839 else {
2840 vcpu->run->psw_mask = psw.mask;
2841 vcpu->run->psw_addr = psw.addr;
2842 }
b0c632db
HC		 2843 return rc;
2844}
2845
2846int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2847 struct kvm_translation *tr)
2848{
2849 return -EINVAL; /* not implemented yet */
2850}
2851
27291e21
DH		 2852#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2853 KVM_GUESTDBG_USE_HW_BP | \
2854 KVM_GUESTDBG_ENABLE)
2855
d0bfb940
JK		 2856int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2857 struct kvm_guest_debug *dbg)
b0c632db 2858{
27291e21
DH		 2859 int rc = 0;
2860
66b56562
CD		 2861 vcpu_load(vcpu);
2862
27291e21
DH		 2863 vcpu->guest_debug = 0;
2864 kvm_s390_clear_bp_data(vcpu);
2865
66b56562
CD		 2866 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
2867 rc = -EINVAL;
2868 goto out;
2869 }
2870 if (!sclp.has_gpere) {
2871 rc = -EINVAL;
2872 goto out;
2873 }
27291e21
DH		 2874
2875 if (dbg->control & KVM_GUESTDBG_ENABLE) {
2876 vcpu->guest_debug = dbg->control;
2877 /* enforce guest PER */
ef8f4f49 2878 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 2879
2880 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2881 rc = kvm_s390_import_bp_data(vcpu, dbg);
2882 } else {
9daecfc6 2883 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 2884 vcpu->arch.guestdbg.last_bp = 0;
2885 }
2886
2887 if (rc) {
2888 vcpu->guest_debug = 0;
2889 kvm_s390_clear_bp_data(vcpu);
9daecfc6 2890 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 2891 }
2892
66b56562
CD		 2893out:
2894 vcpu_put(vcpu);
27291e21 2895 return rc;
b0c632db
HC		 2896}
2897
62d9f0db
MT		 2898int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2899 struct kvm_mp_state *mp_state)
2900{
fd232561
CD		 2901 int ret;
2902
2903 vcpu_load(vcpu);
2904
6352e4d2 2905 /* CHECK_STOP and LOAD are not supported yet */
fd232561
CD		 2906 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2907 KVM_MP_STATE_OPERATING;
2908
2909 vcpu_put(vcpu);
2910 return ret;
62d9f0db
MT		 2911}
2912
2913int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2914 struct kvm_mp_state *mp_state)
2915{
6352e4d2
DH		 2916 int rc = 0;
2917
e83dff5e
CD		 2918 vcpu_load(vcpu);
2919
6352e4d2
DH		 2920 /* user space knows about this interface - let it control the state */
2921 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2922
2923 switch (mp_state->mp_state) {
2924 case KVM_MP_STATE_STOPPED:
2925 kvm_s390_vcpu_stop(vcpu);
2926 break;
2927 case KVM_MP_STATE_OPERATING:
2928 kvm_s390_vcpu_start(vcpu);
2929 break;
2930 case KVM_MP_STATE_LOAD:
2931 case KVM_MP_STATE_CHECK_STOP:
2932 /* fall through - CHECK_STOP and LOAD are not supported yet */
2933 default:
2934 rc = -ENXIO;
2935 }
2936
e83dff5e 2937 vcpu_put(vcpu);
6352e4d2 2938 return rc;
62d9f0db
MT		 2939}
2940
8ad35755
DH		 2941static bool ibs_enabled(struct kvm_vcpu *vcpu)
2942{
8d5fb0dc 2943 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 2944}
2945
2c70fe44
CB		 2946static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2947{
8ad35755 2948retry:
8e236546 2949 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 2950 if (!kvm_request_pending(vcpu))
586b7ccd 2951 return 0;
2c70fe44
CB		 2952 /*
2953 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 2954 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 2955 * This ensures that the ipte instruction for this request has
2956 * already finished. We might race against a second unmapper that
2957 * wants to set the blocking bit. Lets just retry the request loop.
2958 */
8ad35755 2959 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 2960 int rc;
b2d73b2a
MS		 2961 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2962 kvm_s390_get_prefix(vcpu),
2963 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 2964 if (rc) {
2965 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 2966 return rc;
aca411a4 2967 }
8ad35755 2968 goto retry;
2c70fe44 2969 }
8ad35755 2970
d3d692c8
DH		 2971 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2972 vcpu->arch.sie_block->ihcpu = 0xffff;
2973 goto retry;
2974 }
2975
8ad35755
DH		 2976 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2977 if (!ibs_enabled(vcpu)) {
2978 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
ef8f4f49 2979 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 2980 }
2981 goto retry;
2c70fe44 2982 }
8ad35755
DH		 2983
2984 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2985 if (ibs_enabled(vcpu)) {
2986 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
9daecfc6 2987 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 2988 }
2989 goto retry;
2990 }
2991
6502a34c
DH		 2992 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2993 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2994 goto retry;
2995 }
2996
190df4a2
CI		 2997 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
2998 /*
2999 * Disable CMMA virtualization; we will emulate the ESSA
3000 * instruction manually, in order to provide additional
3001 * functionalities needed for live migration.
3002 */
3003 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3004 goto retry;
3005 }
3006
3007 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3008 /*
3009 * Re-enable CMMA virtualization if CMMA is available and
3010 * was used.
3011 */
3012 if ((vcpu->kvm->arch.use_cmma) &&
3013 (vcpu->kvm->mm->context.use_cmma))
3014 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3015 goto retry;
3016 }
3017
0759d068 3018 /* nothing to do, just clear the request */
72875d8a 3019 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
0759d068 3020
2c70fe44
CB		 3021 return 0;
3022}
3023
8fa1696e
CW		 3024void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
3025 const struct kvm_s390_vm_tod_clock *gtod)
3026{
3027 struct kvm_vcpu *vcpu;
3028 struct kvm_s390_tod_clock_ext htod;
3029 int i;
3030
3031 mutex_lock(&kvm->lock);
3032 preempt_disable();
3033
3034 get_tod_clock_ext((char *)&htod);
3035
3036 kvm->arch.epoch = gtod->tod - htod.tod;
3037 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3038
3039 if (kvm->arch.epoch > gtod->tod)
3040 kvm->arch.epdx -= 1;
3041
3042 kvm_s390_vcpu_block_all(kvm);
3043 kvm_for_each_vcpu(i, vcpu, kvm) {
3044 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3045 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3046 }
3047
3048 kvm_s390_vcpu_unblock_all(kvm);
3049 preempt_enable();
3050 mutex_unlock(&kvm->lock);
3051}
3052
25ed1675
DH		 3053void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
3054{
3055 struct kvm_vcpu *vcpu;
3056 int i;
3057
3058 mutex_lock(&kvm->lock);
3059 preempt_disable();
3060 kvm->arch.epoch = tod - get_tod_clock();
3061 kvm_s390_vcpu_block_all(kvm);
3062 kvm_for_each_vcpu(i, vcpu, kvm)
3063 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3064 kvm_s390_vcpu_unblock_all(kvm);
3065 preempt_enable();
3066 mutex_unlock(&kvm->lock);
3067}
3068
fa576c58
TH		 3069/**
3070 * kvm_arch_fault_in_page - fault-in guest page if necessary
3071 * @vcpu: The corresponding virtual cpu
3072 * @gpa: Guest physical address
3073 * @writable: Whether the page should be writable or not
3074 *
3075 * Make sure that a guest page has been faulted-in on the host.
3076 *
3077 * Return: Zero on success, negative error code otherwise.
3078 */
3079long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 3080{
527e30b4
MS		 3081 return gmap_fault(vcpu->arch.gmap, gpa,
3082 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 3083}
3084
3c038e6b
DD		 3085static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3086 unsigned long token)
3087{
3088 struct kvm_s390_interrupt inti;
383d0b05 3089 struct kvm_s390_irq irq;
3c038e6b
DD		 3090
3091 if (start_token) {
383d0b05
JF		 3092 irq.u.ext.ext_params2 = token;
3093 irq.type = KVM_S390_INT_PFAULT_INIT;
3094 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 3095 } else {
3096 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 3097 inti.parm64 = token;
3c038e6b
DD		 3098 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3099 }
3100}
3101
3102void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3103 struct kvm_async_pf *work)
3104{
3105 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3106 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3107}
3108
3109void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3110 struct kvm_async_pf *work)
3111{
3112 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3113 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3114}
3115
3116void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3117 struct kvm_async_pf *work)
3118{
3119 /* s390 will always inject the page directly */
3120}
3121
3122bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3123{
3124 /*
3125 * s390 will always inject the page directly,
3126 * but we still want check_async_completion to cleanup
3127 */
3128 return true;
3129}
3130
3131static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3132{
3133 hva_t hva;
3134 struct kvm_arch_async_pf arch;
3135 int rc;
3136
3137 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3138 return 0;
3139 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3140 vcpu->arch.pfault_compare)
3141 return 0;
3142 if (psw_extint_disabled(vcpu))
3143 return 0;
9a022067 3144 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 3145 return 0;
3146 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3147 return 0;
3148 if (!vcpu->arch.gmap->pfault_enabled)
3149 return 0;
3150
81480cc1
HC		 3151 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3152 hva += current->thread.gmap_addr & ~PAGE_MASK;
3153 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 3154 return 0;
3155
3156 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3157 return rc;
3158}
3159
3fb4c40f 3160static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 3161{
3fb4c40f 3162 int rc, cpuflags;
e168bf8d 3163
3c038e6b
DD		 3164 /*
3165 * On s390 notifications for arriving pages will be delivered directly
3166 * to the guest but the house keeping for completed pfaults is
3167 * handled outside the worker.
3168 */
3169 kvm_check_async_pf_completion(vcpu);
3170
7ec7c8c7
CB		 3171 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3172 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 3173
3174 if (need_resched())
3175 schedule();
3176
d3a73acb 3177 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 3178 s390_handle_mcck();
3179
79395031
JF		 3180 if (!kvm_is_ucontrol(vcpu->kvm)) {
3181 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3182 if (rc)
3183 return rc;
3184 }
0ff31867 3185
2c70fe44
CB		 3186 rc = kvm_s390_handle_requests(vcpu);
3187 if (rc)
3188 return rc;
3189
27291e21
DH		 3190 if (guestdbg_enabled(vcpu)) {
3191 kvm_s390_backup_guest_per_regs(vcpu);
3192 kvm_s390_patch_guest_per_regs(vcpu);
3193 }
3194
b0c632db 3195 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 3196 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3197 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3198 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 3199
3fb4c40f
TH		 3200 return 0;
3201}
3202
492d8642
TH		 3203static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3204{
56317920
DH		 3205 struct kvm_s390_pgm_info pgm_info = {
3206 .code = PGM_ADDRESSING,
3207 };
3208 u8 opcode, ilen;
492d8642
TH		 3209 int rc;
3210
3211 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3212 trace_kvm_s390_sie_fault(vcpu);
3213
3214 /*
3215 * We want to inject an addressing exception, which is defined as a
3216 * suppressing or terminating exception. However, since we came here
3217 * by a DAT access exception, the PSW still points to the faulting
3218 * instruction since DAT exceptions are nullifying. So we've got
3219 * to look up the current opcode to get the length of the instruction
3220 * to be able to forward the PSW.
3221 */
3fa8cad7 3222 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 3223 ilen = insn_length(opcode);
9b0d721a
DH		 3224 if (rc < 0) {
3225 return rc;
3226 } else if (rc) {
3227 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3228 * Forward by arbitrary ilc, injection will take care of
3229 * nullification if necessary.
3230 */
3231 pgm_info = vcpu->arch.pgm;
3232 ilen = 4;
3233 }
56317920
DH		 3234 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3235 kvm_s390_forward_psw(vcpu, ilen);
3236 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 3237}
3238
3fb4c40f
TH		 3239static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3240{
4d62fcc0
QH		 3241 struct mcck_volatile_info *mcck_info;
3242 struct sie_page *sie_page;
3243
2b29a9fd
DD		 3244 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3245 vcpu->arch.sie_block->icptcode);
3246 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3247
27291e21
DH		 3248 if (guestdbg_enabled(vcpu))
3249 kvm_s390_restore_guest_per_regs(vcpu);
3250
7ec7c8c7
CB		 3251 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3252 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 3253
4d62fcc0
QH		 3254 if (exit_reason == -EINTR) {
3255 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3256 sie_page = container_of(vcpu->arch.sie_block,
3257 struct sie_page, sie_block);
3258 mcck_info = &sie_page->mcck_info;
3259 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3260 return 0;
3261 }
3262
71f116bf
DH		 3263 if (vcpu->arch.sie_block->icptcode > 0) {
3264 int rc = kvm_handle_sie_intercept(vcpu);
3265
3266 if (rc != -EOPNOTSUPP)
3267 return rc;
3268 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3269 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3270 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3271 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3272 return -EREMOTE;
3273 } else if (exit_reason != -EFAULT) {
3274 vcpu->stat.exit_null++;
3275 return 0;
210b1607
TH		 3276 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3277 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3278 vcpu->run->s390_ucontrol.trans_exc_code =
3279 current->thread.gmap_addr;
3280 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 3281 return -EREMOTE;
24eb3a82 3282 } else if (current->thread.gmap_pfault) {
3c038e6b 3283 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 3284 current->thread.gmap_pfault = 0;
71f116bf
DH		 3285 if (kvm_arch_setup_async_pf(vcpu))
3286 return 0;
3287 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 3288 }
71f116bf 3289 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 3290}
3291
3292static int __vcpu_run(struct kvm_vcpu *vcpu)
3293{
3294 int rc, exit_reason;
3295
800c1065
TH		 3296 /*
3297 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3298 * ning the guest), so that memslots (and other stuff) are protected
3299 */
3300 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3301
a76ccff6
TH		 3302 do {
3303 rc = vcpu_pre_run(vcpu);
3304 if (rc)
3305 break;
3fb4c40f 3306
800c1065 3307 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 3308 /*
3309 * As PF_VCPU will be used in fault handler, between
3310 * guest_enter and guest_exit should be no uaccess.
3311 */
0097d12e 3312 local_irq_disable();
6edaa530 3313 guest_enter_irqoff();
db0758b2 3314 __disable_cpu_timer_accounting(vcpu);
0097d12e 3315 local_irq_enable();
a76ccff6
TH		 3316 exit_reason = sie64a(vcpu->arch.sie_block,
3317 vcpu->run->s.regs.gprs);
0097d12e 3318 local_irq_disable();
db0758b2 3319 __enable_cpu_timer_accounting(vcpu);
6edaa530 3320 guest_exit_irqoff();
0097d12e 3321 local_irq_enable();
800c1065 3322 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 3323
3324 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 3325 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 3326
800c1065 3327 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 3328 return rc;
b0c632db
HC		 3329}
3330
b028ee3e
DH		 3331static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3332{
4d5f2c04 3333 struct runtime_instr_cb *riccb;
4e0b1ab7 3334 struct gs_cb *gscb;
4d5f2c04
CB		 3335
3336 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 3337 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 3338 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3339 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3340 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3341 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3342 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3343 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 3344 /* some control register changes require a tlb flush */
3345 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 3346 }
3347 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 3348 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 3349 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3350 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3351 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3352 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3353 }
3354 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3355 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3356 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3357 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 3358 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3359 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 3360 }
80cd8763
FZ		 3361 /*
3362 * If userspace sets the riccb (e.g. after migration) to a valid state,
3363 * we should enable RI here instead of doing the lazy enablement.
3364 */
3365 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 3366 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 3367 riccb->v &&
0c9d8683 3368 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 3369 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 3370 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 3371 }
4e0b1ab7
FZ		 3372 /*
3373 * If userspace sets the gscb (e.g. after migration) to non-zero,
3374 * we should enable GS here instead of doing the lazy enablement.
3375 */
3376 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3377 test_kvm_facility(vcpu->kvm, 133) &&
3378 gscb->gssm &&
3379 !vcpu->arch.gs_enabled) {
3380 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3381 vcpu->arch.sie_block->ecb |= ECB_GS;
3382 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3383 vcpu->arch.gs_enabled = 1;
80cd8763 3384 }
35b3fde6
CB		 3385 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3386 test_kvm_facility(vcpu->kvm, 82)) {
3387 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3388 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3389 }
31d8b8d4
CB		 3390 save_access_regs(vcpu->arch.host_acrs);
3391 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 3392 /* save host (userspace) fprs/vrs */
3393 save_fpu_regs();
3394 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3395 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3396 if (MACHINE_HAS_VX)
3397 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3398 else
3399 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3400 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3401 if (test_fp_ctl(current->thread.fpu.fpc))
3402 /* User space provided an invalid FPC, let's clear it */
3403 current->thread.fpu.fpc = 0;
4e0b1ab7
FZ		 3404 if (MACHINE_HAS_GS) {
3405 preempt_disable();
3406 __ctl_set_bit(2, 4);
3407 if (current->thread.gs_cb) {
3408 vcpu->arch.host_gscb = current->thread.gs_cb;
3409 save_gs_cb(vcpu->arch.host_gscb);
3410 }
3411 if (vcpu->arch.gs_enabled) {
3412 current->thread.gs_cb = (struct gs_cb *)
3413 &vcpu->run->s.regs.gscb;
3414 restore_gs_cb(current->thread.gs_cb);
3415 }
3416 preempt_enable();
3417 }
80cd8763 3418
b028ee3e
DH		 3419 kvm_run->kvm_dirty_regs = 0;
3420}
3421
3422static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3423{
3424 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3425 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3426 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3427 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 3428 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 3429 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3430 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3431 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3432 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3433 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3434 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3435 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
35b3fde6 3436 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
31d8b8d4
CB		 3437 save_access_regs(vcpu->run->s.regs.acrs);
3438 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 3439 /* Save guest register state */
3440 save_fpu_regs();
3441 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3442 /* Restore will be done lazily at return */
3443 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3444 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4e0b1ab7
FZ		 3445 if (MACHINE_HAS_GS) {
3446 __ctl_set_bit(2, 4);
3447 if (vcpu->arch.gs_enabled)
3448 save_gs_cb(current->thread.gs_cb);
3449 preempt_disable();
3450 current->thread.gs_cb = vcpu->arch.host_gscb;
3451 restore_gs_cb(vcpu->arch.host_gscb);
3452 preempt_enable();
3453 if (!vcpu->arch.host_gscb)
3454 __ctl_clear_bit(2, 4);
3455 vcpu->arch.host_gscb = NULL;
3456 }
e1788bb9 3457
b028ee3e
DH		 3458}
3459
b0c632db
HC		 3460int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3461{
8f2abe6a 3462 int rc;
b0c632db 3463
460df4c1
PB		 3464 if (kvm_run->immediate_exit)
3465 return -EINTR;
3466
accb757d
CD		 3467 vcpu_load(vcpu);
3468
27291e21
DH		 3469 if (guestdbg_exit_pending(vcpu)) {
3470 kvm_s390_prepare_debug_exit(vcpu);
accb757d
CD		 3471 rc = 0;
3472 goto out;
27291e21
DH		 3473 }
3474
20b7035c 3475 kvm_sigset_activate(vcpu);
b0c632db 3476
6352e4d2
DH		 3477 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3478 kvm_s390_vcpu_start(vcpu);
3479 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 3480 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2 3481 vcpu->vcpu_id);
accb757d
CD		 3482 rc = -EINVAL;
3483 goto out;
6352e4d2 3484 }
b0c632db 3485
b028ee3e 3486 sync_regs(vcpu, kvm_run);
db0758b2 3487 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 3488
dab4079d 3489 might_fault();
a76ccff6 3490 rc = __vcpu_run(vcpu);
9ace903d 3491
b1d16c49
CE		 3492 if (signal_pending(current) && !rc) {
3493 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 3494 rc = -EINTR;
b1d16c49 3495 }
8f2abe6a 3496
27291e21
DH		 3497 if (guestdbg_exit_pending(vcpu) && !rc) {
3498 kvm_s390_prepare_debug_exit(vcpu);
3499 rc = 0;
3500 }
3501
8f2abe6a 3502 if (rc == -EREMOTE) {
71f116bf 3503 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 3504 rc = 0;
3505 }
b0c632db 3506
db0758b2 3507 disable_cpu_timer_accounting(vcpu);
b028ee3e 3508 store_regs(vcpu, kvm_run);
d7b0b5eb 3509
20b7035c 3510 kvm_sigset_deactivate(vcpu);
b0c632db 3511
b0c632db 3512 vcpu->stat.exit_userspace++;
accb757d
CD		 3513out:
3514 vcpu_put(vcpu);
7e8e6ab4 3515 return rc;
b0c632db
HC		 3516}
3517
b0c632db
HC		 3518/*
3519 * store status at address
3520 * we use have two special cases:
3521 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3522 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3523 */
d0bce605 3524int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 3525{
092670cd 3526 unsigned char archmode = 1;
9abc2a08 3527 freg_t fprs[NUM_FPRS];
fda902cb 3528 unsigned int px;
4287f247 3529 u64 clkcomp, cputm;
d0bce605 3530 int rc;
b0c632db 3531
d9a3a09a 3532 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 3533 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3534 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 3535 return -EFAULT;
d9a3a09a 3536 gpa = 0;
d0bce605
HC		 3537 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3538 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 3539 return -EFAULT;
d9a3a09a
MS		 3540 gpa = px;
3541 } else
3542 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 3543
3544 /* manually convert vector registers if necessary */
3545 if (MACHINE_HAS_VX) {
9522b37f 3546 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 3547 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3548 fprs, 128);
3549 } else {
3550 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 3551 vcpu->run->s.regs.fprs, 128);
9abc2a08 3552 }
d9a3a09a 3553 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 3554 vcpu->run->s.regs.gprs, 128);
d9a3a09a 3555 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 3556 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 3557 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 3558 &px, 4);
d9a3a09a 3559 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 3560 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 3561 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 3562 &vcpu->arch.sie_block->todpr, 4);
4287f247 3563 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 3564 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 3565 &cputm, 8);
178bd789 3566 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 3567 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 3568 &clkcomp, 8);
d9a3a09a 3569 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 3570 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 3571 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 3572 &vcpu->arch.sie_block->gcr, 128);
3573 return rc ? -EFAULT : 0;
b0c632db
HC		 3574}
3575
e879892c
TH		 3576int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3577{
3578 /*
3579 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 3580 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 3581 * it into the save area
3582 */
d0164ee2 3583 save_fpu_regs();
9abc2a08 3584 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 3585 save_access_regs(vcpu->run->s.regs.acrs);
3586
3587 return kvm_s390_store_status_unloaded(vcpu, addr);
3588}
3589
8ad35755
DH		 3590static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3591{
3592 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 3593 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 3594}
3595
3596static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3597{
3598 unsigned int i;
3599 struct kvm_vcpu *vcpu;
3600
3601 kvm_for_each_vcpu(i, vcpu, kvm) {
3602 __disable_ibs_on_vcpu(vcpu);
3603 }
3604}
3605
3606static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3607{
09a400e7
DH		 3608 if (!sclp.has_ibs)
3609 return;
8ad35755 3610 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 3611 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 3612}
3613
6852d7b6
DH		 3614void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3615{
8ad35755
DH		 3616 int i, online_vcpus, started_vcpus = 0;
3617
3618 if (!is_vcpu_stopped(vcpu))
3619 return;
3620
6852d7b6 3621 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 3622 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3623 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3624 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3625
3626 for (i = 0; i < online_vcpus; i++) {
3627 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3628 started_vcpus++;
3629 }
3630
3631 if (started_vcpus == 0) {
3632 /* we're the only active VCPU -> speed it up */
3633 __enable_ibs_on_vcpu(vcpu);
3634 } else if (started_vcpus == 1) {
3635 /*
3636 * As we are starting a second VCPU, we have to disable
3637 * the IBS facility on all VCPUs to remove potentially
3638 * oustanding ENABLE requests.
3639 */
3640 __disable_ibs_on_all_vcpus(vcpu->kvm);
3641 }
3642
9daecfc6 3643 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
8ad35755
DH		 3644 /*
3645 * Another VCPU might have used IBS while we were offline.
3646 * Let's play safe and flush the VCPU at startup.
3647 */
d3d692c8 3648 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 3649 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3650 return;
6852d7b6
DH		 3651}
3652
3653void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3654{
8ad35755
DH		 3655 int i, online_vcpus, started_vcpus = 0;
3656 struct kvm_vcpu *started_vcpu = NULL;
3657
3658 if (is_vcpu_stopped(vcpu))
3659 return;
3660
6852d7b6 3661 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 3662 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 3663 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 3664 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3665
32f5ff63 3666 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 3667 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 3668
ef8f4f49 3669 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
8ad35755
DH		 3670 __disable_ibs_on_vcpu(vcpu);
3671
3672 for (i = 0; i < online_vcpus; i++) {
3673 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3674 started_vcpus++;
3675 started_vcpu = vcpu->kvm->vcpus[i];
3676 }
3677 }
3678
3679 if (started_vcpus == 1) {
3680 /*
3681 * As we only have one VCPU left, we want to enable the
3682 * IBS facility for that VCPU to speed it up.
3683 */
3684 __enable_ibs_on_vcpu(started_vcpu);
3685 }
3686
433b9ee4 3687 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 3688 return;
6852d7b6
DH		 3689}
3690
d6712df9
CH		 3691static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3692 struct kvm_enable_cap *cap)
3693{
3694 int r;
3695
3696 if (cap->flags)
3697 return -EINVAL;
3698
3699 switch (cap->cap) {
fa6b7fe9
CH		 3700 case KVM_CAP_S390_CSS_SUPPORT:
3701 if (!vcpu->kvm->arch.css_support) {
3702 vcpu->kvm->arch.css_support = 1;
c92ea7b9 3703 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 3704 trace_kvm_s390_enable_css(vcpu->kvm);
3705 }
3706 r = 0;
3707 break;
d6712df9
CH		 3708 default:
3709 r = -EINVAL;
3710 break;
3711 }
3712 return r;
3713}
3714
41408c28
TH		 3715static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3716 struct kvm_s390_mem_op *mop)
3717{
3718 void __user *uaddr = (void __user *)mop->buf;
3719 void *tmpbuf = NULL;
3720 int r, srcu_idx;
3721 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3722 | KVM_S390_MEMOP_F_CHECK_ONLY;
3723
3724 if (mop->flags & ~supported_flags)
3725 return -EINVAL;
3726
3727 if (mop->size > MEM_OP_MAX_SIZE)
3728 return -E2BIG;
3729
3730 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3731 tmpbuf = vmalloc(mop->size);
3732 if (!tmpbuf)
3733 return -ENOMEM;
3734 }
3735
3736 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3737
3738 switch (mop->op) {
3739 case KVM_S390_MEMOP_LOGICAL_READ:
3740 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3741 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3742 mop->size, GACC_FETCH);
41408c28
TH		 3743 break;
3744 }
3745 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3746 if (r == 0) {
3747 if (copy_to_user(uaddr, tmpbuf, mop->size))
3748 r = -EFAULT;
3749 }
3750 break;
3751 case KVM_S390_MEMOP_LOGICAL_WRITE:
3752 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 3753 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3754 mop->size, GACC_STORE);
41408c28
TH		 3755 break;
3756 }
3757 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3758 r = -EFAULT;
3759 break;
3760 }
3761 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3762 break;
3763 default:
3764 r = -EINVAL;
3765 }
3766
3767 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3768
3769 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3770 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3771
3772 vfree(tmpbuf);
3773 return r;
3774}
3775
5cb0944c
PB		 3776long kvm_arch_vcpu_async_ioctl(struct file *filp,
3777 unsigned int ioctl, unsigned long arg)
b0c632db
HC		 3778{
3779 struct kvm_vcpu *vcpu = filp->private_data;
3780 void __user *argp = (void __user *)arg;
3781
93736624 3782 switch (ioctl) {
47b43c52
JF		 3783 case KVM_S390_IRQ: {
3784 struct kvm_s390_irq s390irq;
3785
47b43c52 3786 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
9b062471
CD		 3787 return -EFAULT;
3788 return kvm_s390_inject_vcpu(vcpu, &s390irq);
47b43c52 3789 }
93736624 3790 case KVM_S390_INTERRUPT: {
ba5c1e9b 3791 struct kvm_s390_interrupt s390int;
383d0b05 3792 struct kvm_s390_irq s390irq;
ba5c1e9b
CO		 3793
3794 if (copy_from_user(&s390int, argp, sizeof(s390int)))
9b062471 3795 return -EFAULT;
383d0b05
JF		 3796 if (s390int_to_s390irq(&s390int, &s390irq))
3797 return -EINVAL;
9b062471 3798 return kvm_s390_inject_vcpu(vcpu, &s390irq);
ba5c1e9b 3799 }
9b062471 3800 }
5cb0944c
PB		 3801 return -ENOIOCTLCMD;
3802}
3803
3804long kvm_arch_vcpu_ioctl(struct file *filp,
3805 unsigned int ioctl, unsigned long arg)
3806{
3807 struct kvm_vcpu *vcpu = filp->private_data;
3808 void __user *argp = (void __user *)arg;
3809 int idx;
3810 long r;
9b062471
CD		 3811
3812 vcpu_load(vcpu);
3813
3814 switch (ioctl) {
b0c632db 3815 case KVM_S390_STORE_STATUS:
800c1065 3816 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 3817 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 3818 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 3819 break;
b0c632db
HC		 3820 case KVM_S390_SET_INITIAL_PSW: {
3821 psw_t psw;
3822
bc923cc9 3823 r = -EFAULT;
b0c632db 3824 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 3825 break;
3826 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3827 break;
b0c632db
HC		 3828 }
3829 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 3830 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3831 break;
14eebd91
CO		 3832 case KVM_SET_ONE_REG:
3833 case KVM_GET_ONE_REG: {
3834 struct kvm_one_reg reg;
3835 r = -EFAULT;
3836 if (copy_from_user(&reg, argp, sizeof(reg)))
3837 break;
3838 if (ioctl == KVM_SET_ONE_REG)
3839 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3840 else
3841 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3842 break;
3843 }
27e0393f
CO		 3844#ifdef CONFIG_KVM_S390_UCONTROL
3845 case KVM_S390_UCAS_MAP: {
3846 struct kvm_s390_ucas_mapping ucasmap;
3847
3848 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3849 r = -EFAULT;
3850 break;
3851 }
3852
3853 if (!kvm_is_ucontrol(vcpu->kvm)) {
3854 r = -EINVAL;
3855 break;
3856 }
3857
3858 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3859 ucasmap.vcpu_addr, ucasmap.length);
3860 break;
3861 }
3862 case KVM_S390_UCAS_UNMAP: {
3863 struct kvm_s390_ucas_mapping ucasmap;
3864
3865 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3866 r = -EFAULT;
3867 break;
3868 }
3869
3870 if (!kvm_is_ucontrol(vcpu->kvm)) {
3871 r = -EINVAL;
3872 break;
3873 }
3874
3875 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3876 ucasmap.length);
3877 break;
3878 }
3879#endif
ccc7910f 3880 case KVM_S390_VCPU_FAULT: {
527e30b4 3881 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 3882 break;
3883 }
d6712df9
CH		 3884 case KVM_ENABLE_CAP:
3885 {
3886 struct kvm_enable_cap cap;
3887 r = -EFAULT;
3888 if (copy_from_user(&cap, argp, sizeof(cap)))
3889 break;
3890 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3891 break;
3892 }
41408c28
TH		 3893 case KVM_S390_MEM_OP: {
3894 struct kvm_s390_mem_op mem_op;
3895
3896 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3897 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3898 else
3899 r = -EFAULT;
3900 break;
3901 }
816c7667
JF		 3902 case KVM_S390_SET_IRQ_STATE: {
3903 struct kvm_s390_irq_state irq_state;
3904
3905 r = -EFAULT;
3906 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3907 break;
3908 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3909 irq_state.len == 0 ||
3910 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3911 r = -EINVAL;
3912 break;
3913 }
bb64da9a 3914 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 3915 r = kvm_s390_set_irq_state(vcpu,
3916 (void __user *) irq_state.buf,
3917 irq_state.len);
3918 break;
3919 }
3920 case KVM_S390_GET_IRQ_STATE: {
3921 struct kvm_s390_irq_state irq_state;
3922
3923 r = -EFAULT;
3924 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3925 break;
3926 if (irq_state.len == 0) {
3927 r = -EINVAL;
3928 break;
3929 }
bb64da9a 3930 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 3931 r = kvm_s390_get_irq_state(vcpu,
3932 (__u8 __user *) irq_state.buf,
3933 irq_state.len);
3934 break;
3935 }
b0c632db 3936 default:
3e6afcf1 3937 r = -ENOTTY;
b0c632db 3938 }
9b062471
CD		 3939
3940 vcpu_put(vcpu);
bc923cc9 3941 return r;
b0c632db
HC		 3942}
3943
5b1c1493
CO		 3944int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3945{
3946#ifdef CONFIG_KVM_S390_UCONTROL
3947 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3948 && (kvm_is_ucontrol(vcpu->kvm))) {
3949 vmf->page = virt_to_page(vcpu->arch.sie_block);
3950 get_page(vmf->page);
3951 return 0;
3952 }
3953#endif
3954 return VM_FAULT_SIGBUS;
3955}
3956
5587027c
AK		 3957int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3958 unsigned long npages)
db3fe4eb
TY		 3959{
3960 return 0;
3961}
3962
b0c632db 3963/* Section: memory related */
f7784b8e
MT		 3964int kvm_arch_prepare_memory_region(struct kvm *kvm,
3965 struct kvm_memory_slot *memslot,
09170a49 3966 const struct kvm_userspace_memory_region *mem,
7b6195a9 3967 enum kvm_mr_change change)
b0c632db 3968{
dd2887e7
NW		 3969 /* A few sanity checks. We can have memory slots which have to be
3970 located/ended at a segment boundary (1MB). The memory in userland is
3971 ok to be fragmented into various different vmas. It is okay to mmap()
3972 and munmap() stuff in this slot after doing this call at any time */
b0c632db 3973
598841ca 3974 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 3975 return -EINVAL;
3976
598841ca 3977 if (mem->memory_size & 0xffffful)
b0c632db
HC		 3978 return -EINVAL;
3979
a3a92c31
DD		 3980 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3981 return -EINVAL;
3982
f7784b8e
MT		 3983 return 0;
3984}
3985
3986void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 3987 const struct kvm_userspace_memory_region *mem,
8482644a 3988 const struct kvm_memory_slot *old,
f36f3f28 3989 const struct kvm_memory_slot *new,
8482644a 3990 enum kvm_mr_change change)
f7784b8e 3991{
f7850c92 3992 int rc;
f7784b8e 3993
2cef4deb
CB		 3994 /* If the basics of the memslot do not change, we do not want
3995 * to update the gmap. Every update causes several unnecessary
3996 * segment translation exceptions. This is usually handled just
3997 * fine by the normal fault handler + gmap, but it will also
3998 * cause faults on the prefix page of running guest CPUs.
3999 */
4000 if (old->userspace_addr == mem->userspace_addr &&
4001 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
4002 old->npages * PAGE_SIZE == mem->memory_size)
4003 return;
598841ca
CO		 4004
4005 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4006 mem->guest_phys_addr, mem->memory_size);
4007 if (rc)
ea2cdd27 4008 pr_warn("failed to commit memory region\n");
598841ca 4009 return;
b0c632db
HC		 4010}
4011
60a37709
AY		 4012static inline unsigned long nonhyp_mask(int i)
4013{
4014 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4015
4016 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4017}
4018
3491caf2
CB		 4019void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4020{
4021 vcpu->valid_wakeup = false;
4022}
4023
b0c632db
HC		 4024static int __init kvm_s390_init(void)
4025{
60a37709
AY		 4026 int i;
4027
07197fd0
DH		 4028 if (!sclp.has_sief2) {
4029 pr_info("SIE not available\n");
4030 return -ENODEV;
4031 }
4032
60a37709
AY		 4033 for (i = 0; i < 16; i++)
4034 kvm_s390_fac_list_mask[i] |=
4035 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4036
9d8d5786 4037 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 4038}
4039
4040static void __exit kvm_s390_exit(void)
4041{
4042 kvm_exit();
4043}
4044
4045module_init(kvm_s390_init);
4046module_exit(kvm_s390_exit);
566af940
CH		 4047
4048/*
4049 * Enable autoloading of the kvm module.
4050 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4051 * since x86 takes a different approach.
4052 */
4053#include <linux/miscdevice.h>
4054MODULE_ALIAS_MISCDEV(KVM_MINOR);
4055MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)