projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'core-objtool-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
d809aa23 1// SPDX-License-Identifier: GPL-2.0
b0c632db 2/*
bb64da9a 3 * hosting IBM Z kernel virtual machines (s390x)
b0c632db 4 *
a37cb07a 5 * Copyright IBM Corp. 2008, 2018
b0c632db
HC		 6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 11 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 12 */
13
7aedd9d4
MM		 14#define KMSG_COMPONENT "kvm-s390"
15#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
16
b0c632db
HC		 17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
b2d73b2a 24#include <linux/mman.h>
b0c632db 25#include <linux/module.h>
d3217967 26#include <linux/moduleparam.h>
a374e892 27#include <linux/random.h>
b0c632db 28#include <linux/slab.h>
ba5c1e9b 29#include <linux/timer.h>
41408c28 30#include <linux/vmalloc.h>
15c9705f 31#include <linux/bitmap.h>
174cd4b1 32#include <linux/sched/signal.h>
190df4a2 33#include <linux/string.h>
174cd4b1 34
cbb870c8 35#include <asm/asm-offsets.h>
b0c632db 36#include <asm/lowcore.h>
fd5ada04 37#include <asm/stp.h>
b0c632db 38#include <asm/pgtable.h>
1e133ab2 39#include <asm/gmap.h>
f5daba1d 40#include <asm/nmi.h>
a0616cde 41#include <asm/switch_to.h>
6d3da241 42#include <asm/isc.h>
1526bf9c 43#include <asm/sclp.h>
0a763c78 44#include <asm/cpacf.h>
221bb8a4 45#include <asm/timex.h>
e585b24a 46#include <asm/ap.h>
8f2abe6a 47#include "kvm-s390.h"
b0c632db
HC		 48#include "gaccess.h"
49
5786fffa
CH		 50#define CREATE_TRACE_POINTS
51#include "trace.h"
ade38c31 52#include "trace-s390.h"
5786fffa 53
41408c28 54#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
816c7667
JF		 55#define LOCAL_IRQS 32
56#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
41408c28 58
b0c632db 59#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
ccc40c53 60#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
b0c632db
HC		 61
62struct kvm_stats_debugfs_item debugfs_entries[] = {
63 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 64 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 65 { "exit_validity", VCPU_STAT(exit_validity) },
66 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 { "exit_external_request", VCPU_STAT(exit_external_request) },
a5e0acea 68 { "exit_io_request", VCPU_STAT(exit_io_request) },
8f2abe6a 69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b 70 { "exit_instruction", VCPU_STAT(exit_instruction) },
9ec6de19 71 { "exit_pei", VCPU_STAT(exit_pei) },
ba5c1e9b
CO		 72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
a011eeb2 74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
f7819512 75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
62bea5bf 76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
3491caf2 77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
8b905d28 78 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
ce2e4f0b 79 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 80 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 81 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 82 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ccc40c53
CB		 84 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
85 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
ba5c1e9b 86 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 87 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b 88 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
ccc40c53 89 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
ba5c1e9b
CO		 90 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
91 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
92 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
ccc40c53
CB		 93 { "deliver_program", VCPU_STAT(deliver_program) },
94 { "deliver_io", VCPU_STAT(deliver_io) },
32de0749 95 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
ba5c1e9b 96 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
ccc40c53
CB		 97 { "inject_ckc", VCPU_STAT(inject_ckc) },
98 { "inject_cputm", VCPU_STAT(inject_cputm) },
99 { "inject_external_call", VCPU_STAT(inject_external_call) },
100 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
101 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
102 { "inject_io", VM_STAT(inject_io) },
103 { "inject_mchk", VCPU_STAT(inject_mchk) },
104 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
105 { "inject_program", VCPU_STAT(inject_program) },
106 { "inject_restart", VCPU_STAT(inject_restart) },
107 { "inject_service_signal", VM_STAT(inject_service_signal) },
108 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
109 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
110 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
111 { "inject_virtio", VM_STAT(inject_virtio) },
a37cb07a
CB		 112 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
113 { "instruction_gs", VCPU_STAT(instruction_gs) },
114 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
115 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
116 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
69d0d3a3 117 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
a37cb07a 118 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
453423dc 119 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
a37cb07a
CB		 120 { "instruction_sck", VCPU_STAT(instruction_sck) },
121 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
453423dc
CB		 122 { "instruction_spx", VCPU_STAT(instruction_spx) },
123 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
124 { "instruction_stap", VCPU_STAT(instruction_stap) },
a37cb07a
CB		 125 { "instruction_iske", VCPU_STAT(instruction_iske) },
126 { "instruction_ri", VCPU_STAT(instruction_ri) },
127 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
128 { "instruction_sske", VCPU_STAT(instruction_sske) },
8a242234 129 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
b31288fa 130 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 131 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
132 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
a37cb07a
CB		 133 { "instruction_tb", VCPU_STAT(instruction_tb) },
134 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
bb25b9ba 135 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
a37cb07a 136 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
95ca2cb5 137 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
a3508fbe 138 { "instruction_sie", VCPU_STAT(instruction_sie) },
5288fbf0 139 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 140 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 141 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 142 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 143 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
144 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 145 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 146 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
147 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 148 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 149 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
150 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
151 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 152 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
153 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
154 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
866c138c
CB		 155 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
156 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
157 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
8474e5ca 158 { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
866c138c
CB		 159 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
160 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
161 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
a37cb07a 162 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
b0c632db
HC		 163 { NULL }
164};
165
8fa1696e
CW		 166struct kvm_s390_tod_clock_ext {
167 __u8 epoch_idx;
168 __u64 tod;
169 __u8 reserved[7];
170} __packed;
171
a411edf1
DH		 172/* allow nested virtualization in KVM (if enabled by user space) */
173static int nested;
174module_param(nested, int, S_IRUGO);
175MODULE_PARM_DESC(nested, "Nested virtualization support");
176
a4499382
JF		 177/* allow 1m huge page guest backing, if !nested */
178static int hpage;
179module_param(hpage, int, 0444);
180MODULE_PARM_DESC(hpage, "1m huge page backing support");
b0c632db 181
8b905d28
CB		 182/* maximum percentage of steal time for polling. >100 is treated like 100 */
183static u8 halt_poll_max_steal = 10;
184module_param(halt_poll_max_steal, byte, 0644);
b41fb528 185MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
8b905d28 186
c3b9e3e1
CB		 187/*
188 * For now we handle at most 16 double words as this is what the s390 base
189 * kernel handles and stores in the prefix page. If we ever need to go beyond
190 * this, this requires changes to code, but the external uapi can stay.
191 */
192#define SIZE_INTERNAL 16
193
194/*
195 * Base feature mask that defines default mask for facilities. Consists of the
196 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
197 */
198static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
199/*
200 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
201 * and defines the facilities that can be enabled via a cpu model.
202 */
203static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
204
205static unsigned long kvm_s390_fac_size(void)
78c4b59f 206{
c3b9e3e1
CB		 207 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
208 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
209 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
210 sizeof(S390_lowcore.stfle_fac_list));
211
212 return SIZE_INTERNAL;
78c4b59f
MM		 213}
214
15c9705f
DH		 215/* available cpu features supported by kvm */
216static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
0a763c78
DH		 217/* available subfunctions indicated via query / "test bit" */
218static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
15c9705f 219
9d8d5786 220static struct gmap_notifier gmap_notifier;
a3508fbe 221static struct gmap_notifier vsie_gmap_notifier;
78f26131 222debug_info_t *kvm_s390_dbf;
9d8d5786 223
b0c632db 224/* Section: not file related */
13a34e06 225int kvm_arch_hardware_enable(void)
b0c632db
HC		 226{
227 /* every s390 is virtualization enabled ;-) */
10474ae8 228 return 0;
b0c632db
HC		 229}
230
f257d6dc
SC		 231int kvm_arch_check_processor_compat(void)
232{
233 return 0;
234}
235
414d3b07
MS		 236static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
237 unsigned long end);
2c70fe44 238
1575767e
DH		 239static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
240{
241 u8 delta_idx = 0;
242
243 /*
244 * The TOD jumps by delta, we have to compensate this by adding
245 * -delta to the epoch.
246 */
247 delta = -delta;
248
249 /* sign-extension - we're adding to signed values below */
250 if ((s64)delta < 0)
251 delta_idx = -1;
252
253 scb->epoch += delta;
254 if (scb->ecd & ECD_MEF) {
255 scb->epdx += delta_idx;
256 if (scb->epoch < delta)
257 scb->epdx += 1;
258 }
259}
260
fdf03650
FZ		 261/*
262 * This callback is executed during stop_machine(). All CPUs are therefore
263 * temporarily stopped. In order not to change guest behavior, we have to
264 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
265 * so a CPU won't be stopped while calculating with the epoch.
266 */
267static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
268 void *v)
269{
270 struct kvm *kvm;
271 struct kvm_vcpu *vcpu;
272 int i;
273 unsigned long long *delta = v;
274
275 list_for_each_entry(kvm, &vm_list, vm_list) {
fdf03650 276 kvm_for_each_vcpu(i, vcpu, kvm) {
1575767e
DH		 277 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
278 if (i == 0) {
279 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
280 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
281 }
db0758b2
DH		 282 if (vcpu->arch.cputm_enabled)
283 vcpu->arch.cputm_start += *delta;
91473b48 284 if (vcpu->arch.vsie_block)
1575767e
DH		 285 kvm_clock_sync_scb(vcpu->arch.vsie_block,
286 *delta);
fdf03650
FZ		 287 }
288 }
289 return NOTIFY_OK;
290}
291
292static struct notifier_block kvm_clock_notifier = {
293 .notifier_call = kvm_clock_sync,
294};
295
b0c632db
HC		 296int kvm_arch_hardware_setup(void)
297{
2c70fe44 298 gmap_notifier.notifier_call = kvm_gmap_notifier;
b2d73b2a 299 gmap_register_pte_notifier(&gmap_notifier);
a3508fbe
DH		 300 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
301 gmap_register_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 302 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
303 &kvm_clock_notifier);
b0c632db
HC		 304 return 0;
305}
306
307void kvm_arch_hardware_unsetup(void)
308{
b2d73b2a 309 gmap_unregister_pte_notifier(&gmap_notifier);
a3508fbe 310 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
fdf03650
FZ		 311 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
312 &kvm_clock_notifier);
b0c632db
HC		 313}
314
22be5a13
DH		 315static void allow_cpu_feat(unsigned long nr)
316{
317 set_bit_inv(nr, kvm_s390_available_cpu_feat);
318}
319
0a763c78
DH		 320static inline int plo_test_bit(unsigned char nr)
321{
322 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
d051ae53 323 int cc;
0a763c78
DH		 324
325 asm volatile(
326 /* Parameter registers are ignored for "test bit" */
327 " plo 0,0,0,0(0)\n"
328 " ipm %0\n"
329 " srl %0,28\n"
330 : "=d" (cc)
331 : "d" (r0)
332 : "cc");
333 return cc == 0;
334}
335
d0dea733 336static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
d6681397
CB		 337{
338 register unsigned long r0 asm("0") = 0; /* query function */
339 register unsigned long r1 asm("1") = (unsigned long) query;
340
341 asm volatile(
342 /* Parameter regs are ignored */
343 " .insn rrf,%[opc] << 16,2,4,6,0\n"
b1c41ac3 344 :
d6681397 345 : "d" (r0), "a" (r1), [opc] "i" (opcode)
b1c41ac3 346 : "cc", "memory");
d6681397
CB		 347}
348
173aec2d 349#define INSN_SORTL 0xb938
4f45b90e 350#define INSN_DFLTCC 0xb939
173aec2d 351
22be5a13
DH		 352static void kvm_s390_cpu_feat_init(void)
353{
0a763c78
DH		 354 int i;
355
356 for (i = 0; i < 256; ++i) {
357 if (plo_test_bit(i))
358 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
359 }
360
361 if (test_facility(28)) /* TOD-clock steering */
221bb8a4
LT		 362 ptff(kvm_s390_available_subfunc.ptff,
363 sizeof(kvm_s390_available_subfunc.ptff),
364 PTFF_QAF);
0a763c78
DH		 365
366 if (test_facility(17)) { /* MSA */
69c0e360
MS		 367 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
368 kvm_s390_available_subfunc.kmac);
369 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
370 kvm_s390_available_subfunc.kmc);
371 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
372 kvm_s390_available_subfunc.km);
373 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
374 kvm_s390_available_subfunc.kimd);
375 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
376 kvm_s390_available_subfunc.klmd);
0a763c78
DH		 377 }
378 if (test_facility(76)) /* MSA3 */
69c0e360
MS		 379 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
380 kvm_s390_available_subfunc.pckmo);
0a763c78 381 if (test_facility(77)) { /* MSA4 */
69c0e360
MS		 382 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
383 kvm_s390_available_subfunc.kmctr);
384 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
385 kvm_s390_available_subfunc.kmf);
386 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
387 kvm_s390_available_subfunc.kmo);
388 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
389 kvm_s390_available_subfunc.pcc);
0a763c78
DH		 390 }
391 if (test_facility(57)) /* MSA5 */
985a9d20 392 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
69c0e360 393 kvm_s390_available_subfunc.ppno);
0a763c78 394
e000b8e0
JH		 395 if (test_facility(146)) /* MSA8 */
396 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
397 kvm_s390_available_subfunc.kma);
398
13209ad0
CB		 399 if (test_facility(155)) /* MSA9 */
400 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
401 kvm_s390_available_subfunc.kdsa);
402
173aec2d
CB		 403 if (test_facility(150)) /* SORTL */
404 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
405
4f45b90e
CB		 406 if (test_facility(151)) /* DFLTCC */
407 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
408
22be5a13
DH		 409 if (MACHINE_HAS_ESOP)
410 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
a3508fbe
DH		 411 /*
412 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
413 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
414 */
415 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
a411edf1 416 !test_facility(3) || !nested)
a3508fbe
DH		 417 return;
418 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
19c439b5
DH		 419 if (sclp.has_64bscao)
420 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
0615a326
DH		 421 if (sclp.has_siif)
422 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
77d18f6d
DH		 423 if (sclp.has_gpere)
424 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
a1b7b9b2
DH		 425 if (sclp.has_gsls)
426 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
5630a8e8
DH		 427 if (sclp.has_ib)
428 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
13ee3f67
DH		 429 if (sclp.has_cei)
430 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
7fd7f39d
DH		 431 if (sclp.has_ibs)
432 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
730cd632
FA		 433 if (sclp.has_kss)
434 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
5d3876a8
DH		 435 /*
436 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
437 * all skey handling functions read/set the skey from the PGSTE
438 * instead of the real storage key.
439 *
440 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
441 * pages being detected as preserved although they are resident.
442 *
443 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
444 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
445 *
446 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
447 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
448 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
449 *
450 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
451 * cannot easily shadow the SCA because of the ipte lock.
452 */
22be5a13
DH		 453}
454
b0c632db
HC		 455int kvm_arch_init(void *opaque)
456{
f76f6371 457 int rc = -ENOMEM;
308c3e66 458
78f26131
CB		 459 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
460 if (!kvm_s390_dbf)
461 return -ENOMEM;
462
f76f6371
JF		 463 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view))
464 goto out;
78f26131 465
22be5a13
DH		 466 kvm_s390_cpu_feat_init();
467
84877d93 468 /* Register floating interrupt controller interface. */
308c3e66
MM		 469 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
470 if (rc) {
8d43d570 471 pr_err("A FLIC registration call failed with rc=%d\n", rc);
f76f6371 472 goto out;
308c3e66 473 }
b1d1e76e
MM		 474
475 rc = kvm_s390_gib_init(GAL_ISC);
476 if (rc)
f76f6371 477 goto out;
b1d1e76e 478
308c3e66
MM		 479 return 0;
480
f76f6371
JF		 481out:
482 kvm_arch_exit();
308c3e66 483 return rc;
b0c632db
HC		 484}
485
78f26131
CB		 486void kvm_arch_exit(void)
487{
1282c21e 488 kvm_s390_gib_destroy();
78f26131
CB		 489 debug_unregister(kvm_s390_dbf);
490}
491
b0c632db
HC		 492/* Section: device related */
493long kvm_arch_dev_ioctl(struct file *filp,
494 unsigned int ioctl, unsigned long arg)
495{
496 if (ioctl == KVM_S390_ENABLE_SIE)
497 return s390_enable_sie();
498 return -EINVAL;
499}
500
784aa3d7 501int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 502{
d7b0b5eb
CO		 503 int r;
504
2bd0ac4e 505 switch (ext) {
d7b0b5eb 506 case KVM_CAP_S390_PSW:
b6cf8788 507 case KVM_CAP_S390_GMAP:
52e16b18 508 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 509#ifdef CONFIG_KVM_S390_UCONTROL
510 case KVM_CAP_S390_UCONTROL:
511#endif
3c038e6b 512 case KVM_CAP_ASYNC_PF:
60b413c9 513 case KVM_CAP_SYNC_REGS:
14eebd91 514 case KVM_CAP_ONE_REG:
d6712df9 515 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 516 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 517 case KVM_CAP_IOEVENTFD:
c05c4186 518 case KVM_CAP_DEVICE_CTRL:
78599d90 519 case KVM_CAP_S390_IRQCHIP:
f2061656 520 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 521 case KVM_CAP_MP_STATE:
460df4c1 522 case KVM_CAP_IMMEDIATE_EXIT:
47b43c52 523 case KVM_CAP_S390_INJECT_IRQ:
2444b352 524 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 525 case KVM_CAP_S390_USER_STSI:
30ee2a98 526 case KVM_CAP_S390_SKEYS:
816c7667 527 case KVM_CAP_S390_IRQ_STATE:
6502a34c 528 case KVM_CAP_S390_USER_INSTR0:
4036e387 529 case KVM_CAP_S390_CMMA_MIGRATION:
47a4693e 530 case KVM_CAP_S390_AIS:
da9a1446 531 case KVM_CAP_S390_AIS_MIGRATION:
7de3f142 532 case KVM_CAP_S390_VCPU_RESETS:
d7b0b5eb
CO		 533 r = 1;
534 break;
a4499382
JF		 535 case KVM_CAP_S390_HPAGE_1M:
536 r = 0;
40ebdb8e 537 if (hpage && !kvm_is_ucontrol(kvm))
a4499382
JF		 538 r = 1;
539 break;
41408c28
TH		 540 case KVM_CAP_S390_MEM_OP:
541 r = MEM_OP_MAX_SIZE;
542 break;
e726b1bd
CB		 543 case KVM_CAP_NR_VCPUS:
544 case KVM_CAP_MAX_VCPUS:
a86cb413 545 case KVM_CAP_MAX_VCPU_ID:
76a6dd72 546 r = KVM_S390_BSCA_CPU_SLOTS;
a6940674
DH		 547 if (!kvm_s390_use_sca_entries())
548 r = KVM_MAX_VCPUS;
549 else if (sclp.has_esca && sclp.has_64bscao)
76a6dd72 550 r = KVM_S390_ESCA_CPU_SLOTS;
e726b1bd 551 break;
1526bf9c 552 case KVM_CAP_S390_COW:
abf09bed 553 r = MACHINE_HAS_ESOP;
1526bf9c 554 break;
68c55750
EF		 555 case KVM_CAP_S390_VECTOR_REGISTERS:
556 r = MACHINE_HAS_VX;
557 break;
c6e5f166
FZ		 558 case KVM_CAP_S390_RI:
559 r = test_facility(64);
560 break;
4e0b1ab7
FZ		 561 case KVM_CAP_S390_GS:
562 r = test_facility(133);
563 break;
35b3fde6
CB		 564 case KVM_CAP_S390_BPB:
565 r = test_facility(82);
566 break;
2bd0ac4e 567 default:
d7b0b5eb 568 r = 0;
2bd0ac4e 569 }
d7b0b5eb 570 return r;
b0c632db
HC		 571}
572
15f36ebd 573static void kvm_s390_sync_dirty_log(struct kvm *kvm,
0959e168 574 struct kvm_memory_slot *memslot)
15f36ebd 575{
0959e168 576 int i;
15f36ebd 577 gfn_t cur_gfn, last_gfn;
0959e168 578 unsigned long gaddr, vmaddr;
15f36ebd 579 struct gmap *gmap = kvm->arch.gmap;
0959e168 580 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
15f36ebd 581
0959e168
JF		 582 /* Loop over all guest segments */
583 cur_gfn = memslot->base_gfn;
15f36ebd 584 last_gfn = memslot->base_gfn + memslot->npages;
0959e168
JF		 585 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
586 gaddr = gfn_to_gpa(cur_gfn);
587 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
588 if (kvm_is_error_hva(vmaddr))
589 continue;
590
591 bitmap_zero(bitmap, _PAGE_ENTRIES);
592 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
593 for (i = 0; i < _PAGE_ENTRIES; i++) {
594 if (test_bit(i, bitmap))
595 mark_page_dirty(kvm, cur_gfn + i);
596 }
15f36ebd 597
1763f8d0
CB		 598 if (fatal_signal_pending(current))
599 return;
70c88a00 600 cond_resched();
15f36ebd 601 }
15f36ebd
JH		 602}
603
b0c632db 604/* Section: vm related */
a6e2f683
ED		 605static void sca_del_vcpu(struct kvm_vcpu *vcpu);
606
b0c632db
HC		 607/*
608 * Get (and clear) the dirty memory log for a memory slot.
609 */
610int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
611 struct kvm_dirty_log *log)
612{
15f36ebd
JH		 613 int r;
614 unsigned long n;
9f6b8029 615 struct kvm_memslots *slots;
15f36ebd
JH		 616 struct kvm_memory_slot *memslot;
617 int is_dirty = 0;
618
e1e8a962
JF		 619 if (kvm_is_ucontrol(kvm))
620 return -EINVAL;
621
15f36ebd
JH		 622 mutex_lock(&kvm->slots_lock);
623
624 r = -EINVAL;
625 if (log->slot >= KVM_USER_MEM_SLOTS)
626 goto out;
627
9f6b8029
PB		 628 slots = kvm_memslots(kvm);
629 memslot = id_to_memslot(slots, log->slot);
15f36ebd
JH		 630 r = -ENOENT;
631 if (!memslot->dirty_bitmap)
632 goto out;
633
634 kvm_s390_sync_dirty_log(kvm, memslot);
635 r = kvm_get_dirty_log(kvm, log, &is_dirty);
636 if (r)
637 goto out;
638
639 /* Clear the dirty log */
640 if (is_dirty) {
641 n = kvm_dirty_bitmap_bytes(memslot);
642 memset(memslot->dirty_bitmap, 0, n);
643 }
644 r = 0;
645out:
646 mutex_unlock(&kvm->slots_lock);
647 return r;
b0c632db
HC		 648}
649
6502a34c
DH		 650static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
651{
652 unsigned int i;
653 struct kvm_vcpu *vcpu;
654
655 kvm_for_each_vcpu(i, vcpu, kvm) {
656 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
657 }
658}
659
e5d83c74 660int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
d938dc55
CH		 661{
662 int r;
663
664 if (cap->flags)
665 return -EINVAL;
666
667 switch (cap->cap) {
84223598 668 case KVM_CAP_S390_IRQCHIP:
c92ea7b9 669 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
84223598
CH		 670 kvm->arch.use_irqchip = 1;
671 r = 0;
672 break;
2444b352 673 case KVM_CAP_S390_USER_SIGP:
c92ea7b9 674 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
2444b352
DH		 675 kvm->arch.user_sigp = 1;
676 r = 0;
677 break;
68c55750 678 case KVM_CAP_S390_VECTOR_REGISTERS:
5967c17b 679 mutex_lock(&kvm->lock);
a03825bb 680 if (kvm->created_vcpus) {
5967c17b
DH		 681 r = -EBUSY;
682 } else if (MACHINE_HAS_VX) {
c54f0d6a
DH		 683 set_kvm_facility(kvm->arch.model.fac_mask, 129);
684 set_kvm_facility(kvm->arch.model.fac_list, 129);
2f87d942
GH		 685 if (test_facility(134)) {
686 set_kvm_facility(kvm->arch.model.fac_mask, 134);
687 set_kvm_facility(kvm->arch.model.fac_list, 134);
688 }
53743aa7
MS		 689 if (test_facility(135)) {
690 set_kvm_facility(kvm->arch.model.fac_mask, 135);
691 set_kvm_facility(kvm->arch.model.fac_list, 135);
692 }
7832e91c
CB		 693 if (test_facility(148)) {
694 set_kvm_facility(kvm->arch.model.fac_mask, 148);
695 set_kvm_facility(kvm->arch.model.fac_list, 148);
696 }
d5cb6ab1
CB		 697 if (test_facility(152)) {
698 set_kvm_facility(kvm->arch.model.fac_mask, 152);
699 set_kvm_facility(kvm->arch.model.fac_list, 152);
700 }
18280d8b
MM		 701 r = 0;
702 } else
703 r = -EINVAL;
5967c17b 704 mutex_unlock(&kvm->lock);
c92ea7b9
CB		 705 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
706 r ? "(not available)" : "(success)");
68c55750 707 break;
c6e5f166
FZ		 708 case KVM_CAP_S390_RI:
709 r = -EINVAL;
710 mutex_lock(&kvm->lock);
a03825bb 711 if (kvm->created_vcpus) {
c6e5f166
FZ		 712 r = -EBUSY;
713 } else if (test_facility(64)) {
c54f0d6a
DH		 714 set_kvm_facility(kvm->arch.model.fac_mask, 64);
715 set_kvm_facility(kvm->arch.model.fac_list, 64);
c6e5f166
FZ		 716 r = 0;
717 }
718 mutex_unlock(&kvm->lock);
719 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
720 r ? "(not available)" : "(success)");
721 break;
47a4693e
YMZ		 722 case KVM_CAP_S390_AIS:
723 mutex_lock(&kvm->lock);
724 if (kvm->created_vcpus) {
725 r = -EBUSY;
726 } else {
727 set_kvm_facility(kvm->arch.model.fac_mask, 72);
728 set_kvm_facility(kvm->arch.model.fac_list, 72);
47a4693e
YMZ		 729 r = 0;
730 }
731 mutex_unlock(&kvm->lock);
732 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
733 r ? "(not available)" : "(success)");
734 break;
4e0b1ab7
FZ		 735 case KVM_CAP_S390_GS:
736 r = -EINVAL;
737 mutex_lock(&kvm->lock);
241e3ec0 738 if (kvm->created_vcpus) {
4e0b1ab7
FZ		 739 r = -EBUSY;
740 } else if (test_facility(133)) {
741 set_kvm_facility(kvm->arch.model.fac_mask, 133);
742 set_kvm_facility(kvm->arch.model.fac_list, 133);
743 r = 0;
744 }
745 mutex_unlock(&kvm->lock);
746 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
747 r ? "(not available)" : "(success)");
748 break;
a4499382
JF		 749 case KVM_CAP_S390_HPAGE_1M:
750 mutex_lock(&kvm->lock);
751 if (kvm->created_vcpus)
752 r = -EBUSY;
40ebdb8e 753 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
a4499382
JF		 754 r = -EINVAL;
755 else {
756 r = 0;
df88f318 757 down_write(&kvm->mm->mmap_sem);
a4499382 758 kvm->mm->context.allow_gmap_hpage_1m = 1;
df88f318 759 up_write(&kvm->mm->mmap_sem);
a4499382
JF		 760 /*
761 * We might have to create fake 4k page
762 * tables. To avoid that the hardware works on
763 * stale PGSTEs, we emulate these instructions.
764 */
765 kvm->arch.use_skf = 0;
766 kvm->arch.use_pfmfi = 0;
767 }
768 mutex_unlock(&kvm->lock);
769 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
770 r ? "(not available)" : "(success)");
771 break;
e44fc8c9 772 case KVM_CAP_S390_USER_STSI:
c92ea7b9 773 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
e44fc8c9
ET		 774 kvm->arch.user_stsi = 1;
775 r = 0;
776 break;
6502a34c
DH		 777 case KVM_CAP_S390_USER_INSTR0:
778 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
779 kvm->arch.user_instr0 = 1;
780 icpt_operexc_on_all_vcpus(kvm);
781 r = 0;
782 break;
d938dc55
CH		 783 default:
784 r = -EINVAL;
785 break;
786 }
787 return r;
788}
789
8c0a7ce6
DD		 790static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
791{
792 int ret;
793
794 switch (attr->attr) {
795 case KVM_S390_VM_MEM_LIMIT_SIZE:
796 ret = 0;
c92ea7b9 797 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
a3a92c31
DD		 798 kvm->arch.mem_limit);
799 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
8c0a7ce6
DD		 800 ret = -EFAULT;
801 break;
802 default:
803 ret = -ENXIO;
804 break;
805 }
806 return ret;
807}
808
809static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 810{
811 int ret;
812 unsigned int idx;
813 switch (attr->attr) {
814 case KVM_S390_VM_MEM_ENABLE_CMMA:
f9cbd9b0 815 ret = -ENXIO;
c24cc9c8 816 if (!sclp.has_cmma)
e6db1d61
DD		 817 break;
818
c92ea7b9 819 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
4f718eab 820 mutex_lock(&kvm->lock);
a4499382
JF		 821 if (kvm->created_vcpus)
822 ret = -EBUSY;
823 else if (kvm->mm->context.allow_gmap_hpage_1m)
824 ret = -EINVAL;
825 else {
4f718eab 826 kvm->arch.use_cmma = 1;
c9f0a2b8
JF		 827 /* Not compatible with cmma. */
828 kvm->arch.use_pfmfi = 0;
4f718eab
DD		 829 ret = 0;
830 }
831 mutex_unlock(&kvm->lock);
832 break;
833 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 834 ret = -ENXIO;
835 if (!sclp.has_cmma)
836 break;
c3489155
DD		 837 ret = -EINVAL;
838 if (!kvm->arch.use_cmma)
839 break;
840
c92ea7b9 841 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
4f718eab
DD		 842 mutex_lock(&kvm->lock);
843 idx = srcu_read_lock(&kvm->srcu);
a13cff31 844 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 845 srcu_read_unlock(&kvm->srcu, idx);
846 mutex_unlock(&kvm->lock);
847 ret = 0;
848 break;
8c0a7ce6
DD		 849 case KVM_S390_VM_MEM_LIMIT_SIZE: {
850 unsigned long new_limit;
851
852 if (kvm_is_ucontrol(kvm))
853 return -EINVAL;
854
855 if (get_user(new_limit, (u64 __user *)attr->addr))
856 return -EFAULT;
857
a3a92c31
DD		 858 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
859 new_limit > kvm->arch.mem_limit)
8c0a7ce6
DD		 860 return -E2BIG;
861
a3a92c31
DD		 862 if (!new_limit)
863 return -EINVAL;
864
6ea427bb 865 /* gmap_create takes last usable address */
a3a92c31
DD		 866 if (new_limit != KVM_S390_NO_MEM_LIMIT)
867 new_limit -= 1;
868
8c0a7ce6
DD		 869 ret = -EBUSY;
870 mutex_lock(&kvm->lock);
a03825bb 871 if (!kvm->created_vcpus) {
6ea427bb
MS		 872 /* gmap_create will round the limit up */
873 struct gmap *new = gmap_create(current->mm, new_limit);
8c0a7ce6
DD		 874
875 if (!new) {
876 ret = -ENOMEM;
877 } else {
6ea427bb 878 gmap_remove(kvm->arch.gmap);
8c0a7ce6
DD		 879 new->private = kvm;
880 kvm->arch.gmap = new;
881 ret = 0;
882 }
883 }
884 mutex_unlock(&kvm->lock);
a3a92c31
DD		 885 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
886 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
887 (void *) kvm->arch.gmap->asce);
8c0a7ce6
DD		 888 break;
889 }
4f718eab
DD		 890 default:
891 ret = -ENXIO;
892 break;
893 }
894 return ret;
895}
896
a374e892
TK		 897static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
898
20c922f0 899void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
a374e892
TK		 900{
901 struct kvm_vcpu *vcpu;
902 int i;
903
20c922f0
TK		 904 kvm_s390_vcpu_block_all(kvm);
905
3194cdb7 906 kvm_for_each_vcpu(i, vcpu, kvm) {
20c922f0 907 kvm_s390_vcpu_crypto_setup(vcpu);
3194cdb7
DH		 908 /* recreate the shadow crycb by leaving the VSIE handler */
909 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
910 }
20c922f0
TK		 911
912 kvm_s390_vcpu_unblock_all(kvm);
913}
914
915static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
916{
a374e892
TK		 917 mutex_lock(&kvm->lock);
918 switch (attr->attr) {
919 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
8e41bd54
CB		 920 if (!test_kvm_facility(kvm, 76)) {
921 mutex_unlock(&kvm->lock);
37940fb0 922 return -EINVAL;
8e41bd54 923 }
a374e892
TK		 924 get_random_bytes(
925 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
926 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
927 kvm->arch.crypto.aes_kw = 1;
c92ea7b9 928 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
a374e892
TK		 929 break;
930 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
8e41bd54
CB		 931 if (!test_kvm_facility(kvm, 76)) {
932 mutex_unlock(&kvm->lock);
37940fb0 933 return -EINVAL;
8e41bd54 934 }
a374e892
TK		 935 get_random_bytes(
936 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
937 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
938 kvm->arch.crypto.dea_kw = 1;
c92ea7b9 939 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
a374e892
TK		 940 break;
941 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
8e41bd54
CB		 942 if (!test_kvm_facility(kvm, 76)) {
943 mutex_unlock(&kvm->lock);
37940fb0 944 return -EINVAL;
8e41bd54 945 }
a374e892
TK		 946 kvm->arch.crypto.aes_kw = 0;
947 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
948 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
c92ea7b9 949 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
a374e892
TK		 950 break;
951 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
8e41bd54
CB		 952 if (!test_kvm_facility(kvm, 76)) {
953 mutex_unlock(&kvm->lock);
37940fb0 954 return -EINVAL;
8e41bd54 955 }
a374e892
TK		 956 kvm->arch.crypto.dea_kw = 0;
957 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
958 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
c92ea7b9 959 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
a374e892 960 break;
37940fb0
TK		 961 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
962 if (!ap_instructions_available()) {
963 mutex_unlock(&kvm->lock);
964 return -EOPNOTSUPP;
965 }
966 kvm->arch.crypto.apie = 1;
967 break;
968 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
969 if (!ap_instructions_available()) {
970 mutex_unlock(&kvm->lock);
971 return -EOPNOTSUPP;
972 }
973 kvm->arch.crypto.apie = 0;
974 break;
a374e892
TK		 975 default:
976 mutex_unlock(&kvm->lock);
977 return -ENXIO;
978 }
979
20c922f0 980 kvm_s390_vcpu_crypto_reset_all(kvm);
a374e892
TK		 981 mutex_unlock(&kvm->lock);
982 return 0;
983}
984
190df4a2
CI		 985static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
986{
987 int cx;
988 struct kvm_vcpu *vcpu;
989
990 kvm_for_each_vcpu(cx, vcpu, kvm)
991 kvm_s390_sync_request(req, vcpu);
992}
993
994/*
995 * Must be called with kvm->srcu held to avoid races on memslots, and with
1de1ea7e 996 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
190df4a2
CI		 997 */
998static int kvm_s390_vm_start_migration(struct kvm *kvm)
999{
190df4a2 1000 struct kvm_memory_slot *ms;
190df4a2 1001 struct kvm_memslots *slots;
afdad616 1002 unsigned long ram_pages = 0;
190df4a2
CI		 1003 int slotnr;
1004
1005 /* migration mode already enabled */
afdad616 1006 if (kvm->arch.migration_mode)
190df4a2 1007 return 0;
190df4a2
CI		 1008 slots = kvm_memslots(kvm);
1009 if (!slots || !slots->used_slots)
1010 return -EINVAL;
1011
afdad616
CI		 1012 if (!kvm->arch.use_cmma) {
1013 kvm->arch.migration_mode = 1;
1014 return 0;
1015 }
1016 /* mark all the pages in active slots as dirty */
1017 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1018 ms = slots->memslots + slotnr;
13a17cc0
IM		 1019 if (!ms->dirty_bitmap)
1020 return -EINVAL;
190df4a2 1021 /*
afdad616
CI		 1022 * The second half of the bitmap is only used on x86,
1023 * and would be wasted otherwise, so we put it to good
1024 * use here to keep track of the state of the storage
1025 * attributes.
190df4a2 1026 */
afdad616
CI		 1027 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1028 ram_pages += ms->npages;
190df4a2 1029 }
afdad616
CI		 1030 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1031 kvm->arch.migration_mode = 1;
1032 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
190df4a2
CI		 1033 return 0;
1034}
1035
1036/*
1de1ea7e 1037 * Must be called with kvm->slots_lock to avoid races with ourselves and
190df4a2
CI		 1038 * kvm_s390_vm_start_migration.
1039 */
1040static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1041{
190df4a2 1042 /* migration mode already disabled */
afdad616 1043 if (!kvm->arch.migration_mode)
190df4a2 1044 return 0;
afdad616
CI		 1045 kvm->arch.migration_mode = 0;
1046 if (kvm->arch.use_cmma)
190df4a2 1047 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
190df4a2
CI		 1048 return 0;
1049}
1050
1051static int kvm_s390_vm_set_migration(struct kvm *kvm,
1052 struct kvm_device_attr *attr)
1053{
1de1ea7e 1054 int res = -ENXIO;
190df4a2 1055
1de1ea7e 1056 mutex_lock(&kvm->slots_lock);
190df4a2
CI		 1057 switch (attr->attr) {
1058 case KVM_S390_VM_MIGRATION_START:
190df4a2 1059 res = kvm_s390_vm_start_migration(kvm);
190df4a2
CI		 1060 break;
1061 case KVM_S390_VM_MIGRATION_STOP:
1062 res = kvm_s390_vm_stop_migration(kvm);
1063 break;
1064 default:
1065 break;
1066 }
1de1ea7e 1067 mutex_unlock(&kvm->slots_lock);
190df4a2
CI		 1068
1069 return res;
1070}
1071
1072static int kvm_s390_vm_get_migration(struct kvm *kvm,
1073 struct kvm_device_attr *attr)
1074{
afdad616 1075 u64 mig = kvm->arch.migration_mode;
190df4a2
CI		 1076
1077 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1078 return -ENXIO;
1079
1080 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1081 return -EFAULT;
1082 return 0;
1083}
1084
8fa1696e
CW		 1085static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1086{
1087 struct kvm_s390_vm_tod_clock gtod;
1088
1089 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
1090 return -EFAULT;
1091
0e7def5f 1092 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
8fa1696e 1093 return -EINVAL;
0e7def5f 1094 kvm_s390_set_tod_clock(kvm, &gtod);
8fa1696e
CW		 1095
1096 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1097 gtod.epoch_idx, gtod.tod);
1098
1099 return 0;
1100}
1101
72f25020
JH		 1102static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1103{
1104 u8 gtod_high;
1105
1106 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
1107 sizeof(gtod_high)))
1108 return -EFAULT;
1109
1110 if (gtod_high != 0)
1111 return -EINVAL;
58c383c6 1112 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1113
1114 return 0;
1115}
1116
1117static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1118{
0e7def5f 1119 struct kvm_s390_vm_tod_clock gtod = { 0 };
72f25020 1120
0e7def5f
DH		 1121 if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1122 sizeof(gtod.tod)))
72f25020
JH		 1123 return -EFAULT;
1124
0e7def5f
DH		 1125 kvm_s390_set_tod_clock(kvm, &gtod);
1126 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
72f25020
JH		 1127 return 0;
1128}
1129
1130static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1131{
1132 int ret;
1133
1134 if (attr->flags)
1135 return -EINVAL;
1136
1137 switch (attr->attr) {
8fa1696e
CW		 1138 case KVM_S390_VM_TOD_EXT:
1139 ret = kvm_s390_set_tod_ext(kvm, attr);
1140 break;
72f25020
JH		 1141 case KVM_S390_VM_TOD_HIGH:
1142 ret = kvm_s390_set_tod_high(kvm, attr);
1143 break;
1144 case KVM_S390_VM_TOD_LOW:
1145 ret = kvm_s390_set_tod_low(kvm, attr);
1146 break;
1147 default:
1148 ret = -ENXIO;
1149 break;
1150 }
1151 return ret;
1152}
1153
33d1b272
DH		 1154static void kvm_s390_get_tod_clock(struct kvm *kvm,
1155 struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 1156{
1157 struct kvm_s390_tod_clock_ext htod;
1158
1159 preempt_disable();
1160
1161 get_tod_clock_ext((char *)&htod);
1162
1163 gtod->tod = htod.tod + kvm->arch.epoch;
33d1b272
DH		 1164 gtod->epoch_idx = 0;
1165 if (test_kvm_facility(kvm, 139)) {
1166 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1167 if (gtod->tod < htod.tod)
1168 gtod->epoch_idx += 1;
1169 }
8fa1696e
CW		 1170
1171 preempt_enable();
1172}
1173
1174static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1175{
1176 struct kvm_s390_vm_tod_clock gtod;
1177
1178 memset(&gtod, 0, sizeof(gtod));
33d1b272 1179 kvm_s390_get_tod_clock(kvm, &gtod);
8fa1696e
CW		 1180 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1181 return -EFAULT;
1182
1183 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1184 gtod.epoch_idx, gtod.tod);
1185 return 0;
1186}
1187
72f25020
JH		 1188static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1189{
1190 u8 gtod_high = 0;
1191
1192 if (copy_to_user((void __user *)attr->addr, &gtod_high,
1193 sizeof(gtod_high)))
1194 return -EFAULT;
58c383c6 1195 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
72f25020
JH		 1196
1197 return 0;
1198}
1199
1200static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1201{
5a3d883a 1202 u64 gtod;
72f25020 1203
60417fcc 1204 gtod = kvm_s390_get_tod_clock_fast(kvm);
72f25020
JH		 1205 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1206 return -EFAULT;
58c383c6 1207 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
72f25020
JH		 1208
1209 return 0;
1210}
1211
1212static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1213{
1214 int ret;
1215
1216 if (attr->flags)
1217 return -EINVAL;
1218
1219 switch (attr->attr) {
8fa1696e
CW		 1220 case KVM_S390_VM_TOD_EXT:
1221 ret = kvm_s390_get_tod_ext(kvm, attr);
1222 break;
72f25020
JH		 1223 case KVM_S390_VM_TOD_HIGH:
1224 ret = kvm_s390_get_tod_high(kvm, attr);
1225 break;
1226 case KVM_S390_VM_TOD_LOW:
1227 ret = kvm_s390_get_tod_low(kvm, attr);
1228 break;
1229 default:
1230 ret = -ENXIO;
1231 break;
1232 }
1233 return ret;
1234}
1235
658b6eda
MM		 1236static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1237{
1238 struct kvm_s390_vm_cpu_processor *proc;
053dd230 1239 u16 lowest_ibc, unblocked_ibc;
658b6eda
MM		 1240 int ret = 0;
1241
1242 mutex_lock(&kvm->lock);
a03825bb 1243 if (kvm->created_vcpus) {
658b6eda
MM		 1244 ret = -EBUSY;
1245 goto out;
1246 }
1247 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1248 if (!proc) {
1249 ret = -ENOMEM;
1250 goto out;
1251 }
1252 if (!copy_from_user(proc, (void __user *)attr->addr,
1253 sizeof(*proc))) {
9bb0ec09 1254 kvm->arch.model.cpuid = proc->cpuid;
053dd230
DH		 1255 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1256 unblocked_ibc = sclp.ibc & 0xfff;
0487c44d 1257 if (lowest_ibc && proc->ibc) {
053dd230
DH		 1258 if (proc->ibc > unblocked_ibc)
1259 kvm->arch.model.ibc = unblocked_ibc;
1260 else if (proc->ibc < lowest_ibc)
1261 kvm->arch.model.ibc = lowest_ibc;
1262 else
1263 kvm->arch.model.ibc = proc->ibc;
1264 }
c54f0d6a 1265 memcpy(kvm->arch.model.fac_list, proc->fac_list,
658b6eda 1266 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1267 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1268 kvm->arch.model.ibc,
1269 kvm->arch.model.cpuid);
1270 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1271 kvm->arch.model.fac_list[0],
1272 kvm->arch.model.fac_list[1],
1273 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1274 } else
1275 ret = -EFAULT;
1276 kfree(proc);
1277out:
1278 mutex_unlock(&kvm->lock);
1279 return ret;
1280}
1281
15c9705f
DH		 1282static int kvm_s390_set_processor_feat(struct kvm *kvm,
1283 struct kvm_device_attr *attr)
1284{
1285 struct kvm_s390_vm_cpu_feat data;
15c9705f
DH		 1286
1287 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1288 return -EFAULT;
1289 if (!bitmap_subset((unsigned long *) data.feat,
1290 kvm_s390_available_cpu_feat,
1291 KVM_S390_VM_CPU_FEAT_NR_BITS))
1292 return -EINVAL;
1293
1294 mutex_lock(&kvm->lock);
2f8311c9
CB		 1295 if (kvm->created_vcpus) {
1296 mutex_unlock(&kvm->lock);
1297 return -EBUSY;
15c9705f 1298 }
2f8311c9
CB		 1299 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1300 KVM_S390_VM_CPU_FEAT_NR_BITS);
15c9705f 1301 mutex_unlock(&kvm->lock);
2f8311c9
CB		 1302 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1303 data.feat[0],
1304 data.feat[1],
1305 data.feat[2]);
1306 return 0;
15c9705f
DH		 1307}
1308
0a763c78
DH		 1309static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1310 struct kvm_device_attr *attr)
1311{
346fa2f8
CB		 1312 mutex_lock(&kvm->lock);
1313 if (kvm->created_vcpus) {
1314 mutex_unlock(&kvm->lock);
1315 return -EBUSY;
1316 }
1317
1318 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1319 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1320 mutex_unlock(&kvm->lock);
1321 return -EFAULT;
1322 }
1323 mutex_unlock(&kvm->lock);
1324
11ba5961
CB		 1325 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1326 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1327 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1328 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1329 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1330 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1331 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1332 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1333 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1334 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1335 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1336 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1337 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1338 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1339 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1340 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1341 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1342 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1343 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1344 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1345 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1346 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1347 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1348 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1349 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1350 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1351 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1352 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1353 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1354 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1355 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1356 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1357 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1358 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1359 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1360 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1361 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1362 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1363 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1364 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1365 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1366 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1367 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1368 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1369 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1370 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1371 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1372 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1373 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1374 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1375 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1376 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1377 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1378 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1379 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1380 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1381 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1382
346fa2f8 1383 return 0;
0a763c78
DH		 1384}
1385
658b6eda
MM		 1386static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1387{
1388 int ret = -ENXIO;
1389
1390 switch (attr->attr) {
1391 case KVM_S390_VM_CPU_PROCESSOR:
1392 ret = kvm_s390_set_processor(kvm, attr);
1393 break;
15c9705f
DH		 1394 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1395 ret = kvm_s390_set_processor_feat(kvm, attr);
1396 break;
0a763c78
DH		 1397 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1398 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1399 break;
658b6eda
MM		 1400 }
1401 return ret;
1402}
1403
1404static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1405{
1406 struct kvm_s390_vm_cpu_processor *proc;
1407 int ret = 0;
1408
1409 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1410 if (!proc) {
1411 ret = -ENOMEM;
1412 goto out;
1413 }
9bb0ec09 1414 proc->cpuid = kvm->arch.model.cpuid;
658b6eda 1415 proc->ibc = kvm->arch.model.ibc;
c54f0d6a
DH		 1416 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1417 S390_ARCH_FAC_LIST_SIZE_BYTE);
a8c39dd7
CB		 1418 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1419 kvm->arch.model.ibc,
1420 kvm->arch.model.cpuid);
1421 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1422 kvm->arch.model.fac_list[0],
1423 kvm->arch.model.fac_list[1],
1424 kvm->arch.model.fac_list[2]);
658b6eda
MM		 1425 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1426 ret = -EFAULT;
1427 kfree(proc);
1428out:
1429 return ret;
1430}
1431
1432static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1433{
1434 struct kvm_s390_vm_cpu_machine *mach;
1435 int ret = 0;
1436
1437 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1438 if (!mach) {
1439 ret = -ENOMEM;
1440 goto out;
1441 }
1442 get_cpu_id((struct cpuid *) &mach->cpuid);
37c5f6c8 1443 mach->ibc = sclp.ibc;
c54f0d6a 1444 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
981467c9 1445 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 1446 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
04478197 1447 sizeof(S390_lowcore.stfle_fac_list));
a8c39dd7
CB		 1448 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1449 kvm->arch.model.ibc,
1450 kvm->arch.model.cpuid);
1451 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1452 mach->fac_mask[0],
1453 mach->fac_mask[1],
1454 mach->fac_mask[2]);
1455 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1456 mach->fac_list[0],
1457 mach->fac_list[1],
1458 mach->fac_list[2]);
658b6eda
MM		 1459 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1460 ret = -EFAULT;
1461 kfree(mach);
1462out:
1463 return ret;
1464}
1465
15c9705f
DH		 1466static int kvm_s390_get_processor_feat(struct kvm *kvm,
1467 struct kvm_device_attr *attr)
1468{
1469 struct kvm_s390_vm_cpu_feat data;
1470
1471 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1472 KVM_S390_VM_CPU_FEAT_NR_BITS);
1473 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1474 return -EFAULT;
2f8311c9
CB		 1475 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1476 data.feat[0],
1477 data.feat[1],
1478 data.feat[2]);
15c9705f
DH		 1479 return 0;
1480}
1481
1482static int kvm_s390_get_machine_feat(struct kvm *kvm,
1483 struct kvm_device_attr *attr)
1484{
1485 struct kvm_s390_vm_cpu_feat data;
1486
1487 bitmap_copy((unsigned long *) data.feat,
1488 kvm_s390_available_cpu_feat,
1489 KVM_S390_VM_CPU_FEAT_NR_BITS);
1490 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1491 return -EFAULT;
2f8311c9
CB		 1492 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1493 data.feat[0],
1494 data.feat[1],
1495 data.feat[2]);
15c9705f
DH		 1496 return 0;
1497}
1498
0a763c78
DH		 1499static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1500 struct kvm_device_attr *attr)
1501{
346fa2f8
CB		 1502 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1503 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1504 return -EFAULT;
1505
11ba5961
CB		 1506 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1507 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1508 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1509 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1510 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1511 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1512 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1513 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1514 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1515 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1516 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1517 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1518 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1519 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1520 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1521 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1522 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1523 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1524 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1525 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1526 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1527 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1528 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1529 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1530 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1531 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1532 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1533 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1534 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1535 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1536 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1537 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1538 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1539 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1540 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1541 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1542 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1543 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1544 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1545 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1546 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1547 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1548 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1549 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
13209ad0
CB		 1550 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1552 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
173aec2d
CB		 1553 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1554 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1555 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1556 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
4f45b90e
CB		 1558 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1561 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1562 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
11ba5961 1563
346fa2f8 1564 return 0;
0a763c78
DH		 1565}
1566
1567static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1568 struct kvm_device_attr *attr)
1569{
1570 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1571 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1572 return -EFAULT;
11ba5961
CB		 1573
1574 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1575 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1576 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1577 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1578 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1579 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1580 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1581 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1582 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1583 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1584 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1585 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1586 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1587 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1588 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1589 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1590 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1591 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1592 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1593 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1594 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1595 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1596 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1597 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1598 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1599 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1600 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1601 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1602 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1603 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1604 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1605 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1606 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1607 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1608 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1609 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1610 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1611 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1612 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1613 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1614 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1615 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1616 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1617 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
13209ad0
CB		 1618 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1619 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1620 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
173aec2d
CB		 1621 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1622 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1623 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1624 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1625 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
4f45b90e
CB		 1626 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1627 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1628 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1629 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1630 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
11ba5961 1631
0a763c78
DH		 1632 return 0;
1633}
346fa2f8 1634
658b6eda
MM		 1635static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1636{
1637 int ret = -ENXIO;
1638
1639 switch (attr->attr) {
1640 case KVM_S390_VM_CPU_PROCESSOR:
1641 ret = kvm_s390_get_processor(kvm, attr);
1642 break;
1643 case KVM_S390_VM_CPU_MACHINE:
1644 ret = kvm_s390_get_machine(kvm, attr);
1645 break;
15c9705f
DH		 1646 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1647 ret = kvm_s390_get_processor_feat(kvm, attr);
1648 break;
1649 case KVM_S390_VM_CPU_MACHINE_FEAT:
1650 ret = kvm_s390_get_machine_feat(kvm, attr);
1651 break;
0a763c78
DH		 1652 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1653 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1654 break;
1655 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1656 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1657 break;
658b6eda
MM		 1658 }
1659 return ret;
1660}
1661
f2061656
DD		 1662static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1663{
1664 int ret;
1665
1666 switch (attr->group) {
4f718eab 1667 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 1668 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 1669 break;
72f25020
JH		 1670 case KVM_S390_VM_TOD:
1671 ret = kvm_s390_set_tod(kvm, attr);
1672 break;
658b6eda
MM		 1673 case KVM_S390_VM_CPU_MODEL:
1674 ret = kvm_s390_set_cpu_model(kvm, attr);
1675 break;
a374e892
TK		 1676 case KVM_S390_VM_CRYPTO:
1677 ret = kvm_s390_vm_set_crypto(kvm, attr);
1678 break;
190df4a2
CI		 1679 case KVM_S390_VM_MIGRATION:
1680 ret = kvm_s390_vm_set_migration(kvm, attr);
1681 break;
f2061656
DD		 1682 default:
1683 ret = -ENXIO;
1684 break;
1685 }
1686
1687 return ret;
1688}
1689
1690static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1691{
8c0a7ce6
DD		 1692 int ret;
1693
1694 switch (attr->group) {
1695 case KVM_S390_VM_MEM_CTRL:
1696 ret = kvm_s390_get_mem_control(kvm, attr);
1697 break;
72f25020
JH		 1698 case KVM_S390_VM_TOD:
1699 ret = kvm_s390_get_tod(kvm, attr);
1700 break;
658b6eda
MM		 1701 case KVM_S390_VM_CPU_MODEL:
1702 ret = kvm_s390_get_cpu_model(kvm, attr);
1703 break;
190df4a2
CI		 1704 case KVM_S390_VM_MIGRATION:
1705 ret = kvm_s390_vm_get_migration(kvm, attr);
1706 break;
8c0a7ce6
DD		 1707 default:
1708 ret = -ENXIO;
1709 break;
1710 }
1711
1712 return ret;
f2061656
DD		 1713}
1714
1715static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1716{
1717 int ret;
1718
1719 switch (attr->group) {
4f718eab
DD		 1720 case KVM_S390_VM_MEM_CTRL:
1721 switch (attr->attr) {
1722 case KVM_S390_VM_MEM_ENABLE_CMMA:
1723 case KVM_S390_VM_MEM_CLR_CMMA:
f9cbd9b0
DH		 1724 ret = sclp.has_cmma ? 0 : -ENXIO;
1725 break;
8c0a7ce6 1726 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 1727 ret = 0;
1728 break;
1729 default:
1730 ret = -ENXIO;
1731 break;
1732 }
1733 break;
72f25020
JH		 1734 case KVM_S390_VM_TOD:
1735 switch (attr->attr) {
1736 case KVM_S390_VM_TOD_LOW:
1737 case KVM_S390_VM_TOD_HIGH:
1738 ret = 0;
1739 break;
1740 default:
1741 ret = -ENXIO;
1742 break;
1743 }
1744 break;
658b6eda
MM		 1745 case KVM_S390_VM_CPU_MODEL:
1746 switch (attr->attr) {
1747 case KVM_S390_VM_CPU_PROCESSOR:
1748 case KVM_S390_VM_CPU_MACHINE:
15c9705f
DH		 1749 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1750 case KVM_S390_VM_CPU_MACHINE_FEAT:
0a763c78 1751 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
346fa2f8 1752 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
658b6eda
MM		 1753 ret = 0;
1754 break;
1755 default:
1756 ret = -ENXIO;
1757 break;
1758 }
1759 break;
a374e892
TK		 1760 case KVM_S390_VM_CRYPTO:
1761 switch (attr->attr) {
1762 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1763 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1764 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1765 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1766 ret = 0;
1767 break;
37940fb0
TK		 1768 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1769 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1770 ret = ap_instructions_available() ? 0 : -ENXIO;
1771 break;
a374e892
TK		 1772 default:
1773 ret = -ENXIO;
1774 break;
1775 }
1776 break;
190df4a2
CI		 1777 case KVM_S390_VM_MIGRATION:
1778 ret = 0;
1779 break;
f2061656
DD		 1780 default:
1781 ret = -ENXIO;
1782 break;
1783 }
1784
1785 return ret;
1786}
1787
30ee2a98
JH		 1788static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1789{
1790 uint8_t *keys;
1791 uint64_t hva;
4f899147 1792 int srcu_idx, i, r = 0;
30ee2a98
JH		 1793
1794 if (args->flags != 0)
1795 return -EINVAL;
1796
1797 /* Is this guest using storage keys? */
55531b74 1798 if (!mm_uses_skeys(current->mm))
30ee2a98
JH		 1799 return KVM_S390_GET_SKEYS_NONE;
1800
1801 /* Enforce sane limit on memory allocation */
1802 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1803 return -EINVAL;
1804
752ade68 1805 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1806 if (!keys)
1807 return -ENOMEM;
1808
d3ed1cee 1809 down_read(&current->mm->mmap_sem);
4f899147 1810 srcu_idx = srcu_read_lock(&kvm->srcu);
30ee2a98
JH		 1811 for (i = 0; i < args->count; i++) {
1812 hva = gfn_to_hva(kvm, args->start_gfn + i);
1813 if (kvm_is_error_hva(hva)) {
1814 r = -EFAULT;
d3ed1cee 1815 break;
30ee2a98
JH		 1816 }
1817
154c8c19
DH		 1818 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1819 if (r)
d3ed1cee 1820 break;
30ee2a98 1821 }
4f899147 1822 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee
MS		 1823 up_read(&current->mm->mmap_sem);
1824
1825 if (!r) {
1826 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1827 sizeof(uint8_t) * args->count);
1828 if (r)
1829 r = -EFAULT;
30ee2a98
JH		 1830 }
1831
30ee2a98
JH		 1832 kvfree(keys);
1833 return r;
1834}
1835
1836static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1837{
1838 uint8_t *keys;
1839 uint64_t hva;
4f899147 1840 int srcu_idx, i, r = 0;
bd096f64 1841 bool unlocked;
30ee2a98
JH		 1842
1843 if (args->flags != 0)
1844 return -EINVAL;
1845
1846 /* Enforce sane limit on memory allocation */
1847 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1848 return -EINVAL;
1849
752ade68 1850 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
30ee2a98
JH		 1851 if (!keys)
1852 return -ENOMEM;
1853
1854 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1855 sizeof(uint8_t) * args->count);
1856 if (r) {
1857 r = -EFAULT;
1858 goto out;
1859 }
1860
1861 /* Enable storage key handling for the guest */
14d4a425
DD		 1862 r = s390_enable_skey();
1863 if (r)
1864 goto out;
30ee2a98 1865
bd096f64 1866 i = 0;
d3ed1cee 1867 down_read(&current->mm->mmap_sem);
4f899147 1868 srcu_idx = srcu_read_lock(&kvm->srcu);
bd096f64
JF		 1869 while (i < args->count) {
1870 unlocked = false;
30ee2a98
JH		 1871 hva = gfn_to_hva(kvm, args->start_gfn + i);
1872 if (kvm_is_error_hva(hva)) {
1873 r = -EFAULT;
d3ed1cee 1874 break;
30ee2a98
JH		 1875 }
1876
1877 /* Lowest order bit is reserved */
1878 if (keys[i] & 0x01) {
1879 r = -EINVAL;
d3ed1cee 1880 break;
30ee2a98
JH		 1881 }
1882
fe69eabf 1883 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
bd096f64
JF		 1884 if (r) {
1885 r = fixup_user_fault(current, current->mm, hva,
1886 FAULT_FLAG_WRITE, &unlocked);
1887 if (r)
1888 break;
1889 }
1890 if (!r)
1891 i++;
30ee2a98 1892 }
4f899147 1893 srcu_read_unlock(&kvm->srcu, srcu_idx);
d3ed1cee 1894 up_read(&current->mm->mmap_sem);
30ee2a98
JH		 1895out:
1896 kvfree(keys);
1897 return r;
1898}
1899
4036e387
CI		 1900/*
1901 * Base address and length must be sent at the start of each block, therefore
1902 * it's cheaper to send some clean data, as long as it's less than the size of
1903 * two longs.
1904 */
1905#define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1906/* for consistency */
1907#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1908
afdad616
CI		 1909/*
1910 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1911 * address falls in a hole. In that case the index of one of the memslots
1912 * bordering the hole is returned.
1913 */
1914static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1915{
1916 int start = 0, end = slots->used_slots;
1917 int slot = atomic_read(&slots->lru_slot);
1918 struct kvm_memory_slot *memslots = slots->memslots;
1919
1920 if (gfn >= memslots[slot].base_gfn &&
1921 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1922 return slot;
1923
1924 while (start < end) {
1925 slot = start + (end - start) / 2;
1926
1927 if (gfn >= memslots[slot].base_gfn)
1928 end = slot;
1929 else
1930 start = slot + 1;
1931 }
1932
1933 if (gfn >= memslots[start].base_gfn &&
1934 gfn < memslots[start].base_gfn + memslots[start].npages) {
1935 atomic_set(&slots->lru_slot, start);
1936 }
1937
1938 return start;
1939}
1940
1941static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1942 u8 *res, unsigned long bufsize)
1943{
1944 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1945
1946 args->count = 0;
1947 while (args->count < bufsize) {
1948 hva = gfn_to_hva(kvm, cur_gfn);
1949 /*
1950 * We return an error if the first value was invalid, but we
1951 * return successfully if at least one value was copied.
1952 */
1953 if (kvm_is_error_hva(hva))
1954 return args->count ? 0 : -EFAULT;
1955 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1956 pgstev = 0;
1957 res[args->count++] = (pgstev >> 24) & 0x43;
1958 cur_gfn++;
1959 }
1960
1961 return 0;
1962}
1963
1964static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1965 unsigned long cur_gfn)
1966{
1967 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1968 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1969 unsigned long ofs = cur_gfn - ms->base_gfn;
1970
1971 if (ms->base_gfn + ms->npages <= cur_gfn) {
1972 slotidx--;
1973 /* If we are above the highest slot, wrap around */
1974 if (slotidx < 0)
1975 slotidx = slots->used_slots - 1;
1976
1977 ms = slots->memslots + slotidx;
1978 ofs = 0;
1979 }
1980 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1981 while ((slotidx > 0) && (ofs >= ms->npages)) {
1982 slotidx--;
1983 ms = slots->memslots + slotidx;
1984 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1985 }
1986 return ms->base_gfn + ofs;
1987}
1988
1989static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1990 u8 *res, unsigned long bufsize)
1991{
1992 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1993 struct kvm_memslots *slots = kvm_memslots(kvm);
1994 struct kvm_memory_slot *ms;
1995
1996 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1997 ms = gfn_to_memslot(kvm, cur_gfn);
1998 args->count = 0;
1999 args->start_gfn = cur_gfn;
2000 if (!ms)
2001 return 0;
2002 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2003 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2004
2005 while (args->count < bufsize) {
2006 hva = gfn_to_hva(kvm, cur_gfn);
2007 if (kvm_is_error_hva(hva))
2008 return 0;
2009 /* Decrement only if we actually flipped the bit to 0 */
2010 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2011 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2012 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2013 pgstev = 0;
2014 /* Save the value */
2015 res[args->count++] = (pgstev >> 24) & 0x43;
2016 /* If the next bit is too far away, stop. */
2017 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2018 return 0;
2019 /* If we reached the previous "next", find the next one */
2020 if (cur_gfn == next_gfn)
2021 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2022 /* Reached the end of memory or of the buffer, stop */
2023 if ((next_gfn >= mem_end) ||
2024 (next_gfn - args->start_gfn >= bufsize))
2025 return 0;
2026 cur_gfn++;
2027 /* Reached the end of the current memslot, take the next one. */
2028 if (cur_gfn - ms->base_gfn >= ms->npages) {
2029 ms = gfn_to_memslot(kvm, cur_gfn);
2030 if (!ms)
2031 return 0;
2032 }
2033 }
2034 return 0;
2035}
2036
4036e387
CI		 2037/*
2038 * This function searches for the next page with dirty CMMA attributes, and
2039 * saves the attributes in the buffer up to either the end of the buffer or
2040 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2041 * no trailing clean bytes are saved.
2042 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2043 * output buffer will indicate 0 as length.
2044 */
2045static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2046 struct kvm_s390_cmma_log *args)
2047{
afdad616
CI		 2048 unsigned long bufsize;
2049 int srcu_idx, peek, ret;
2050 u8 *values;
4036e387 2051
afdad616 2052 if (!kvm->arch.use_cmma)
4036e387
CI		 2053 return -ENXIO;
2054 /* Invalid/unsupported flags were specified */
2055 if (args->flags & ~KVM_S390_CMMA_PEEK)
2056 return -EINVAL;
2057 /* Migration mode query, and we are not doing a migration */
2058 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
afdad616 2059 if (!peek && !kvm->arch.migration_mode)
4036e387
CI		 2060 return -EINVAL;
2061 /* CMMA is disabled or was not used, or the buffer has length zero */
2062 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
c9f0a2b8 2063 if (!bufsize || !kvm->mm->context.uses_cmm) {
4036e387
CI		 2064 memset(args, 0, sizeof(*args));
2065 return 0;
2066 }
afdad616
CI		 2067 /* We are not peeking, and there are no dirty pages */
2068 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2069 memset(args, 0, sizeof(*args));
2070 return 0;
4036e387
CI		 2071 }
2072
afdad616
CI		 2073 values = vmalloc(bufsize);
2074 if (!values)
4036e387
CI		 2075 return -ENOMEM;
2076
4036e387
CI		 2077 down_read(&kvm->mm->mmap_sem);
2078 srcu_idx = srcu_read_lock(&kvm->srcu);
afdad616
CI		 2079 if (peek)
2080 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2081 else
2082 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
4036e387
CI		 2083 srcu_read_unlock(&kvm->srcu, srcu_idx);
2084 up_read(&kvm->mm->mmap_sem);
4036e387 2085
afdad616
CI		 2086 if (kvm->arch.migration_mode)
2087 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2088 else
2089 args->remaining = 0;
4036e387 2090
afdad616
CI		 2091 if (copy_to_user((void __user *)args->values, values, args->count))
2092 ret = -EFAULT;
2093
2094 vfree(values);
2095 return ret;
4036e387
CI		 2096}
2097
2098/*
2099 * This function sets the CMMA attributes for the given pages. If the input
2100 * buffer has zero length, no action is taken, otherwise the attributes are
c9f0a2b8 2101 * set and the mm->context.uses_cmm flag is set.
4036e387
CI		 2102 */
2103static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2104 const struct kvm_s390_cmma_log *args)
2105{
2106 unsigned long hva, mask, pgstev, i;
2107 uint8_t *bits;
2108 int srcu_idx, r = 0;
2109
2110 mask = args->mask;
2111
2112 if (!kvm->arch.use_cmma)
2113 return -ENXIO;
2114 /* invalid/unsupported flags */
2115 if (args->flags != 0)
2116 return -EINVAL;
2117 /* Enforce sane limit on memory allocation */
2118 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2119 return -EINVAL;
2120 /* Nothing to do */
2121 if (args->count == 0)
2122 return 0;
2123
42bc47b3 2124 bits = vmalloc(array_size(sizeof(*bits), args->count));
4036e387
CI		 2125 if (!bits)
2126 return -ENOMEM;
2127
2128 r = copy_from_user(bits, (void __user *)args->values, args->count);
2129 if (r) {
2130 r = -EFAULT;
2131 goto out;
2132 }
2133
2134 down_read(&kvm->mm->mmap_sem);
2135 srcu_idx = srcu_read_lock(&kvm->srcu);
2136 for (i = 0; i < args->count; i++) {
2137 hva = gfn_to_hva(kvm, args->start_gfn + i);
2138 if (kvm_is_error_hva(hva)) {
2139 r = -EFAULT;
2140 break;
2141 }
2142
2143 pgstev = bits[i];
2144 pgstev = pgstev << 24;
1bab1c02 2145 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
4036e387
CI		 2146 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2147 }
2148 srcu_read_unlock(&kvm->srcu, srcu_idx);
2149 up_read(&kvm->mm->mmap_sem);
2150
c9f0a2b8 2151 if (!kvm->mm->context.uses_cmm) {
4036e387 2152 down_write(&kvm->mm->mmap_sem);
c9f0a2b8 2153 kvm->mm->context.uses_cmm = 1;
4036e387
CI		 2154 up_write(&kvm->mm->mmap_sem);
2155 }
2156out:
2157 vfree(bits);
2158 return r;
2159}
2160
b0c632db
HC		 2161long kvm_arch_vm_ioctl(struct file *filp,
2162 unsigned int ioctl, unsigned long arg)
2163{
2164 struct kvm *kvm = filp->private_data;
2165 void __user *argp = (void __user *)arg;
f2061656 2166 struct kvm_device_attr attr;
b0c632db
HC		 2167 int r;
2168
2169 switch (ioctl) {
ba5c1e9b
CO		 2170 case KVM_S390_INTERRUPT: {
2171 struct kvm_s390_interrupt s390int;
2172
2173 r = -EFAULT;
2174 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2175 break;
2176 r = kvm_s390_inject_vm(kvm, &s390int);
2177 break;
2178 }
84223598
CH		 2179 case KVM_CREATE_IRQCHIP: {
2180 struct kvm_irq_routing_entry routing;
2181
2182 r = -EINVAL;
2183 if (kvm->arch.use_irqchip) {
2184 /* Set up dummy routing. */
2185 memset(&routing, 0, sizeof(routing));
152b2839 2186 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
84223598
CH		 2187 }
2188 break;
2189 }
f2061656
DD		 2190 case KVM_SET_DEVICE_ATTR: {
2191 r = -EFAULT;
2192 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2193 break;
2194 r = kvm_s390_vm_set_attr(kvm, &attr);
2195 break;
2196 }
2197 case KVM_GET_DEVICE_ATTR: {
2198 r = -EFAULT;
2199 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2200 break;
2201 r = kvm_s390_vm_get_attr(kvm, &attr);
2202 break;
2203 }
2204 case KVM_HAS_DEVICE_ATTR: {
2205 r = -EFAULT;
2206 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2207 break;
2208 r = kvm_s390_vm_has_attr(kvm, &attr);
2209 break;
2210 }
30ee2a98
JH		 2211 case KVM_S390_GET_SKEYS: {
2212 struct kvm_s390_skeys args;
2213
2214 r = -EFAULT;
2215 if (copy_from_user(&args, argp,
2216 sizeof(struct kvm_s390_skeys)))
2217 break;
2218 r = kvm_s390_get_skeys(kvm, &args);
2219 break;
2220 }
2221 case KVM_S390_SET_SKEYS: {
2222 struct kvm_s390_skeys args;
2223
2224 r = -EFAULT;
2225 if (copy_from_user(&args, argp,
2226 sizeof(struct kvm_s390_skeys)))
2227 break;
2228 r = kvm_s390_set_skeys(kvm, &args);
2229 break;
2230 }
4036e387
CI		 2231 case KVM_S390_GET_CMMA_BITS: {
2232 struct kvm_s390_cmma_log args;
2233
2234 r = -EFAULT;
2235 if (copy_from_user(&args, argp, sizeof(args)))
2236 break;
1de1ea7e 2237 mutex_lock(&kvm->slots_lock);
4036e387 2238 r = kvm_s390_get_cmma_bits(kvm, &args);
1de1ea7e 2239 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2240 if (!r) {
2241 r = copy_to_user(argp, &args, sizeof(args));
2242 if (r)
2243 r = -EFAULT;
2244 }
2245 break;
2246 }
2247 case KVM_S390_SET_CMMA_BITS: {
2248 struct kvm_s390_cmma_log args;
2249
2250 r = -EFAULT;
2251 if (copy_from_user(&args, argp, sizeof(args)))
2252 break;
1de1ea7e 2253 mutex_lock(&kvm->slots_lock);
4036e387 2254 r = kvm_s390_set_cmma_bits(kvm, &args);
1de1ea7e 2255 mutex_unlock(&kvm->slots_lock);
4036e387
CI		 2256 break;
2257 }
b0c632db 2258 default:
367e1319 2259 r = -ENOTTY;
b0c632db
HC		 2260 }
2261
2262 return r;
2263}
2264
45c9b47c
TK		 2265static int kvm_s390_apxa_installed(void)
2266{
e585b24a 2267 struct ap_config_info info;
45c9b47c 2268
e585b24a
TK		 2269 if (ap_instructions_available()) {
2270 if (ap_qci(&info) == 0)
2271 return info.apxa;
45c9b47c
TK		 2272 }
2273
2274 return 0;
2275}
2276
e585b24a
TK		 2277/*
2278 * The format of the crypto control block (CRYCB) is specified in the 3 low
2279 * order bits of the CRYCB designation (CRYCBD) field as follows:
2280 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2281 * AP extended addressing (APXA) facility are installed.
2282 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2283 * Format 2: Both the APXA and MSAX3 facilities are installed
2284 */
45c9b47c
TK		 2285static void kvm_s390_set_crycb_format(struct kvm *kvm)
2286{
2287 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2288
e585b24a
TK		 2289 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2290 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2291
2292 /* Check whether MSAX3 is installed */
2293 if (!test_kvm_facility(kvm, 76))
2294 return;
2295
45c9b47c
TK		 2296 if (kvm_s390_apxa_installed())
2297 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2298 else
2299 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2300}
2301
0e237e44
PM		 2302void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2303 unsigned long *aqm, unsigned long *adm)
2304{
2305 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2306
2307 mutex_lock(&kvm->lock);
2308 kvm_s390_vcpu_block_all(kvm);
2309
2310 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2311 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2312 memcpy(crycb->apcb1.apm, apm, 32);
2313 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2314 apm[0], apm[1], apm[2], apm[3]);
2315 memcpy(crycb->apcb1.aqm, aqm, 32);
2316 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2317 aqm[0], aqm[1], aqm[2], aqm[3]);
2318 memcpy(crycb->apcb1.adm, adm, 32);
2319 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2320 adm[0], adm[1], adm[2], adm[3]);
2321 break;
2322 case CRYCB_FORMAT1:
2323 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2324 memcpy(crycb->apcb0.apm, apm, 8);
2325 memcpy(crycb->apcb0.aqm, aqm, 2);
2326 memcpy(crycb->apcb0.adm, adm, 2);
2327 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2328 apm[0], *((unsigned short *)aqm),
2329 *((unsigned short *)adm));
2330 break;
2331 default: /* Can not happen */
2332 break;
2333 }
2334
2335 /* recreate the shadow crycb for each vcpu */
2336 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2337 kvm_s390_vcpu_unblock_all(kvm);
2338 mutex_unlock(&kvm->lock);
2339}
2340EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2341
42104598
TK		 2342void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2343{
2344 mutex_lock(&kvm->lock);
2345 kvm_s390_vcpu_block_all(kvm);
2346
2347 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2348 sizeof(kvm->arch.crypto.crycb->apcb0));
2349 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2350 sizeof(kvm->arch.crypto.crycb->apcb1));
2351
0e237e44 2352 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
6cc571b1
PM		 2353 /* recreate the shadow crycb for each vcpu */
2354 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
42104598
TK		 2355 kvm_s390_vcpu_unblock_all(kvm);
2356 mutex_unlock(&kvm->lock);
2357}
2358EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2359
9bb0ec09 2360static u64 kvm_s390_get_initial_cpuid(void)
9d8d5786 2361{
9bb0ec09
DH		 2362 struct cpuid cpuid;
2363
2364 get_cpu_id(&cpuid);
2365 cpuid.version = 0xff;
2366 return *((u64 *) &cpuid);
9d8d5786
MM		 2367}
2368
c54f0d6a 2369static void kvm_s390_crypto_init(struct kvm *kvm)
5102ee87 2370{
c54f0d6a 2371 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
45c9b47c 2372 kvm_s390_set_crycb_format(kvm);
5102ee87 2373
e585b24a
TK		 2374 if (!test_kvm_facility(kvm, 76))
2375 return;
2376
ed6f76b4
TK		 2377 /* Enable AES/DEA protected key functions by default */
2378 kvm->arch.crypto.aes_kw = 1;
2379 kvm->arch.crypto.dea_kw = 1;
2380 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2381 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2382 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2383 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
5102ee87
TK		 2384}
2385
7d43bafc
ED		 2386static void sca_dispose(struct kvm *kvm)
2387{
2388 if (kvm->arch.use_esca)
5e044315 2389 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
7d43bafc
ED		 2390 else
2391 free_page((unsigned long)(kvm->arch.sca));
2392 kvm->arch.sca = NULL;
2393}
2394
e08b9637 2395int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 2396{
76a6dd72 2397 gfp_t alloc_flags = GFP_KERNEL;
9d8d5786 2398 int i, rc;
b0c632db 2399 char debug_name[16];
f6c137ff 2400 static unsigned long sca_offset;
b0c632db 2401
e08b9637
CO		 2402 rc = -EINVAL;
2403#ifdef CONFIG_KVM_S390_UCONTROL
2404 if (type & ~KVM_VM_S390_UCONTROL)
2405 goto out_err;
2406 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2407 goto out_err;
2408#else
2409 if (type)
2410 goto out_err;
2411#endif
2412
b0c632db
HC		 2413 rc = s390_enable_sie();
2414 if (rc)
d89f5eff 2415 goto out_err;
b0c632db 2416
b290411a
CO		 2417 rc = -ENOMEM;
2418
76a6dd72
DH		 2419 if (!sclp.has_64bscao)
2420 alloc_flags |= GFP_DMA;
5e044315 2421 rwlock_init(&kvm->arch.sca_lock);
9ac96d75 2422 /* start with basic SCA */
76a6dd72 2423 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
b0c632db 2424 if (!kvm->arch.sca)
d89f5eff 2425 goto out_err;
0d9ce162 2426 mutex_lock(&kvm_lock);
c5c2c393 2427 sca_offset += 16;
bc784cce 2428 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
c5c2c393 2429 sca_offset = 0;
bc784cce
ED		 2430 kvm->arch.sca = (struct bsca_block *)
2431 ((char *) kvm->arch.sca + sca_offset);
0d9ce162 2432 mutex_unlock(&kvm_lock);
b0c632db
HC		 2433
2434 sprintf(debug_name, "kvm-%u", current->pid);
2435
1cb9cf72 2436 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
b0c632db 2437 if (!kvm->arch.dbf)
40f5b735 2438 goto out_err;
b0c632db 2439
19114beb 2440 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
c54f0d6a
DH		 2441 kvm->arch.sie_page2 =
2442 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2443 if (!kvm->arch.sie_page2)
40f5b735 2444 goto out_err;
9d8d5786 2445
25c84dba 2446 kvm->arch.sie_page2->kvm = kvm;
c54f0d6a 2447 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
c3b9e3e1
CB		 2448
2449 for (i = 0; i < kvm_s390_fac_size(); i++) {
2450 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2451 (kvm_s390_fac_base[i] |
2452 kvm_s390_fac_ext[i]);
2453 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2454 kvm_s390_fac_base[i];
2455 }
346fa2f8 2456 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
981467c9 2457
1935222d
DH		 2458 /* we are always in czam mode - even on pre z14 machines */
2459 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2460 set_kvm_facility(kvm->arch.model.fac_list, 138);
2461 /* we emulate STHYI in kvm */
95ca2cb5
JF		 2462 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2463 set_kvm_facility(kvm->arch.model.fac_list, 74);
1bab1c02
CI		 2464 if (MACHINE_HAS_TLB_GUEST) {
2465 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2466 set_kvm_facility(kvm->arch.model.fac_list, 147);
2467 }
95ca2cb5 2468
05f31e3b
PM		 2469 if (css_general_characteristics.aiv && test_facility(65))
2470 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2471
9bb0ec09 2472 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
37c5f6c8 2473 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
9d8d5786 2474
c54f0d6a 2475 kvm_s390_crypto_init(kvm);
5102ee87 2476
51978393 2477 mutex_init(&kvm->arch.float_int.ais_lock);
ba5c1e9b 2478 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 2479 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2480 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 2481 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 2482 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 2483
b0c632db 2484 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
78f26131 2485 VM_EVENT(kvm, 3, "vm created with type %lu", type);
b0c632db 2486
e08b9637
CO		 2487 if (type & KVM_VM_S390_UCONTROL) {
2488 kvm->arch.gmap = NULL;
a3a92c31 2489 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
e08b9637 2490 } else {
32e6b236 2491 if (sclp.hamax == U64_MAX)
ee71d16d 2492 kvm->arch.mem_limit = TASK_SIZE_MAX;
32e6b236 2493 else
ee71d16d 2494 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
32e6b236 2495 sclp.hamax + 1);
6ea427bb 2496 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
e08b9637 2497 if (!kvm->arch.gmap)
40f5b735 2498 goto out_err;
2c70fe44 2499 kvm->arch.gmap->private = kvm;
24eb3a82 2500 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 2501 }
fa6b7fe9 2502
c9f0a2b8 2503 kvm->arch.use_pfmfi = sclp.has_pfmfi;
55531b74 2504 kvm->arch.use_skf = sclp.has_skey;
8ad35755 2505 spin_lock_init(&kvm->arch.start_stop_lock);
a3508fbe 2506 kvm_s390_vsie_init(kvm);
d7c5cb01 2507 kvm_s390_gisa_init(kvm);
8335713a 2508 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
8ad35755 2509
d89f5eff 2510 return 0;
40f5b735 2511out_err:
c54f0d6a 2512 free_page((unsigned long)kvm->arch.sie_page2);
598841ca 2513 debug_unregister(kvm->arch.dbf);
7d43bafc 2514 sca_dispose(kvm);
78f26131 2515 KVM_EVENT(3, "creation of vm failed: %d", rc);
d89f5eff 2516 return rc;
b0c632db
HC		 2517}
2518
d329c035
CB		 2519void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2520{
2521 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 2522 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 2523 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 2524 kvm_clear_async_pf_completion_queue(vcpu);
bc784cce 2525 if (!kvm_is_ucontrol(vcpu->kvm))
a6e2f683 2526 sca_del_vcpu(vcpu);
27e0393f
CO		 2527
2528 if (kvm_is_ucontrol(vcpu->kvm))
6ea427bb 2529 gmap_remove(vcpu->arch.gmap);
27e0393f 2530
e6db1d61 2531 if (vcpu->kvm->arch.use_cmma)
b31605c1 2532 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 2533 free_page((unsigned long)(vcpu->arch.sie_block));
d329c035
CB		 2534}
2535
2536static void kvm_free_vcpus(struct kvm *kvm)
2537{
2538 unsigned int i;
988a2cae 2539 struct kvm_vcpu *vcpu;
d329c035 2540
988a2cae 2541 kvm_for_each_vcpu(i, vcpu, kvm)
4543bdc0 2542 kvm_vcpu_destroy(vcpu);
988a2cae
GN		 2543
2544 mutex_lock(&kvm->lock);
2545 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2546 kvm->vcpus[i] = NULL;
2547
2548 atomic_set(&kvm->online_vcpus, 0);
2549 mutex_unlock(&kvm->lock);
d329c035
CB		 2550}
2551
b0c632db
HC		 2552void kvm_arch_destroy_vm(struct kvm *kvm)
2553{
d329c035 2554 kvm_free_vcpus(kvm);
7d43bafc 2555 sca_dispose(kvm);
d329c035 2556 debug_unregister(kvm->arch.dbf);
d7c5cb01 2557 kvm_s390_gisa_destroy(kvm);
c54f0d6a 2558 free_page((unsigned long)kvm->arch.sie_page2);
27e0393f 2559 if (!kvm_is_ucontrol(kvm))
6ea427bb 2560 gmap_remove(kvm->arch.gmap);
841b91c5 2561 kvm_s390_destroy_adapters(kvm);
67335e63 2562 kvm_s390_clear_float_irqs(kvm);
a3508fbe 2563 kvm_s390_vsie_destroy(kvm);
8335713a 2564 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
b0c632db
HC		 2565}
2566
2567/* Section: vcpu related */
dafd032a
DD		 2568static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2569{
6ea427bb 2570 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
dafd032a
DD		 2571 if (!vcpu->arch.gmap)
2572 return -ENOMEM;
2573 vcpu->arch.gmap->private = vcpu->kvm;
2574
2575 return 0;
2576}
2577
a6e2f683
ED		 2578static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2579{
a6940674
DH		 2580 if (!kvm_s390_use_sca_entries())
2581 return;
5e044315 2582 read_lock(&vcpu->kvm->arch.sca_lock);
7d43bafc
ED		 2583 if (vcpu->kvm->arch.use_esca) {
2584 struct esca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2585
7d43bafc 2586 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
10ce32d5 2587 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc
ED		 2588 } else {
2589 struct bsca_block *sca = vcpu->kvm->arch.sca;
2590
2591 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
10ce32d5 2592 sca->cpu[vcpu->vcpu_id].sda = 0;
7d43bafc 2593 }
5e044315 2594 read_unlock(&vcpu->kvm->arch.sca_lock);
a6e2f683
ED		 2595}
2596
eaa78f34 2597static void sca_add_vcpu(struct kvm_vcpu *vcpu)
a6e2f683 2598{
a6940674
DH		 2599 if (!kvm_s390_use_sca_entries()) {
2600 struct bsca_block *sca = vcpu->kvm->arch.sca;
2601
2602 /* we still need the basic sca for the ipte control */
2603 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2604 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
f07afa04 2605 return;
a6940674 2606 }
eaa78f34
DH		 2607 read_lock(&vcpu->kvm->arch.sca_lock);
2608 if (vcpu->kvm->arch.use_esca) {
2609 struct esca_block *sca = vcpu->kvm->arch.sca;
7d43bafc 2610
eaa78f34 2611 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2612 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2613 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
0c9d8683 2614 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
eaa78f34 2615 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
7d43bafc 2616 } else {
eaa78f34 2617 struct bsca_block *sca = vcpu->kvm->arch.sca;
a6e2f683 2618
eaa78f34 2619 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
7d43bafc
ED		 2620 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2621 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
eaa78f34 2622 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
7d43bafc 2623 }
eaa78f34 2624 read_unlock(&vcpu->kvm->arch.sca_lock);
5e044315
ED		 2625}
2626
2627/* Basic SCA to Extended SCA data copy routines */
2628static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2629{
2630 d->sda = s->sda;
2631 d->sigp_ctrl.c = s->sigp_ctrl.c;
2632 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2633}
2634
2635static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2636{
2637 int i;
2638
2639 d->ipte_control = s->ipte_control;
2640 d->mcn[0] = s->mcn;
2641 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2642 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2643}
2644
2645static int sca_switch_to_extended(struct kvm *kvm)
2646{
2647 struct bsca_block *old_sca = kvm->arch.sca;
2648 struct esca_block *new_sca;
2649 struct kvm_vcpu *vcpu;
2650 unsigned int vcpu_idx;
2651 u32 scaol, scaoh;
2652
2653 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2654 if (!new_sca)
2655 return -ENOMEM;
2656
2657 scaoh = (u32)((u64)(new_sca) >> 32);
2658 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2659
2660 kvm_s390_vcpu_block_all(kvm);
2661 write_lock(&kvm->arch.sca_lock);
2662
2663 sca_copy_b_to_e(new_sca, old_sca);
2664
2665 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2666 vcpu->arch.sie_block->scaoh = scaoh;
2667 vcpu->arch.sie_block->scaol = scaol;
0c9d8683 2668 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
5e044315
ED		 2669 }
2670 kvm->arch.sca = new_sca;
2671 kvm->arch.use_esca = 1;
2672
2673 write_unlock(&kvm->arch.sca_lock);
2674 kvm_s390_vcpu_unblock_all(kvm);
2675
2676 free_page((unsigned long)old_sca);
2677
8335713a
CB		 2678 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2679 old_sca, kvm->arch.sca);
5e044315 2680 return 0;
a6e2f683
ED		 2681}
2682
2683static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2684{
5e044315
ED		 2685 int rc;
2686
a6940674
DH		 2687 if (!kvm_s390_use_sca_entries()) {
2688 if (id < KVM_MAX_VCPUS)
2689 return true;
2690 return false;
2691 }
5e044315
ED		 2692 if (id < KVM_S390_BSCA_CPU_SLOTS)
2693 return true;
76a6dd72 2694 if (!sclp.has_esca || !sclp.has_64bscao)
5e044315
ED		 2695 return false;
2696
2697 mutex_lock(&kvm->lock);
2698 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2699 mutex_unlock(&kvm->lock);
2700
2701 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
a6e2f683
ED		 2702}
2703
db0758b2
DH		 2704/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2705static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2706{
2707 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
9c23a131 2708 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2 2709 vcpu->arch.cputm_start = get_tod_clock_fast();
9c23a131 2710 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2711}
2712
2713/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2714static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2715{
2716 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
9c23a131 2717 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2718 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2719 vcpu->arch.cputm_start = 0;
9c23a131 2720 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2721}
2722
2723/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2724static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2725{
2726 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2727 vcpu->arch.cputm_enabled = true;
2728 __start_cpu_timer_accounting(vcpu);
2729}
2730
2731/* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2732static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2733{
2734 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2735 __stop_cpu_timer_accounting(vcpu);
2736 vcpu->arch.cputm_enabled = false;
2737}
2738
2739static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2740{
2741 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2742 __enable_cpu_timer_accounting(vcpu);
2743 preempt_enable();
2744}
2745
2746static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2747{
2748 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2749 __disable_cpu_timer_accounting(vcpu);
2750 preempt_enable();
2751}
2752
4287f247
DH		 2753/* set the cpu timer - may only be called from the VCPU thread itself */
2754void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2755{
db0758b2 2756 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
9c23a131 2757 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
db0758b2
DH		 2758 if (vcpu->arch.cputm_enabled)
2759 vcpu->arch.cputm_start = get_tod_clock_fast();
4287f247 2760 vcpu->arch.sie_block->cputm = cputm;
9c23a131 2761 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
db0758b2 2762 preempt_enable();
4287f247
DH		 2763}
2764
db0758b2 2765/* update and get the cpu timer - can also be called from other VCPU threads */
4287f247
DH		 2766__u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2767{
9c23a131 2768 unsigned int seq;
db0758b2 2769 __u64 value;
db0758b2
DH		 2770
2771 if (unlikely(!vcpu->arch.cputm_enabled))
2772 return vcpu->arch.sie_block->cputm;
2773
9c23a131
DH		 2774 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2775 do {
2776 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2777 /*
2778 * If the writer would ever execute a read in the critical
2779 * section, e.g. in irq context, we have a deadlock.
2780 */
2781 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2782 value = vcpu->arch.sie_block->cputm;
2783 /* if cputm_start is 0, accounting is being started/stopped */
2784 if (likely(vcpu->arch.cputm_start))
2785 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2786 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2787 preempt_enable();
db0758b2 2788 return value;
4287f247
DH		 2789}
2790
b0c632db
HC		 2791void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2792{
9977e886 2793
37d9df98 2794 gmap_enable(vcpu->arch.enabled_gmap);
ef8f4f49 2795 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
5ebda316 2796 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2797 __start_cpu_timer_accounting(vcpu);
01a745ac 2798 vcpu->cpu = cpu;
b0c632db
HC		 2799}
2800
2801void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2802{
01a745ac 2803 vcpu->cpu = -1;
5ebda316 2804 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
db0758b2 2805 __stop_cpu_timer_accounting(vcpu);
9daecfc6 2806 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
37d9df98
DH		 2807 vcpu->arch.enabled_gmap = gmap_get_enabled();
2808 gmap_disable(vcpu->arch.enabled_gmap);
9977e886 2809
b0c632db
HC		 2810}
2811
31928aa5 2812void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 2813{
72f25020 2814 mutex_lock(&vcpu->kvm->lock);
fdf03650 2815 preempt_disable();
72f25020 2816 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
d16b52cb 2817 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
fdf03650 2818 preempt_enable();
72f25020 2819 mutex_unlock(&vcpu->kvm->lock);
25508824 2820 if (!kvm_is_ucontrol(vcpu->kvm)) {
dafd032a 2821 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
eaa78f34 2822 sca_add_vcpu(vcpu);
25508824 2823 }
6502a34c
DH		 2824 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2825 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
37d9df98
DH		 2826 /* make vcpu_load load the right gmap on the first trigger */
2827 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
42897d86
MT		 2828}
2829
8ec2fa52
CB		 2830static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2831{
2832 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2833 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2834 return true;
2835 return false;
2836}
2837
2838static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2839{
2840 /* At least one ECC subfunction must be present */
2841 return kvm_has_pckmo_subfunc(kvm, 32) ||
2842 kvm_has_pckmo_subfunc(kvm, 33) ||
2843 kvm_has_pckmo_subfunc(kvm, 34) ||
2844 kvm_has_pckmo_subfunc(kvm, 40) ||
2845 kvm_has_pckmo_subfunc(kvm, 41);
2846
2847}
2848
5102ee87
TK		 2849static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2850{
e585b24a
TK		 2851 /*
2852 * If the AP instructions are not being interpreted and the MSAX3
2853 * facility is not configured for the guest, there is nothing to set up.
2854 */
2855 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 2856 return;
2857
e585b24a 2858 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
a374e892 2859 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
37940fb0 2860 vcpu->arch.sie_block->eca &= ~ECA_APIE;
8ec2fa52 2861 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
a374e892 2862
e585b24a
TK		 2863 if (vcpu->kvm->arch.crypto.apie)
2864 vcpu->arch.sie_block->eca |= ECA_APIE;
a374e892 2865
e585b24a 2866 /* Set up protected key support */
8ec2fa52 2867 if (vcpu->kvm->arch.crypto.aes_kw) {
a374e892 2868 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
8ec2fa52
CB		 2869 /* ecc is also wrapped with AES key */
2870 if (kvm_has_pckmo_ecc(vcpu->kvm))
2871 vcpu->arch.sie_block->ecd |= ECD_ECC;
2872 }
2873
a374e892
TK		 2874 if (vcpu->kvm->arch.crypto.dea_kw)
2875 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
5102ee87
TK		 2876}
2877
b31605c1
DD		 2878void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2879{
2880 free_page(vcpu->arch.sie_block->cbrlo);
2881 vcpu->arch.sie_block->cbrlo = 0;
2882}
2883
2884int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2885{
2886 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2887 if (!vcpu->arch.sie_block->cbrlo)
2888 return -ENOMEM;
b31605c1
DD		 2889 return 0;
2890}
2891
91520f1a
MM		 2892static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2893{
2894 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2895
91520f1a 2896 vcpu->arch.sie_block->ibc = model->ibc;
80bc79dc 2897 if (test_kvm_facility(vcpu->kvm, 7))
c54f0d6a 2898 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
91520f1a
MM		 2899}
2900
ff72bb55 2901static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
b0c632db 2902{
b31605c1 2903 int rc = 0;
b31288fa 2904
9e6dabef
CH		 2905 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2906 CPUSTAT_SM |
a4a4f191
GH		 2907 CPUSTAT_STOPPED);
2908
53df84f8 2909 if (test_kvm_facility(vcpu->kvm, 78))
ef8f4f49 2910 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
53df84f8 2911 else if (test_kvm_facility(vcpu->kvm, 8))
ef8f4f49 2912 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
a4a4f191 2913
91520f1a
MM		 2914 kvm_s390_vcpu_setup_model(vcpu);
2915
bdab09f3
DH		 2916 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2917 if (MACHINE_HAS_ESOP)
0c9d8683 2918 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
bd50e8ec 2919 if (test_kvm_facility(vcpu->kvm, 9))
0c9d8683 2920 vcpu->arch.sie_block->ecb |= ECB_SRSI;
f597d24e 2921 if (test_kvm_facility(vcpu->kvm, 73))
0c9d8683 2922 vcpu->arch.sie_block->ecb |= ECB_TE;
7feb6bb8 2923
c9f0a2b8 2924 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
0c9d8683 2925 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
cd1836f5 2926 if (test_kvm_facility(vcpu->kvm, 130))
0c9d8683
DH		 2927 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2928 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
48ee7d3a 2929 if (sclp.has_cei)
0c9d8683 2930 vcpu->arch.sie_block->eca |= ECA_CEI;
11ad65b7 2931 if (sclp.has_ib)
0c9d8683 2932 vcpu->arch.sie_block->eca |= ECA_IB;
37c5f6c8 2933 if (sclp.has_siif)
0c9d8683 2934 vcpu->arch.sie_block->eca |= ECA_SII;
37c5f6c8 2935 if (sclp.has_sigpif)
0c9d8683 2936 vcpu->arch.sie_block->eca |= ECA_SIGPI;
18280d8b 2937 if (test_kvm_facility(vcpu->kvm, 129)) {
0c9d8683
DH		 2938 vcpu->arch.sie_block->eca |= ECA_VX;
2939 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
13211ea7 2940 }
8fa1696e
CW		 2941 if (test_kvm_facility(vcpu->kvm, 139))
2942 vcpu->arch.sie_block->ecd |= ECD_MEF;
a3da7b4a
CB		 2943 if (test_kvm_facility(vcpu->kvm, 156))
2944 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
d7c5cb01
MM		 2945 if (vcpu->arch.sie_block->gd) {
2946 vcpu->arch.sie_block->eca |= ECA_AIV;
2947 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2948 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2949 }
4e0b1ab7
FZ		 2950 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2951 | SDNXC;
c6e5f166 2952 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
730cd632
FA		 2953
2954 if (sclp.has_kss)
ef8f4f49 2955 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
730cd632
FA		 2956 else
2957 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 2958
e6db1d61 2959 if (vcpu->kvm->arch.use_cmma) {
b31605c1
DD		 2960 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2961 if (rc)
2962 return rc;
b31288fa 2963 }
0ac96caf 2964 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 2965 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 2966
67d49d52
CW		 2967 vcpu->arch.sie_block->hpid = HPID_KVM;
2968
5102ee87
TK		 2969 kvm_s390_vcpu_crypto_setup(vcpu);
2970
b31605c1 2971 return rc;
b0c632db
HC		 2972}
2973
897cc38e
SC		 2974int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
2975{
2976 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2977 return -EINVAL;
2978 return 0;
2979}
2980
e529ef66 2981int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
b0c632db 2982{
7feb6bb8 2983 struct sie_page *sie_page;
897cc38e 2984 int rc;
4d47555a 2985
da72ca4d 2986 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
7feb6bb8
MM		 2987 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2988 if (!sie_page)
e529ef66 2989 return -ENOMEM;
b0c632db 2990
7feb6bb8
MM		 2991 vcpu->arch.sie_block = &sie_page->sie_block;
2992 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2993
efed1104
DH		 2994 /* the real guest size will always be smaller than msl */
2995 vcpu->arch.sie_block->mso = 0;
2996 vcpu->arch.sie_block->msl = sclp.hamax;
2997
e529ef66 2998 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
ba5c1e9b 2999 spin_lock_init(&vcpu->arch.local_int.lock);
e529ef66 3000 vcpu->arch.sie_block->gd = (u32)(u64)vcpu->kvm->arch.gisa_int.origin;
4b9f9525
MM		 3001 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3002 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
9c23a131 3003 seqcount_init(&vcpu->arch.cputm_seqcount);
ba5c1e9b 3004
321f8ee5
SC		 3005 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3006 kvm_clear_async_pf_completion_queue(vcpu);
3007 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3008 KVM_SYNC_GPRS |
3009 KVM_SYNC_ACRS |
3010 KVM_SYNC_CRS |
3011 KVM_SYNC_ARCH0 |
3012 KVM_SYNC_PFAULT;
3013 kvm_s390_set_prefix(vcpu, 0);
3014 if (test_kvm_facility(vcpu->kvm, 64))
3015 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3016 if (test_kvm_facility(vcpu->kvm, 82))
3017 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3018 if (test_kvm_facility(vcpu->kvm, 133))
3019 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3020 if (test_kvm_facility(vcpu->kvm, 156))
3021 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3022 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3023 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
3024 */
3025 if (MACHINE_HAS_VX)
3026 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3027 else
3028 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3029
3030 if (kvm_is_ucontrol(vcpu->kvm)) {
3031 rc = __kvm_ucontrol_vcpu_init(vcpu);
3032 if (rc)
a2017f17 3033 goto out_free_sie_block;
321f8ee5
SC		 3034 }
3035
e529ef66
SC		 3036 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
3037 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3038 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3039
ff72bb55
SC		 3040 rc = kvm_s390_vcpu_setup(vcpu);
3041 if (rc)
3042 goto out_ucontrol_uninit;
e529ef66 3043 return 0;
b0c632db 3044
ff72bb55
SC		 3045out_ucontrol_uninit:
3046 if (kvm_is_ucontrol(vcpu->kvm))
3047 gmap_remove(vcpu->arch.gmap);
7b06bf2f
WY		 3048out_free_sie_block:
3049 free_page((unsigned long)(vcpu->arch.sie_block));
e529ef66 3050 return rc;
b0c632db
HC		 3051}
3052
b0c632db
HC		 3053int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3054{
9a022067 3055 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 3056}
3057
199b5763
LM		 3058bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3059{
0546c63d 3060 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
199b5763
LM		 3061}
3062
27406cd5 3063void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
49b99e1e 3064{
805de8f4 3065 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
61a6df54 3066 exit_sie(vcpu);
49b99e1e
CB		 3067}
3068
27406cd5 3069void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
49b99e1e 3070{
805de8f4 3071 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
49b99e1e
CB		 3072}
3073
8e236546
CB		 3074static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3075{
805de8f4 3076 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
61a6df54 3077 exit_sie(vcpu);
8e236546
CB		 3078}
3079
9ea59728
DH		 3080bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3081{
3082 return atomic_read(&vcpu->arch.sie_block->prog20) &
3083 (PROG_BLOCK_SIE | PROG_REQUEST);
3084}
3085
8e236546
CB		 3086static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3087{
9bf9fde2 3088 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
8e236546
CB		 3089}
3090
49b99e1e 3091/*
9ea59728 3092 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
49b99e1e
CB		 3093 * If the CPU is not running (e.g. waiting as idle) the function will
3094 * return immediately. */
3095void exit_sie(struct kvm_vcpu *vcpu)
3096{
ef8f4f49 3097 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
9ea59728 3098 kvm_s390_vsie_kick(vcpu);
49b99e1e
CB		 3099 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3100 cpu_relax();
3101}
3102
8e236546
CB		 3103/* Kick a guest cpu out of SIE to process a request synchronously */
3104void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
49b99e1e 3105{
8e236546
CB		 3106 kvm_make_request(req, vcpu);
3107 kvm_s390_vcpu_request(vcpu);
49b99e1e
CB		 3108}
3109
414d3b07
MS		 3110static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3111 unsigned long end)
2c70fe44 3112{
2c70fe44
CB		 3113 struct kvm *kvm = gmap->private;
3114 struct kvm_vcpu *vcpu;
414d3b07
MS		 3115 unsigned long prefix;
3116 int i;
2c70fe44 3117
65d0b0d4
DH		 3118 if (gmap_is_shadow(gmap))
3119 return;
414d3b07
MS		 3120 if (start >= 1UL << 31)
3121 /* We are only interested in prefix pages */
3122 return;
2c70fe44
CB		 3123 kvm_for_each_vcpu(i, vcpu, kvm) {
3124 /* match against both prefix pages */
414d3b07
MS		 3125 prefix = kvm_s390_get_prefix(vcpu);
3126 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3127 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3128 start, end);
8e236546 3129 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44
CB		 3130 }
3131 }
3132}
3133
8b905d28
CB		 3134bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3135{
3136 /* do not poll with more than halt_poll_max_steal percent of steal time */
3137 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3138 halt_poll_max_steal) {
3139 vcpu->stat.halt_no_poll_steal++;
3140 return true;
3141 }
3142 return false;
3143}
3144
b6d33834
CD		 3145int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3146{
3147 /* kvm common code refers to this, but never calls it */
3148 BUG();
3149 return 0;
3150}
3151
14eebd91
CO		 3152static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3153 struct kvm_one_reg *reg)
3154{
3155 int r = -EINVAL;
3156
3157 switch (reg->id) {
29b7c71b
CO		 3158 case KVM_REG_S390_TODPR:
3159 r = put_user(vcpu->arch.sie_block->todpr,
3160 (u32 __user *)reg->addr);
3161 break;
3162 case KVM_REG_S390_EPOCHDIFF:
3163 r = put_user(vcpu->arch.sie_block->epoch,
3164 (u64 __user *)reg->addr);
3165 break;
46a6dd1c 3166 case KVM_REG_S390_CPU_TIMER:
4287f247 3167 r = put_user(kvm_s390_get_cpu_timer(vcpu),
46a6dd1c
J		 3168 (u64 __user *)reg->addr);
3169 break;
3170 case KVM_REG_S390_CLOCK_COMP:
3171 r = put_user(vcpu->arch.sie_block->ckc,
3172 (u64 __user *)reg->addr);
3173 break;
536336c2
DD		 3174 case KVM_REG_S390_PFTOKEN:
3175 r = put_user(vcpu->arch.pfault_token,
3176 (u64 __user *)reg->addr);
3177 break;
3178 case KVM_REG_S390_PFCOMPARE:
3179 r = put_user(vcpu->arch.pfault_compare,
3180 (u64 __user *)reg->addr);
3181 break;
3182 case KVM_REG_S390_PFSELECT:
3183 r = put_user(vcpu->arch.pfault_select,
3184 (u64 __user *)reg->addr);
3185 break;
672550fb
CB		 3186 case KVM_REG_S390_PP:
3187 r = put_user(vcpu->arch.sie_block->pp,
3188 (u64 __user *)reg->addr);
3189 break;
afa45ff5
CB		 3190 case KVM_REG_S390_GBEA:
3191 r = put_user(vcpu->arch.sie_block->gbea,
3192 (u64 __user *)reg->addr);
3193 break;
14eebd91
CO		 3194 default:
3195 break;
3196 }
3197
3198 return r;
3199}
3200
3201static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3202 struct kvm_one_reg *reg)
3203{
3204 int r = -EINVAL;
4287f247 3205 __u64 val;
14eebd91
CO		 3206
3207 switch (reg->id) {
29b7c71b
CO		 3208 case KVM_REG_S390_TODPR:
3209 r = get_user(vcpu->arch.sie_block->todpr,
3210 (u32 __user *)reg->addr);
3211 break;
3212 case KVM_REG_S390_EPOCHDIFF:
3213 r = get_user(vcpu->arch.sie_block->epoch,
3214 (u64 __user *)reg->addr);
3215 break;
46a6dd1c 3216 case KVM_REG_S390_CPU_TIMER:
4287f247
DH		 3217 r = get_user(val, (u64 __user *)reg->addr);
3218 if (!r)
3219 kvm_s390_set_cpu_timer(vcpu, val);
46a6dd1c
J		 3220 break;
3221 case KVM_REG_S390_CLOCK_COMP:
3222 r = get_user(vcpu->arch.sie_block->ckc,
3223 (u64 __user *)reg->addr);
3224 break;
536336c2
DD		 3225 case KVM_REG_S390_PFTOKEN:
3226 r = get_user(vcpu->arch.pfault_token,
3227 (u64 __user *)reg->addr);
9fbd8082
DH		 3228 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3229 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 3230 break;
3231 case KVM_REG_S390_PFCOMPARE:
3232 r = get_user(vcpu->arch.pfault_compare,
3233 (u64 __user *)reg->addr);
3234 break;
3235 case KVM_REG_S390_PFSELECT:
3236 r = get_user(vcpu->arch.pfault_select,
3237 (u64 __user *)reg->addr);
3238 break;
672550fb
CB		 3239 case KVM_REG_S390_PP:
3240 r = get_user(vcpu->arch.sie_block->pp,
3241 (u64 __user *)reg->addr);
3242 break;
afa45ff5
CB		 3243 case KVM_REG_S390_GBEA:
3244 r = get_user(vcpu->arch.sie_block->gbea,
3245 (u64 __user *)reg->addr);
3246 break;
14eebd91
CO		 3247 default:
3248 break;
3249 }
3250
3251 return r;
3252}
b6d33834 3253
7de3f142 3254static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
b0c632db 3255{
7de3f142
JF		 3256 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3257 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3258 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3259
3260 kvm_clear_async_pf_completion_queue(vcpu);
3261 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3262 kvm_s390_vcpu_stop(vcpu);
3263 kvm_s390_clear_local_irqs(vcpu);
3264}
3265
3266static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3267{
3268 /* Initial reset is a superset of the normal reset */
3269 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3270
e93fc7b4
CB		 3271 /*
3272 * This equals initial cpu reset in pop, but we don't switch to ESA.
3273 * We do not only reset the internal data, but also ...
3274 */
7de3f142
JF		 3275 vcpu->arch.sie_block->gpsw.mask = 0;
3276 vcpu->arch.sie_block->gpsw.addr = 0;
3277 kvm_s390_set_prefix(vcpu, 0);
3278 kvm_s390_set_cpu_timer(vcpu, 0);
3279 vcpu->arch.sie_block->ckc = 0;
3280 vcpu->arch.sie_block->todpr = 0;
3281 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3282 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3283 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
e93fc7b4
CB		 3284
3285 /* ... the data in sync regs */
3286 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
3287 vcpu->run->s.regs.ckc = 0;
3288 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
3289 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
3290 vcpu->run->psw_addr = 0;
3291 vcpu->run->psw_mask = 0;
3292 vcpu->run->s.regs.todpr = 0;
3293 vcpu->run->s.regs.cputm = 0;
3294 vcpu->run->s.regs.ckc = 0;
3295 vcpu->run->s.regs.pp = 0;
3296 vcpu->run->s.regs.gbea = 1;
7de3f142
JF		 3297 vcpu->run->s.regs.fpc = 0;
3298 vcpu->arch.sie_block->gbea = 1;
3299 vcpu->arch.sie_block->pp = 0;
3300 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3301}
3302
3303static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3304{
3305 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3306
3307 /* Clear reset is a superset of the initial reset */
3308 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3309
3310 memset(&regs->gprs, 0, sizeof(regs->gprs));
3311 memset(&regs->vrs, 0, sizeof(regs->vrs));
3312 memset(&regs->acrs, 0, sizeof(regs->acrs));
3313 memset(&regs->gscb, 0, sizeof(regs->gscb));
3314
3315 regs->etoken = 0;
3316 regs->etoken_extension = 0;
b0c632db
HC		 3317}
3318
3319int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3320{
875656fe 3321 vcpu_load(vcpu);
5a32c1af 3322 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
875656fe 3323 vcpu_put(vcpu);
b0c632db
HC		 3324 return 0;
3325}
3326
3327int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3328{
1fc9b76b 3329 vcpu_load(vcpu);
5a32c1af 3330 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1fc9b76b 3331 vcpu_put(vcpu);
b0c632db
HC		 3332 return 0;
3333}
3334
3335int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3336 struct kvm_sregs *sregs)
3337{
b4ef9d4e
CD		 3338 vcpu_load(vcpu);
3339
59674c1a 3340 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 3341 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
b4ef9d4e
CD		 3342
3343 vcpu_put(vcpu);
b0c632db
HC		 3344 return 0;
3345}
3346
3347int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3348 struct kvm_sregs *sregs)
3349{
bcdec41c
CD		 3350 vcpu_load(vcpu);
3351
59674c1a 3352 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 3353 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
bcdec41c
CD		 3354
3355 vcpu_put(vcpu);
b0c632db
HC		 3356 return 0;
3357}
3358
3359int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3360{
6a96bc7f
CD		 3361 int ret = 0;
3362
3363 vcpu_load(vcpu);
3364
3365 if (test_fp_ctl(fpu->fpc)) {
3366 ret = -EINVAL;
3367 goto out;
3368 }
e1788bb9 3369 vcpu->run->s.regs.fpc = fpu->fpc;
9abc2a08 3370 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3371 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3372 (freg_t *) fpu->fprs);
9abc2a08 3373 else
a7d4b8f2 3374 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
6a96bc7f
CD		 3375
3376out:
3377 vcpu_put(vcpu);
3378 return ret;
b0c632db
HC		 3379}
3380
3381int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3382{
1393123e
CD		 3383 vcpu_load(vcpu);
3384
9abc2a08
DH		 3385 /* make sure we have the latest values */
3386 save_fpu_regs();
3387 if (MACHINE_HAS_VX)
a7d4b8f2
DH		 3388 convert_vx_to_fp((freg_t *) fpu->fprs,
3389 (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08 3390 else
a7d4b8f2 3391 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
e1788bb9 3392 fpu->fpc = vcpu->run->s.regs.fpc;
1393123e
CD		 3393
3394 vcpu_put(vcpu);
b0c632db
HC		 3395 return 0;
3396}
3397
3398static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3399{
3400 int rc = 0;
3401
7a42fdc2 3402 if (!is_vcpu_stopped(vcpu))
b0c632db 3403 rc = -EBUSY;
d7b0b5eb
CO		 3404 else {
3405 vcpu->run->psw_mask = psw.mask;
3406 vcpu->run->psw_addr = psw.addr;
3407 }
b0c632db
HC		 3408 return rc;
3409}
3410
3411int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3412 struct kvm_translation *tr)
3413{
3414 return -EINVAL; /* not implemented yet */
3415}
3416
27291e21
DH		 3417#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3418 KVM_GUESTDBG_USE_HW_BP | \
3419 KVM_GUESTDBG_ENABLE)
3420
d0bfb940
JK		 3421int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3422 struct kvm_guest_debug *dbg)
b0c632db 3423{
27291e21
DH		 3424 int rc = 0;
3425
66b56562
CD		 3426 vcpu_load(vcpu);
3427
27291e21
DH		 3428 vcpu->guest_debug = 0;
3429 kvm_s390_clear_bp_data(vcpu);
3430
66b56562
CD		 3431 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3432 rc = -EINVAL;
3433 goto out;
3434 }
3435 if (!sclp.has_gpere) {
3436 rc = -EINVAL;
3437 goto out;
3438 }
27291e21
DH		 3439
3440 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3441 vcpu->guest_debug = dbg->control;
3442 /* enforce guest PER */
ef8f4f49 3443 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3444
3445 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3446 rc = kvm_s390_import_bp_data(vcpu, dbg);
3447 } else {
9daecfc6 3448 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3449 vcpu->arch.guestdbg.last_bp = 0;
3450 }
3451
3452 if (rc) {
3453 vcpu->guest_debug = 0;
3454 kvm_s390_clear_bp_data(vcpu);
9daecfc6 3455 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
27291e21
DH		 3456 }
3457
66b56562
CD		 3458out:
3459 vcpu_put(vcpu);
27291e21 3460 return rc;
b0c632db
HC		 3461}
3462
62d9f0db
MT		 3463int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3464 struct kvm_mp_state *mp_state)
3465{
fd232561
CD		 3466 int ret;
3467
3468 vcpu_load(vcpu);
3469
6352e4d2 3470 /* CHECK_STOP and LOAD are not supported yet */
fd232561
CD		 3471 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3472 KVM_MP_STATE_OPERATING;
3473
3474 vcpu_put(vcpu);
3475 return ret;
62d9f0db
MT		 3476}
3477
3478int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3479 struct kvm_mp_state *mp_state)
3480{
6352e4d2
DH		 3481 int rc = 0;
3482
e83dff5e
CD		 3483 vcpu_load(vcpu);
3484
6352e4d2
DH		 3485 /* user space knows about this interface - let it control the state */
3486 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3487
3488 switch (mp_state->mp_state) {
3489 case KVM_MP_STATE_STOPPED:
3490 kvm_s390_vcpu_stop(vcpu);
3491 break;
3492 case KVM_MP_STATE_OPERATING:
3493 kvm_s390_vcpu_start(vcpu);
3494 break;
3495 case KVM_MP_STATE_LOAD:
3496 case KVM_MP_STATE_CHECK_STOP:
3497 /* fall through - CHECK_STOP and LOAD are not supported yet */
3498 default:
3499 rc = -ENXIO;
3500 }
3501
e83dff5e 3502 vcpu_put(vcpu);
6352e4d2 3503 return rc;
62d9f0db
MT		 3504}
3505
8ad35755
DH		 3506static bool ibs_enabled(struct kvm_vcpu *vcpu)
3507{
8d5fb0dc 3508 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3509}
3510
2c70fe44
CB		 3511static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3512{
8ad35755 3513retry:
8e236546 3514 kvm_s390_vcpu_request_handled(vcpu);
2fa6e1e1 3515 if (!kvm_request_pending(vcpu))
586b7ccd 3516 return 0;
2c70fe44
CB		 3517 /*
3518 * We use MMU_RELOAD just to re-arm the ipte notifier for the
b2d73b2a 3519 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2c70fe44
CB		 3520 * This ensures that the ipte instruction for this request has
3521 * already finished. We might race against a second unmapper that
3522 * wants to set the blocking bit. Lets just retry the request loop.
3523 */
8ad35755 3524 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44 3525 int rc;
b2d73b2a
MS		 3526 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3527 kvm_s390_get_prefix(vcpu),
3528 PAGE_SIZE * 2, PROT_WRITE);
aca411a4
JN		 3529 if (rc) {
3530 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2c70fe44 3531 return rc;
aca411a4 3532 }
8ad35755 3533 goto retry;
2c70fe44 3534 }
8ad35755 3535
d3d692c8
DH		 3536 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3537 vcpu->arch.sie_block->ihcpu = 0xffff;
3538 goto retry;
3539 }
3540
8ad35755
DH		 3541 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3542 if (!ibs_enabled(vcpu)) {
3543 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
ef8f4f49 3544 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3545 }
3546 goto retry;
2c70fe44 3547 }
8ad35755
DH		 3548
3549 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3550 if (ibs_enabled(vcpu)) {
3551 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
9daecfc6 3552 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
8ad35755
DH		 3553 }
3554 goto retry;
3555 }
3556
6502a34c
DH		 3557 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3558 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3559 goto retry;
3560 }
3561
190df4a2
CI		 3562 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3563 /*
c9f0a2b8 3564 * Disable CMM virtualization; we will emulate the ESSA
190df4a2
CI		 3565 * instruction manually, in order to provide additional
3566 * functionalities needed for live migration.
3567 */
3568 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3569 goto retry;
3570 }
3571
3572 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3573 /*
c9f0a2b8
JF		 3574 * Re-enable CMM virtualization if CMMA is available and
3575 * CMM has been used.
190df4a2
CI		 3576 */
3577 if ((vcpu->kvm->arch.use_cmma) &&
c9f0a2b8 3578 (vcpu->kvm->mm->context.uses_cmm))
190df4a2
CI		 3579 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3580 goto retry;
3581 }
3582
0759d068 3583 /* nothing to do, just clear the request */
72875d8a 3584 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3194cdb7
DH		 3585 /* we left the vsie handler, nothing to do, just clear the request */
3586 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
0759d068 3587
2c70fe44
CB		 3588 return 0;
3589}
3590
0e7def5f
DH		 3591void kvm_s390_set_tod_clock(struct kvm *kvm,
3592 const struct kvm_s390_vm_tod_clock *gtod)
8fa1696e
CW		 3593{
3594 struct kvm_vcpu *vcpu;
3595 struct kvm_s390_tod_clock_ext htod;
3596 int i;
3597
3598 mutex_lock(&kvm->lock);
3599 preempt_disable();
3600
3601 get_tod_clock_ext((char *)&htod);
3602
3603 kvm->arch.epoch = gtod->tod - htod.tod;
0e7def5f
DH		 3604 kvm->arch.epdx = 0;
3605 if (test_kvm_facility(kvm, 139)) {
3606 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3607 if (kvm->arch.epoch > gtod->tod)
3608 kvm->arch.epdx -= 1;
3609 }
8fa1696e
CW		 3610
3611 kvm_s390_vcpu_block_all(kvm);
3612 kvm_for_each_vcpu(i, vcpu, kvm) {
3613 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3614 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3615 }
3616
3617 kvm_s390_vcpu_unblock_all(kvm);
3618 preempt_enable();
3619 mutex_unlock(&kvm->lock);
3620}
3621
fa576c58
TH		 3622/**
3623 * kvm_arch_fault_in_page - fault-in guest page if necessary
3624 * @vcpu: The corresponding virtual cpu
3625 * @gpa: Guest physical address
3626 * @writable: Whether the page should be writable or not
3627 *
3628 * Make sure that a guest page has been faulted-in on the host.
3629 *
3630 * Return: Zero on success, negative error code otherwise.
3631 */
3632long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 3633{
527e30b4
MS		 3634 return gmap_fault(vcpu->arch.gmap, gpa,
3635 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 3636}
3637
3c038e6b
DD		 3638static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3639 unsigned long token)
3640{
3641 struct kvm_s390_interrupt inti;
383d0b05 3642 struct kvm_s390_irq irq;
3c038e6b
DD		 3643
3644 if (start_token) {
383d0b05
JF		 3645 irq.u.ext.ext_params2 = token;
3646 irq.type = KVM_S390_INT_PFAULT_INIT;
3647 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 3648 } else {
3649 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 3650 inti.parm64 = token;
3c038e6b
DD		 3651 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3652 }
3653}
3654
3655void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3656 struct kvm_async_pf *work)
3657{
3658 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3659 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3660}
3661
3662void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3663 struct kvm_async_pf *work)
3664{
3665 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3666 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3667}
3668
3669void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3670 struct kvm_async_pf *work)
3671{
3672 /* s390 will always inject the page directly */
3673}
3674
3675bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3676{
3677 /*
3678 * s390 will always inject the page directly,
3679 * but we still want check_async_completion to cleanup
3680 */
3681 return true;
3682}
3683
3684static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3685{
3686 hva_t hva;
3687 struct kvm_arch_async_pf arch;
3688 int rc;
3689
3690 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3691 return 0;
3692 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3693 vcpu->arch.pfault_compare)
3694 return 0;
3695 if (psw_extint_disabled(vcpu))
3696 return 0;
9a022067 3697 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b 3698 return 0;
b9224cd7 3699 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3c038e6b
DD		 3700 return 0;
3701 if (!vcpu->arch.gmap->pfault_enabled)
3702 return 0;
3703
81480cc1
HC		 3704 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3705 hva += current->thread.gmap_addr & ~PAGE_MASK;
3706 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 3707 return 0;
3708
3709 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3710 return rc;
3711}
3712
3fb4c40f 3713static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 3714{
3fb4c40f 3715 int rc, cpuflags;
e168bf8d 3716
3c038e6b
DD		 3717 /*
3718 * On s390 notifications for arriving pages will be delivered directly
3719 * to the guest but the house keeping for completed pfaults is
3720 * handled outside the worker.
3721 */
3722 kvm_check_async_pf_completion(vcpu);
3723
7ec7c8c7
CB		 3724 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3725 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
b0c632db
HC		 3726
3727 if (need_resched())
3728 schedule();
3729
d3a73acb 3730 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 3731 s390_handle_mcck();
3732
79395031
JF		 3733 if (!kvm_is_ucontrol(vcpu->kvm)) {
3734 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3735 if (rc)
3736 return rc;
3737 }
0ff31867 3738
2c70fe44
CB		 3739 rc = kvm_s390_handle_requests(vcpu);
3740 if (rc)
3741 return rc;
3742
27291e21
DH		 3743 if (guestdbg_enabled(vcpu)) {
3744 kvm_s390_backup_guest_per_regs(vcpu);
3745 kvm_s390_patch_guest_per_regs(vcpu);
3746 }
3747
9f30f621
MM		 3748 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
3749
b0c632db 3750 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 3751 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3752 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3753 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 3754
3fb4c40f
TH		 3755 return 0;
3756}
3757
492d8642
TH		 3758static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3759{
56317920
DH		 3760 struct kvm_s390_pgm_info pgm_info = {
3761 .code = PGM_ADDRESSING,
3762 };
3763 u8 opcode, ilen;
492d8642
TH		 3764 int rc;
3765
3766 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3767 trace_kvm_s390_sie_fault(vcpu);
3768
3769 /*
3770 * We want to inject an addressing exception, which is defined as a
3771 * suppressing or terminating exception. However, since we came here
3772 * by a DAT access exception, the PSW still points to the faulting
3773 * instruction since DAT exceptions are nullifying. So we've got
3774 * to look up the current opcode to get the length of the instruction
3775 * to be able to forward the PSW.
3776 */
3fa8cad7 3777 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
56317920 3778 ilen = insn_length(opcode);
9b0d721a
DH		 3779 if (rc < 0) {
3780 return rc;
3781 } else if (rc) {
3782 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3783 * Forward by arbitrary ilc, injection will take care of
3784 * nullification if necessary.
3785 */
3786 pgm_info = vcpu->arch.pgm;
3787 ilen = 4;
3788 }
56317920
DH		 3789 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3790 kvm_s390_forward_psw(vcpu, ilen);
3791 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
492d8642
TH		 3792}
3793
3fb4c40f
TH		 3794static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3795{
4d62fcc0
QH		 3796 struct mcck_volatile_info *mcck_info;
3797 struct sie_page *sie_page;
3798
2b29a9fd
DD		 3799 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3800 vcpu->arch.sie_block->icptcode);
3801 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3802
27291e21
DH		 3803 if (guestdbg_enabled(vcpu))
3804 kvm_s390_restore_guest_per_regs(vcpu);
3805
7ec7c8c7
CB		 3806 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3807 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
71f116bf 3808
4d62fcc0
QH		 3809 if (exit_reason == -EINTR) {
3810 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3811 sie_page = container_of(vcpu->arch.sie_block,
3812 struct sie_page, sie_block);
3813 mcck_info = &sie_page->mcck_info;
3814 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3815 return 0;
3816 }
3817
71f116bf
DH		 3818 if (vcpu->arch.sie_block->icptcode > 0) {
3819 int rc = kvm_handle_sie_intercept(vcpu);
3820
3821 if (rc != -EOPNOTSUPP)
3822 return rc;
3823 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3824 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3825 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3826 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3827 return -EREMOTE;
3828 } else if (exit_reason != -EFAULT) {
3829 vcpu->stat.exit_null++;
3830 return 0;
210b1607
TH		 3831 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3832 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3833 vcpu->run->s390_ucontrol.trans_exc_code =
3834 current->thread.gmap_addr;
3835 vcpu->run->s390_ucontrol.pgm_code = 0x10;
71f116bf 3836 return -EREMOTE;
24eb3a82 3837 } else if (current->thread.gmap_pfault) {
3c038e6b 3838 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 3839 current->thread.gmap_pfault = 0;
71f116bf
DH		 3840 if (kvm_arch_setup_async_pf(vcpu))
3841 return 0;
3842 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
a76ccff6 3843 }
71f116bf 3844 return vcpu_post_run_fault_in_sie(vcpu);
3fb4c40f
TH		 3845}
3846
3847static int __vcpu_run(struct kvm_vcpu *vcpu)
3848{
3849 int rc, exit_reason;
3850
800c1065
TH		 3851 /*
3852 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3853 * ning the guest), so that memslots (and other stuff) are protected
3854 */
3855 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3856
a76ccff6
TH		 3857 do {
3858 rc = vcpu_pre_run(vcpu);
3859 if (rc)
3860 break;
3fb4c40f 3861
800c1065 3862 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 3863 /*
3864 * As PF_VCPU will be used in fault handler, between
3865 * guest_enter and guest_exit should be no uaccess.
3866 */
0097d12e 3867 local_irq_disable();
6edaa530 3868 guest_enter_irqoff();
db0758b2 3869 __disable_cpu_timer_accounting(vcpu);
0097d12e 3870 local_irq_enable();
a76ccff6
TH		 3871 exit_reason = sie64a(vcpu->arch.sie_block,
3872 vcpu->run->s.regs.gprs);
0097d12e 3873 local_irq_disable();
db0758b2 3874 __enable_cpu_timer_accounting(vcpu);
6edaa530 3875 guest_exit_irqoff();
0097d12e 3876 local_irq_enable();
800c1065 3877 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 3878
3879 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 3880 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 3881
800c1065 3882 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 3883 return rc;
b0c632db
HC		 3884}
3885
b028ee3e
DH		 3886static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3887{
4d5f2c04 3888 struct runtime_instr_cb *riccb;
4e0b1ab7 3889 struct gs_cb *gscb;
4d5f2c04
CB		 3890
3891 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4e0b1ab7 3892 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
b028ee3e
DH		 3893 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3894 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3895 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3896 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3897 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3898 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 3899 /* some control register changes require a tlb flush */
3900 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 3901 }
3902 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4287f247 3903 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
b028ee3e
DH		 3904 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3905 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3906 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3907 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3908 }
3909 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3910 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3911 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3912 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 3913 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3914 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e 3915 }
80cd8763
FZ		 3916 /*
3917 * If userspace sets the riccb (e.g. after migration) to a valid state,
3918 * we should enable RI here instead of doing the lazy enablement.
3919 */
3920 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4d5f2c04 3921 test_kvm_facility(vcpu->kvm, 64) &&
bb59c2da 3922 riccb->v &&
0c9d8683 3923 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4d5f2c04 3924 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
0c9d8683 3925 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
80cd8763 3926 }
4e0b1ab7
FZ		 3927 /*
3928 * If userspace sets the gscb (e.g. after migration) to non-zero,
3929 * we should enable GS here instead of doing the lazy enablement.
3930 */
3931 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3932 test_kvm_facility(vcpu->kvm, 133) &&
3933 gscb->gssm &&
3934 !vcpu->arch.gs_enabled) {
3935 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3936 vcpu->arch.sie_block->ecb |= ECB_GS;
3937 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3938 vcpu->arch.gs_enabled = 1;
80cd8763 3939 }
35b3fde6
CB		 3940 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3941 test_kvm_facility(vcpu->kvm, 82)) {
3942 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3943 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3944 }
31d8b8d4
CB		 3945 save_access_regs(vcpu->arch.host_acrs);
3946 restore_access_regs(vcpu->run->s.regs.acrs);
e1788bb9
CB		 3947 /* save host (userspace) fprs/vrs */
3948 save_fpu_regs();
3949 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3950 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3951 if (MACHINE_HAS_VX)
3952 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3953 else
3954 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3955 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3956 if (test_fp_ctl(current->thread.fpu.fpc))
3957 /* User space provided an invalid FPC, let's clear it */
3958 current->thread.fpu.fpc = 0;
4e0b1ab7
FZ		 3959 if (MACHINE_HAS_GS) {
3960 preempt_disable();
3961 __ctl_set_bit(2, 4);
3962 if (current->thread.gs_cb) {
3963 vcpu->arch.host_gscb = current->thread.gs_cb;
3964 save_gs_cb(vcpu->arch.host_gscb);
3965 }
3966 if (vcpu->arch.gs_enabled) {
3967 current->thread.gs_cb = (struct gs_cb *)
3968 &vcpu->run->s.regs.gscb;
3969 restore_gs_cb(current->thread.gs_cb);
3970 }
3971 preempt_enable();
3972 }
a3da7b4a 3973 /* SIE will load etoken directly from SDNX and therefore kvm_run */
80cd8763 3974
b028ee3e
DH		 3975 kvm_run->kvm_dirty_regs = 0;
3976}
3977
3978static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3979{
3980 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3981 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3982 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3983 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4287f247 3984 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
b028ee3e
DH		 3985 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3986 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3987 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3988 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3989 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3990 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3991 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
35b3fde6 3992 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
31d8b8d4
CB		 3993 save_access_regs(vcpu->run->s.regs.acrs);
3994 restore_access_regs(vcpu->arch.host_acrs);
e1788bb9
CB		 3995 /* Save guest register state */
3996 save_fpu_regs();
3997 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3998 /* Restore will be done lazily at return */
3999 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
4000 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
4e0b1ab7
FZ		 4001 if (MACHINE_HAS_GS) {
4002 __ctl_set_bit(2, 4);
4003 if (vcpu->arch.gs_enabled)
4004 save_gs_cb(current->thread.gs_cb);
4005 preempt_disable();
4006 current->thread.gs_cb = vcpu->arch.host_gscb;
4007 restore_gs_cb(vcpu->arch.host_gscb);
4008 preempt_enable();
4009 if (!vcpu->arch.host_gscb)
4010 __ctl_clear_bit(2, 4);
4011 vcpu->arch.host_gscb = NULL;
4012 }
a3da7b4a 4013 /* SIE will save etoken directly into SDNX and therefore kvm_run */
b028ee3e
DH		 4014}
4015
b0c632db
HC		 4016int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4017{
8f2abe6a 4018 int rc;
b0c632db 4019
460df4c1
PB		 4020 if (kvm_run->immediate_exit)
4021 return -EINTR;
4022
200824f5
TH		 4023 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4024 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4025 return -EINVAL;
4026
accb757d
CD		 4027 vcpu_load(vcpu);
4028
27291e21
DH		 4029 if (guestdbg_exit_pending(vcpu)) {
4030 kvm_s390_prepare_debug_exit(vcpu);
accb757d
CD		 4031 rc = 0;
4032 goto out;
27291e21
DH		 4033 }
4034
20b7035c 4035 kvm_sigset_activate(vcpu);
b0c632db 4036
6352e4d2
DH		 4037 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4038 kvm_s390_vcpu_start(vcpu);
4039 } else if (is_vcpu_stopped(vcpu)) {
ea2cdd27 4040 pr_err_ratelimited("can't run stopped vcpu %d\n",
6352e4d2 4041 vcpu->vcpu_id);
accb757d
CD		 4042 rc = -EINVAL;
4043 goto out;
6352e4d2 4044 }
b0c632db 4045
b028ee3e 4046 sync_regs(vcpu, kvm_run);
db0758b2 4047 enable_cpu_timer_accounting(vcpu);
d7b0b5eb 4048
dab4079d 4049 might_fault();
a76ccff6 4050 rc = __vcpu_run(vcpu);
9ace903d 4051
b1d16c49
CE		 4052 if (signal_pending(current) && !rc) {
4053 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 4054 rc = -EINTR;
b1d16c49 4055 }
8f2abe6a 4056
27291e21
DH		 4057 if (guestdbg_exit_pending(vcpu) && !rc) {
4058 kvm_s390_prepare_debug_exit(vcpu);
4059 rc = 0;
4060 }
4061
8f2abe6a 4062 if (rc == -EREMOTE) {
71f116bf 4063 /* userspace support is needed, kvm_run has been prepared */
8f2abe6a
CB		 4064 rc = 0;
4065 }
b0c632db 4066
db0758b2 4067 disable_cpu_timer_accounting(vcpu);
b028ee3e 4068 store_regs(vcpu, kvm_run);
d7b0b5eb 4069
20b7035c 4070 kvm_sigset_deactivate(vcpu);
b0c632db 4071
b0c632db 4072 vcpu->stat.exit_userspace++;
accb757d
CD		 4073out:
4074 vcpu_put(vcpu);
7e8e6ab4 4075 return rc;
b0c632db
HC		 4076}
4077
b0c632db
HC		 4078/*
4079 * store status at address
4080 * we use have two special cases:
4081 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4082 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4083 */
d0bce605 4084int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 4085{
092670cd 4086 unsigned char archmode = 1;
9abc2a08 4087 freg_t fprs[NUM_FPRS];
fda902cb 4088 unsigned int px;
4287f247 4089 u64 clkcomp, cputm;
d0bce605 4090 int rc;
b0c632db 4091
d9a3a09a 4092 px = kvm_s390_get_prefix(vcpu);
d0bce605
HC		 4093 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4094 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 4095 return -EFAULT;
d9a3a09a 4096 gpa = 0;
d0bce605
HC		 4097 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4098 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 4099 return -EFAULT;
d9a3a09a
MS		 4100 gpa = px;
4101 } else
4102 gpa -= __LC_FPREGS_SAVE_AREA;
9abc2a08
DH		 4103
4104 /* manually convert vector registers if necessary */
4105 if (MACHINE_HAS_VX) {
9522b37f 4106 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
9abc2a08
DH		 4107 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4108 fprs, 128);
4109 } else {
4110 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
6fd8e67d 4111 vcpu->run->s.regs.fprs, 128);
9abc2a08 4112 }
d9a3a09a 4113 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
d0bce605 4114 vcpu->run->s.regs.gprs, 128);
d9a3a09a 4115 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
d0bce605 4116 &vcpu->arch.sie_block->gpsw, 16);
d9a3a09a 4117 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
fda902cb 4118 &px, 4);
d9a3a09a 4119 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
9abc2a08 4120 &vcpu->run->s.regs.fpc, 4);
d9a3a09a 4121 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
d0bce605 4122 &vcpu->arch.sie_block->todpr, 4);
4287f247 4123 cputm = kvm_s390_get_cpu_timer(vcpu);
d9a3a09a 4124 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4287f247 4125 &cputm, 8);
178bd789 4126 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d9a3a09a 4127 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
d0bce605 4128 &clkcomp, 8);
d9a3a09a 4129 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
d0bce605 4130 &vcpu->run->s.regs.acrs, 64);
d9a3a09a 4131 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
d0bce605
HC		 4132 &vcpu->arch.sie_block->gcr, 128);
4133 return rc ? -EFAULT : 0;
b0c632db
HC		 4134}
4135
e879892c
TH		 4136int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4137{
4138 /*
4139 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
31d8b8d4 4140 * switch in the run ioctl. Let's update our copies before we save
e879892c
TH		 4141 * it into the save area
4142 */
d0164ee2 4143 save_fpu_regs();
9abc2a08 4144 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
e879892c
TH		 4145 save_access_regs(vcpu->run->s.regs.acrs);
4146
4147 return kvm_s390_store_status_unloaded(vcpu, addr);
4148}
4149
8ad35755
DH		 4150static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4151{
4152 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
8e236546 4153 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
8ad35755
DH		 4154}
4155
4156static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4157{
4158 unsigned int i;
4159 struct kvm_vcpu *vcpu;
4160
4161 kvm_for_each_vcpu(i, vcpu, kvm) {
4162 __disable_ibs_on_vcpu(vcpu);
4163 }
4164}
4165
4166static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4167{
09a400e7
DH		 4168 if (!sclp.has_ibs)
4169 return;
8ad35755 4170 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
8e236546 4171 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
8ad35755
DH		 4172}
4173
6852d7b6
DH		 4174void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4175{
8ad35755
DH		 4176 int i, online_vcpus, started_vcpus = 0;
4177
4178 if (!is_vcpu_stopped(vcpu))
4179 return;
4180
6852d7b6 4181 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 4182 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4183 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4184 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4185
4186 for (i = 0; i < online_vcpus; i++) {
4187 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4188 started_vcpus++;
4189 }
4190
4191 if (started_vcpus == 0) {
4192 /* we're the only active VCPU -> speed it up */
4193 __enable_ibs_on_vcpu(vcpu);
4194 } else if (started_vcpus == 1) {
4195 /*
4196 * As we are starting a second VCPU, we have to disable
4197 * the IBS facility on all VCPUs to remove potentially
4198 * oustanding ENABLE requests.
4199 */
4200 __disable_ibs_on_all_vcpus(vcpu->kvm);
4201 }
4202
9daecfc6 4203 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
8ad35755
DH		 4204 /*
4205 * Another VCPU might have used IBS while we were offline.
4206 * Let's play safe and flush the VCPU at startup.
4207 */
d3d692c8 4208 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 4209 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 4210 return;
6852d7b6
DH		 4211}
4212
4213void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4214{
8ad35755
DH		 4215 int i, online_vcpus, started_vcpus = 0;
4216 struct kvm_vcpu *started_vcpu = NULL;
4217
4218 if (is_vcpu_stopped(vcpu))
4219 return;
4220
6852d7b6 4221 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 4222 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 4223 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 4224 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4225
32f5ff63 4226 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 4227 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 4228
ef8f4f49 4229 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
8ad35755
DH		 4230 __disable_ibs_on_vcpu(vcpu);
4231
4232 for (i = 0; i < online_vcpus; i++) {
4233 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4234 started_vcpus++;
4235 started_vcpu = vcpu->kvm->vcpus[i];
4236 }
4237 }
4238
4239 if (started_vcpus == 1) {
4240 /*
4241 * As we only have one VCPU left, we want to enable the
4242 * IBS facility for that VCPU to speed it up.
4243 */
4244 __enable_ibs_on_vcpu(started_vcpu);
4245 }
4246
433b9ee4 4247 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 4248 return;
6852d7b6
DH		 4249}
4250
d6712df9
CH		 4251static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4252 struct kvm_enable_cap *cap)
4253{
4254 int r;
4255
4256 if (cap->flags)
4257 return -EINVAL;
4258
4259 switch (cap->cap) {
fa6b7fe9
CH		 4260 case KVM_CAP_S390_CSS_SUPPORT:
4261 if (!vcpu->kvm->arch.css_support) {
4262 vcpu->kvm->arch.css_support = 1;
c92ea7b9 4263 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
fa6b7fe9
CH		 4264 trace_kvm_s390_enable_css(vcpu->kvm);
4265 }
4266 r = 0;
4267 break;
d6712df9
CH		 4268 default:
4269 r = -EINVAL;
4270 break;
4271 }
4272 return r;
4273}
4274
41408c28
TH		 4275static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4276 struct kvm_s390_mem_op *mop)
4277{
4278 void __user *uaddr = (void __user *)mop->buf;
4279 void *tmpbuf = NULL;
4280 int r, srcu_idx;
4281 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4282 | KVM_S390_MEMOP_F_CHECK_ONLY;
4283
a13b03bb 4284 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
41408c28
TH		 4285 return -EINVAL;
4286
4287 if (mop->size > MEM_OP_MAX_SIZE)
4288 return -E2BIG;
4289
4290 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4291 tmpbuf = vmalloc(mop->size);
4292 if (!tmpbuf)
4293 return -ENOMEM;
4294 }
4295
4296 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4297
4298 switch (mop->op) {
4299 case KVM_S390_MEMOP_LOGICAL_READ:
4300 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 4301 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4302 mop->size, GACC_FETCH);
41408c28
TH		 4303 break;
4304 }
4305 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4306 if (r == 0) {
4307 if (copy_to_user(uaddr, tmpbuf, mop->size))
4308 r = -EFAULT;
4309 }
4310 break;
4311 case KVM_S390_MEMOP_LOGICAL_WRITE:
4312 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
92c96321
DH		 4313 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4314 mop->size, GACC_STORE);
41408c28
TH		 4315 break;
4316 }
4317 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4318 r = -EFAULT;
4319 break;
4320 }
4321 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4322 break;
4323 default:
4324 r = -EINVAL;
4325 }
4326
4327 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4328
4329 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4330 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4331
4332 vfree(tmpbuf);
4333 return r;
4334}
4335
5cb0944c
PB		 4336long kvm_arch_vcpu_async_ioctl(struct file *filp,
4337 unsigned int ioctl, unsigned long arg)
b0c632db
HC		 4338{
4339 struct kvm_vcpu *vcpu = filp->private_data;
4340 void __user *argp = (void __user *)arg;
4341
93736624 4342 switch (ioctl) {
47b43c52
JF		 4343 case KVM_S390_IRQ: {
4344 struct kvm_s390_irq s390irq;
4345
47b43c52 4346 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
9b062471
CD		 4347 return -EFAULT;
4348 return kvm_s390_inject_vcpu(vcpu, &s390irq);
47b43c52 4349 }
93736624 4350 case KVM_S390_INTERRUPT: {
ba5c1e9b 4351 struct kvm_s390_interrupt s390int;
53936b5b 4352 struct kvm_s390_irq s390irq = {};
ba5c1e9b
CO		 4353
4354 if (copy_from_user(&s390int, argp, sizeof(s390int)))
9b062471 4355 return -EFAULT;
383d0b05
JF		 4356 if (s390int_to_s390irq(&s390int, &s390irq))
4357 return -EINVAL;
9b062471 4358 return kvm_s390_inject_vcpu(vcpu, &s390irq);
ba5c1e9b 4359 }
9b062471 4360 }
5cb0944c
PB		 4361 return -ENOIOCTLCMD;
4362}
4363
4364long kvm_arch_vcpu_ioctl(struct file *filp,
4365 unsigned int ioctl, unsigned long arg)
4366{
4367 struct kvm_vcpu *vcpu = filp->private_data;
4368 void __user *argp = (void __user *)arg;
4369 int idx;
4370 long r;
9b062471
CD		 4371
4372 vcpu_load(vcpu);
4373
4374 switch (ioctl) {
b0c632db 4375 case KVM_S390_STORE_STATUS:
800c1065 4376 idx = srcu_read_lock(&vcpu->kvm->srcu);
55680890 4377 r = kvm_s390_store_status_unloaded(vcpu, arg);
800c1065 4378 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 4379 break;
b0c632db
HC		 4380 case KVM_S390_SET_INITIAL_PSW: {
4381 psw_t psw;
4382
bc923cc9 4383 r = -EFAULT;
b0c632db 4384 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 4385 break;
4386 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4387 break;
b0c632db 4388 }
7de3f142
JF		 4389 case KVM_S390_CLEAR_RESET:
4390 r = 0;
4391 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4392 break;
b0c632db 4393 case KVM_S390_INITIAL_RESET:
7de3f142
JF		 4394 r = 0;
4395 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4396 break;
4397 case KVM_S390_NORMAL_RESET:
4398 r = 0;
4399 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
bc923cc9 4400 break;
14eebd91
CO		 4401 case KVM_SET_ONE_REG:
4402 case KVM_GET_ONE_REG: {
4403 struct kvm_one_reg reg;
4404 r = -EFAULT;
4405 if (copy_from_user(&reg, argp, sizeof(reg)))
4406 break;
4407 if (ioctl == KVM_SET_ONE_REG)
4408 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
4409 else
4410 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
4411 break;
4412 }
27e0393f
CO		 4413#ifdef CONFIG_KVM_S390_UCONTROL
4414 case KVM_S390_UCAS_MAP: {
4415 struct kvm_s390_ucas_mapping ucasmap;
4416
4417 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4418 r = -EFAULT;
4419 break;
4420 }
4421
4422 if (!kvm_is_ucontrol(vcpu->kvm)) {
4423 r = -EINVAL;
4424 break;
4425 }
4426
4427 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4428 ucasmap.vcpu_addr, ucasmap.length);
4429 break;
4430 }
4431 case KVM_S390_UCAS_UNMAP: {
4432 struct kvm_s390_ucas_mapping ucasmap;
4433
4434 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4435 r = -EFAULT;
4436 break;
4437 }
4438
4439 if (!kvm_is_ucontrol(vcpu->kvm)) {
4440 r = -EINVAL;
4441 break;
4442 }
4443
4444 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4445 ucasmap.length);
4446 break;
4447 }
4448#endif
ccc7910f 4449 case KVM_S390_VCPU_FAULT: {
527e30b4 4450 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 4451 break;
4452 }
d6712df9
CH		 4453 case KVM_ENABLE_CAP:
4454 {
4455 struct kvm_enable_cap cap;
4456 r = -EFAULT;
4457 if (copy_from_user(&cap, argp, sizeof(cap)))
4458 break;
4459 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4460 break;
4461 }
41408c28
TH		 4462 case KVM_S390_MEM_OP: {
4463 struct kvm_s390_mem_op mem_op;
4464
4465 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4466 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4467 else
4468 r = -EFAULT;
4469 break;
4470 }
816c7667
JF		 4471 case KVM_S390_SET_IRQ_STATE: {
4472 struct kvm_s390_irq_state irq_state;
4473
4474 r = -EFAULT;
4475 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4476 break;
4477 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4478 irq_state.len == 0 ||
4479 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4480 r = -EINVAL;
4481 break;
4482 }
bb64da9a 4483 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 4484 r = kvm_s390_set_irq_state(vcpu,
4485 (void __user *) irq_state.buf,
4486 irq_state.len);
4487 break;
4488 }
4489 case KVM_S390_GET_IRQ_STATE: {
4490 struct kvm_s390_irq_state irq_state;
4491
4492 r = -EFAULT;
4493 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4494 break;
4495 if (irq_state.len == 0) {
4496 r = -EINVAL;
4497 break;
4498 }
bb64da9a 4499 /* do not use irq_state.flags, it will break old QEMUs */
816c7667
JF		 4500 r = kvm_s390_get_irq_state(vcpu,
4501 (__u8 __user *) irq_state.buf,
4502 irq_state.len);
4503 break;
4504 }
b0c632db 4505 default:
3e6afcf1 4506 r = -ENOTTY;
b0c632db 4507 }
9b062471
CD		 4508
4509 vcpu_put(vcpu);
bc923cc9 4510 return r;
b0c632db
HC		 4511}
4512
1499fa80 4513vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5b1c1493
CO		 4514{
4515#ifdef CONFIG_KVM_S390_UCONTROL
4516 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4517 && (kvm_is_ucontrol(vcpu->kvm))) {
4518 vmf->page = virt_to_page(vcpu->arch.sie_block);
4519 get_page(vmf->page);
4520 return 0;
4521 }
4522#endif
4523 return VM_FAULT_SIGBUS;
4524}
4525
5587027c
AK		 4526int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4527 unsigned long npages)
db3fe4eb
TY		 4528{
4529 return 0;
4530}
4531
b0c632db 4532/* Section: memory related */
f7784b8e
MT		 4533int kvm_arch_prepare_memory_region(struct kvm *kvm,
4534 struct kvm_memory_slot *memslot,
09170a49 4535 const struct kvm_userspace_memory_region *mem,
7b6195a9 4536 enum kvm_mr_change change)
b0c632db 4537{
dd2887e7
NW		 4538 /* A few sanity checks. We can have memory slots which have to be
4539 located/ended at a segment boundary (1MB). The memory in userland is
4540 ok to be fragmented into various different vmas. It is okay to mmap()
4541 and munmap() stuff in this slot after doing this call at any time */
b0c632db 4542
598841ca 4543 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 4544 return -EINVAL;
4545
598841ca 4546 if (mem->memory_size & 0xffffful)
b0c632db
HC		 4547 return -EINVAL;
4548
a3a92c31
DD		 4549 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4550 return -EINVAL;
4551
f7784b8e
MT		 4552 return 0;
4553}
4554
4555void kvm_arch_commit_memory_region(struct kvm *kvm,
09170a49 4556 const struct kvm_userspace_memory_region *mem,
8482644a 4557 const struct kvm_memory_slot *old,
f36f3f28 4558 const struct kvm_memory_slot *new,
8482644a 4559 enum kvm_mr_change change)
f7784b8e 4560{
19ec166c 4561 int rc = 0;
598841ca 4562
19ec166c
CB		 4563 switch (change) {
4564 case KVM_MR_DELETE:
4565 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4566 old->npages * PAGE_SIZE);
4567 break;
4568 case KVM_MR_MOVE:
4569 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4570 old->npages * PAGE_SIZE);
4571 if (rc)
4572 break;
4573 /* FALLTHROUGH */
4574 case KVM_MR_CREATE:
4575 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4576 mem->guest_phys_addr, mem->memory_size);
4577 break;
4578 case KVM_MR_FLAGS_ONLY:
4579 break;
4580 default:
4581 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4582 }
598841ca 4583 if (rc)
ea2cdd27 4584 pr_warn("failed to commit memory region\n");
598841ca 4585 return;
b0c632db
HC		 4586}
4587
60a37709
AY		 4588static inline unsigned long nonhyp_mask(int i)
4589{
4590 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4591
4592 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4593}
4594
3491caf2
CB		 4595void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4596{
4597 vcpu->valid_wakeup = false;
4598}
4599
b0c632db
HC		 4600static int __init kvm_s390_init(void)
4601{
60a37709
AY		 4602 int i;
4603
07197fd0 4604 if (!sclp.has_sief2) {
8d43d570 4605 pr_info("SIE is not available\n");
07197fd0
DH		 4606 return -ENODEV;
4607 }
4608
a4499382 4609 if (nested && hpage) {
8d43d570 4610 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
a4499382
JF		 4611 return -EINVAL;
4612 }
4613
60a37709 4614 for (i = 0; i < 16; i++)
c3b9e3e1 4615 kvm_s390_fac_base[i] |=
60a37709
AY		 4616 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4617
9d8d5786 4618 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 4619}
4620
4621static void __exit kvm_s390_exit(void)
4622{
4623 kvm_exit();
4624}
4625
4626module_init(kvm_s390_init);
4627module_exit(kvm_s390_exit);
566af940
CH		 4628
4629/*
4630 * Enable autoloading of the kvm module.
4631 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4632 * since x86 takes a different approach.
4633 */
4634#include <linux/miscdevice.h>
4635MODULE_ALIAS_MISCDEV(KVM_MINOR);
4636MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)