Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
2f4a07c5 MZ |
2 | /* |
3 | * Copyright (C) 2012,2013 - ARM Ltd | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
5 | * | |
6 | * Derived from arch/arm/kvm/guest.c: | |
7 | * Copyright (C) 2012 - Virtual Open Systems and Columbia University | |
8 | * Author: Christoffer Dall <c.dall@virtualopensystems.com> | |
2f4a07c5 MZ |
9 | */ |
10 | ||
e1c9c983 | 11 | #include <linux/bits.h> |
2f4a07c5 MZ |
12 | #include <linux/errno.h> |
13 | #include <linux/err.h> | |
e1c9c983 | 14 | #include <linux/nospec.h> |
2f4a07c5 MZ |
15 | #include <linux/kvm_host.h> |
16 | #include <linux/module.h> | |
be25bbb3 | 17 | #include <linux/stddef.h> |
dc52f31a | 18 | #include <linux/string.h> |
2f4a07c5 MZ |
19 | #include <linux/vmalloc.h> |
20 | #include <linux/fs.h> | |
85fbe08e | 21 | #include <kvm/arm_hypercalls.h> |
2f4a07c5 | 22 | #include <asm/cputype.h> |
7c0f6ba6 | 23 | #include <linux/uaccess.h> |
e1c9c983 | 24 | #include <asm/fpsimd.h> |
2f4a07c5 | 25 | #include <asm/kvm.h> |
2f4a07c5 | 26 | #include <asm/kvm_emulate.h> |
e1c9c983 | 27 | #include <asm/sigcontext.h> |
2f4a07c5 | 28 | |
eef8c85a AB |
29 | #include "trace.h" |
30 | ||
fcfe1bae JZ |
31 | const struct _kvm_stats_desc kvm_vm_stats_desc[] = { |
32 | KVM_GENERIC_VM_STATS() | |
33 | }; | |
fcfe1bae JZ |
34 | |
35 | const struct kvm_stats_header kvm_vm_stats_header = { | |
36 | .name_size = KVM_STATS_NAME_SIZE, | |
37 | .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), | |
38 | .id_offset = sizeof(struct kvm_stats_header), | |
39 | .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, | |
40 | .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + | |
41 | sizeof(kvm_vm_stats_desc), | |
42 | }; | |
43 | ||
ce55c049 JZ |
44 | const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { |
45 | KVM_GENERIC_VCPU_STATS(), | |
46 | STATS_DESC_COUNTER(VCPU, hvc_exit_stat), | |
47 | STATS_DESC_COUNTER(VCPU, wfe_exit_stat), | |
48 | STATS_DESC_COUNTER(VCPU, wfi_exit_stat), | |
49 | STATS_DESC_COUNTER(VCPU, mmio_exit_user), | |
50 | STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), | |
fe5161d2 | 51 | STATS_DESC_COUNTER(VCPU, signal_exits), |
ce55c049 JZ |
52 | STATS_DESC_COUNTER(VCPU, exits) |
53 | }; | |
ce55c049 JZ |
54 | |
55 | const struct kvm_stats_header kvm_vcpu_stats_header = { | |
56 | .name_size = KVM_STATS_NAME_SIZE, | |
57 | .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), | |
58 | .id_offset = sizeof(struct kvm_stats_header), | |
59 | .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, | |
60 | .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + | |
61 | sizeof(kvm_vcpu_stats_desc), | |
62 | }; | |
63 | ||
8c86dfe3 DM |
64 | static bool core_reg_offset_is_vreg(u64 off) |
65 | { | |
66 | return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && | |
67 | off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); | |
68 | } | |
69 | ||
2f4a07c5 MZ |
70 | static u64 core_reg_offset_from_id(u64 id) |
71 | { | |
72 | return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); | |
73 | } | |
74 | ||
df205b5c | 75 | static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) |
d26c25a9 | 76 | { |
d26c25a9 DM |
77 | int size; |
78 | ||
79 | switch (off) { | |
80 | case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... | |
81 | KVM_REG_ARM_CORE_REG(regs.regs[30]): | |
82 | case KVM_REG_ARM_CORE_REG(regs.sp): | |
83 | case KVM_REG_ARM_CORE_REG(regs.pc): | |
84 | case KVM_REG_ARM_CORE_REG(regs.pstate): | |
85 | case KVM_REG_ARM_CORE_REG(sp_el1): | |
86 | case KVM_REG_ARM_CORE_REG(elr_el1): | |
87 | case KVM_REG_ARM_CORE_REG(spsr[0]) ... | |
88 | KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): | |
89 | size = sizeof(__u64); | |
90 | break; | |
91 | ||
92 | case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... | |
93 | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): | |
94 | size = sizeof(__uint128_t); | |
95 | break; | |
96 | ||
97 | case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): | |
98 | case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): | |
99 | size = sizeof(__u32); | |
100 | break; | |
101 | ||
102 | default: | |
103 | return -EINVAL; | |
104 | } | |
105 | ||
df205b5c | 106 | if (!IS_ALIGNED(off, size / sizeof(__u32))) |
8c86dfe3 | 107 | return -EINVAL; |
d26c25a9 | 108 | |
8c86dfe3 DM |
109 | /* |
110 | * The KVM_REG_ARM64_SVE regs must be used instead of | |
111 | * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on | |
112 | * SVE-enabled vcpus: | |
113 | */ | |
114 | if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) | |
115 | return -EINVAL; | |
116 | ||
df205b5c DM |
117 | return size; |
118 | } | |
119 | ||
e47c2055 | 120 | static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) |
df205b5c DM |
121 | { |
122 | u64 off = core_reg_offset_from_id(reg->id); | |
123 | int size = core_reg_size_from_offset(vcpu, off); | |
124 | ||
125 | if (size < 0) | |
e47c2055 | 126 | return NULL; |
df205b5c DM |
127 | |
128 | if (KVM_REG_SIZE(reg->id) != size) | |
e47c2055 | 129 | return NULL; |
df205b5c | 130 | |
e47c2055 MZ |
131 | switch (off) { |
132 | case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... | |
133 | KVM_REG_ARM_CORE_REG(regs.regs[30]): | |
134 | off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); | |
135 | off /= 2; | |
136 | return &vcpu->arch.ctxt.regs.regs[off]; | |
137 | ||
138 | case KVM_REG_ARM_CORE_REG(regs.sp): | |
139 | return &vcpu->arch.ctxt.regs.sp; | |
140 | ||
141 | case KVM_REG_ARM_CORE_REG(regs.pc): | |
142 | return &vcpu->arch.ctxt.regs.pc; | |
143 | ||
144 | case KVM_REG_ARM_CORE_REG(regs.pstate): | |
145 | return &vcpu->arch.ctxt.regs.pstate; | |
146 | ||
147 | case KVM_REG_ARM_CORE_REG(sp_el1): | |
1bded23e | 148 | return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); |
e47c2055 MZ |
149 | |
150 | case KVM_REG_ARM_CORE_REG(elr_el1): | |
98909e6d | 151 | return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); |
e47c2055 | 152 | |
fd85b667 | 153 | case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]): |
710f1982 | 154 | return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); |
fd85b667 MZ |
155 | |
156 | case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]): | |
157 | return &vcpu->arch.ctxt.spsr_abt; | |
158 | ||
159 | case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]): | |
160 | return &vcpu->arch.ctxt.spsr_und; | |
161 | ||
162 | case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]): | |
163 | return &vcpu->arch.ctxt.spsr_irq; | |
164 | ||
165 | case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]): | |
166 | return &vcpu->arch.ctxt.spsr_fiq; | |
e47c2055 MZ |
167 | |
168 | case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... | |
169 | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): | |
170 | off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); | |
171 | off /= 4; | |
172 | return &vcpu->arch.ctxt.fp_regs.vregs[off]; | |
173 | ||
174 | case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): | |
175 | return &vcpu->arch.ctxt.fp_regs.fpsr; | |
176 | ||
177 | case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): | |
178 | return &vcpu->arch.ctxt.fp_regs.fpcr; | |
179 | ||
180 | default: | |
181 | return NULL; | |
182 | } | |
d26c25a9 DM |
183 | } |
184 | ||
2f4a07c5 MZ |
185 | static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) |
186 | { | |
187 | /* | |
188 | * Because the kvm_regs structure is a mix of 32, 64 and | |
189 | * 128bit fields, we index it as if it was a 32bit | |
190 | * array. Hence below, nr_regs is the number of entries, and | |
191 | * off the index in the "array". | |
192 | */ | |
193 | __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; | |
e47c2055 MZ |
194 | int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); |
195 | void *addr; | |
2f4a07c5 MZ |
196 | u32 off; |
197 | ||
198 | /* Our ID is an index into the kvm_regs struct. */ | |
199 | off = core_reg_offset_from_id(reg->id); | |
200 | if (off >= nr_regs || | |
201 | (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) | |
202 | return -ENOENT; | |
203 | ||
e47c2055 MZ |
204 | addr = core_reg_addr(vcpu, reg); |
205 | if (!addr) | |
d26c25a9 DM |
206 | return -EINVAL; |
207 | ||
e47c2055 | 208 | if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id))) |
2f4a07c5 MZ |
209 | return -EFAULT; |
210 | ||
211 | return 0; | |
212 | } | |
213 | ||
214 | static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
215 | { | |
216 | __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; | |
e47c2055 | 217 | int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); |
2f4a07c5 | 218 | __uint128_t tmp; |
e47c2055 | 219 | void *valp = &tmp, *addr; |
2f4a07c5 MZ |
220 | u64 off; |
221 | int err = 0; | |
222 | ||
223 | /* Our ID is an index into the kvm_regs struct. */ | |
224 | off = core_reg_offset_from_id(reg->id); | |
225 | if (off >= nr_regs || | |
226 | (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) | |
227 | return -ENOENT; | |
228 | ||
e47c2055 MZ |
229 | addr = core_reg_addr(vcpu, reg); |
230 | if (!addr) | |
d26c25a9 DM |
231 | return -EINVAL; |
232 | ||
2f4a07c5 MZ |
233 | if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) |
234 | return -EINVAL; | |
235 | ||
236 | if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { | |
237 | err = -EFAULT; | |
238 | goto out; | |
239 | } | |
240 | ||
241 | if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { | |
2a3f9345 | 242 | u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; |
2f4a07c5 | 243 | switch (mode) { |
256c0960 | 244 | case PSR_AA32_MODE_USR: |
b10d86fb | 245 | if (!kvm_supports_32bit_el0()) |
2a3f9345 MZ |
246 | return -EINVAL; |
247 | break; | |
256c0960 MR |
248 | case PSR_AA32_MODE_FIQ: |
249 | case PSR_AA32_MODE_IRQ: | |
250 | case PSR_AA32_MODE_SVC: | |
251 | case PSR_AA32_MODE_ABT: | |
252 | case PSR_AA32_MODE_UND: | |
2a3f9345 MZ |
253 | if (!vcpu_el1_is_32bit(vcpu)) |
254 | return -EINVAL; | |
255 | break; | |
2f4a07c5 MZ |
256 | case PSR_MODE_EL0t: |
257 | case PSR_MODE_EL1t: | |
258 | case PSR_MODE_EL1h: | |
2a3f9345 MZ |
259 | if (vcpu_el1_is_32bit(vcpu)) |
260 | return -EINVAL; | |
2f4a07c5 MZ |
261 | break; |
262 | default: | |
263 | err = -EINVAL; | |
264 | goto out; | |
265 | } | |
266 | } | |
267 | ||
e47c2055 | 268 | memcpy(addr, valp, KVM_REG_SIZE(reg->id)); |
0225fd5e MZ |
269 | |
270 | if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { | |
90c1f934 MZ |
271 | int i, nr_reg; |
272 | ||
273 | switch (*vcpu_cpsr(vcpu)) { | |
274 | /* | |
275 | * Either we are dealing with user mode, and only the | |
276 | * first 15 registers (+ PC) must be narrowed to 32bit. | |
277 | * AArch32 r0-r14 conveniently map to AArch64 x0-x14. | |
278 | */ | |
279 | case PSR_AA32_MODE_USR: | |
280 | case PSR_AA32_MODE_SYS: | |
281 | nr_reg = 15; | |
282 | break; | |
283 | ||
284 | /* | |
21ea4578 | 285 | * Otherwise, this is a privileged mode, and *all* the |
90c1f934 MZ |
286 | * registers must be narrowed to 32bit. |
287 | */ | |
288 | default: | |
289 | nr_reg = 31; | |
290 | break; | |
291 | } | |
292 | ||
293 | for (i = 0; i < nr_reg; i++) | |
294 | vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i)); | |
0225fd5e | 295 | |
90c1f934 | 296 | *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu); |
0225fd5e | 297 | } |
2f4a07c5 MZ |
298 | out: |
299 | return err; | |
300 | } | |
301 | ||
9033bba4 DM |
302 | #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) |
303 | #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) | |
e644fa18 | 304 | #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) |
9033bba4 DM |
305 | |
306 | static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
307 | { | |
308 | unsigned int max_vq, vq; | |
4bd774e5 | 309 | u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; |
9033bba4 | 310 | |
52110aa9 DM |
311 | if (!vcpu_has_sve(vcpu)) |
312 | return -ENOENT; | |
313 | ||
9033bba4 DM |
314 | if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) |
315 | return -EINVAL; | |
316 | ||
317 | memset(vqs, 0, sizeof(vqs)); | |
318 | ||
468f3477 | 319 | max_vq = vcpu_sve_max_vq(vcpu); |
9033bba4 DM |
320 | for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) |
321 | if (sve_vq_available(vq)) | |
322 | vqs[vq_word(vq)] |= vq_mask(vq); | |
323 | ||
324 | if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) | |
325 | return -EFAULT; | |
326 | ||
327 | return 0; | |
328 | } | |
329 | ||
330 | static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
331 | { | |
332 | unsigned int max_vq, vq; | |
4bd774e5 | 333 | u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; |
9033bba4 | 334 | |
52110aa9 DM |
335 | if (!vcpu_has_sve(vcpu)) |
336 | return -ENOENT; | |
337 | ||
9033bba4 DM |
338 | if (kvm_arm_vcpu_sve_finalized(vcpu)) |
339 | return -EPERM; /* too late! */ | |
340 | ||
341 | if (WARN_ON(vcpu->arch.sve_state)) | |
342 | return -EINVAL; | |
343 | ||
344 | if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) | |
345 | return -EFAULT; | |
346 | ||
347 | max_vq = 0; | |
348 | for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) | |
0c529ff7 | 349 | if (vq_present(vqs, vq)) |
9033bba4 DM |
350 | max_vq = vq; |
351 | ||
352 | if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) | |
353 | return -EINVAL; | |
354 | ||
ecfb6ed4 DM |
355 | /* |
356 | * Vector lengths supported by the host can't currently be | |
357 | * hidden from the guest individually: instead we can only set a | |
656012c7 | 358 | * maximum via ZCR_EL2.LEN. So, make sure the available vector |
ecfb6ed4 DM |
359 | * lengths match the set requested exactly up to the requested |
360 | * maximum: | |
361 | */ | |
9033bba4 | 362 | for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) |
0c529ff7 | 363 | if (vq_present(vqs, vq) != sve_vq_available(vq)) |
9033bba4 DM |
364 | return -EINVAL; |
365 | ||
366 | /* Can't run with no vector lengths at all: */ | |
367 | if (max_vq < SVE_VQ_MIN) | |
368 | return -EINVAL; | |
369 | ||
370 | /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ | |
371 | vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); | |
372 | ||
373 | return 0; | |
374 | } | |
375 | ||
e1c9c983 DM |
376 | #define SVE_REG_SLICE_SHIFT 0 |
377 | #define SVE_REG_SLICE_BITS 5 | |
378 | #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) | |
379 | #define SVE_REG_ID_BITS 5 | |
380 | ||
381 | #define SVE_REG_SLICE_MASK \ | |
382 | GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ | |
383 | SVE_REG_SLICE_SHIFT) | |
384 | #define SVE_REG_ID_MASK \ | |
385 | GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) | |
386 | ||
387 | #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) | |
388 | ||
389 | #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) | |
390 | #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) | |
391 | ||
8e3c54c8 | 392 | /* |
f8d4635a DM |
393 | * Number of register slices required to cover each whole SVE register. |
394 | * NOTE: Only the first slice every exists, for now. | |
395 | * If you are tempted to modify this, you must also rework sve_reg_to_region() | |
396 | * to match: | |
8e3c54c8 DM |
397 | */ |
398 | #define vcpu_sve_slices(vcpu) 1 | |
399 | ||
e1c9c983 DM |
400 | /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ |
401 | struct sve_state_reg_region { | |
402 | unsigned int koffset; /* offset into sve_state in kernel memory */ | |
403 | unsigned int klen; /* length in kernel memory */ | |
404 | unsigned int upad; /* extra trailing padding in user memory */ | |
405 | }; | |
406 | ||
52110aa9 DM |
407 | /* |
408 | * Validate SVE register ID and get sanitised bounds for user/kernel SVE | |
409 | * register copy | |
410 | */ | |
e1c9c983 DM |
411 | static int sve_reg_to_region(struct sve_state_reg_region *region, |
412 | struct kvm_vcpu *vcpu, | |
413 | const struct kvm_one_reg *reg) | |
414 | { | |
415 | /* reg ID ranges for Z- registers */ | |
416 | const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); | |
417 | const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, | |
418 | SVE_NUM_SLICES - 1); | |
419 | ||
420 | /* reg ID ranges for P- registers and FFR (which are contiguous) */ | |
421 | const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); | |
422 | const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); | |
423 | ||
424 | unsigned int vq; | |
425 | unsigned int reg_num; | |
426 | ||
427 | unsigned int reqoffset, reqlen; /* User-requested offset and length */ | |
656012c7 | 428 | unsigned int maxlen; /* Maximum permitted length */ |
e1c9c983 DM |
429 | |
430 | size_t sve_state_size; | |
431 | ||
8ae6efdd DM |
432 | const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, |
433 | SVE_NUM_SLICES - 1); | |
434 | ||
435 | /* Verify that the P-regs and FFR really do have contiguous IDs: */ | |
436 | BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); | |
437 | ||
438 | /* Verify that we match the UAPI header: */ | |
439 | BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); | |
440 | ||
e1c9c983 DM |
441 | reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; |
442 | ||
443 | if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { | |
52110aa9 DM |
444 | if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) |
445 | return -ENOENT; | |
446 | ||
468f3477 | 447 | vq = vcpu_sve_max_vq(vcpu); |
52110aa9 | 448 | |
e1c9c983 DM |
449 | reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - |
450 | SVE_SIG_REGS_OFFSET; | |
451 | reqlen = KVM_SVE_ZREG_SIZE; | |
452 | maxlen = SVE_SIG_ZREG_SIZE(vq); | |
453 | } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { | |
52110aa9 DM |
454 | if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) |
455 | return -ENOENT; | |
456 | ||
468f3477 | 457 | vq = vcpu_sve_max_vq(vcpu); |
52110aa9 | 458 | |
e1c9c983 DM |
459 | reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - |
460 | SVE_SIG_REGS_OFFSET; | |
461 | reqlen = KVM_SVE_PREG_SIZE; | |
462 | maxlen = SVE_SIG_PREG_SIZE(vq); | |
463 | } else { | |
52110aa9 | 464 | return -EINVAL; |
e1c9c983 DM |
465 | } |
466 | ||
467 | sve_state_size = vcpu_sve_state_size(vcpu); | |
55ffad3b | 468 | if (WARN_ON(!sve_state_size)) |
e1c9c983 DM |
469 | return -EINVAL; |
470 | ||
471 | region->koffset = array_index_nospec(reqoffset, sve_state_size); | |
472 | region->klen = min(maxlen, reqlen); | |
473 | region->upad = reqlen - region->klen; | |
474 | ||
475 | return 0; | |
476 | } | |
477 | ||
478 | static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
479 | { | |
52110aa9 | 480 | int ret; |
e1c9c983 DM |
481 | struct sve_state_reg_region region; |
482 | char __user *uptr = (char __user *)reg->addr; | |
483 | ||
9033bba4 DM |
484 | /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ |
485 | if (reg->id == KVM_REG_ARM64_SVE_VLS) | |
486 | return get_sve_vls(vcpu, reg); | |
487 | ||
52110aa9 DM |
488 | /* Try to interpret reg ID as an architectural SVE register... */ |
489 | ret = sve_reg_to_region(®ion, vcpu, reg); | |
490 | if (ret) | |
491 | return ret; | |
9033bba4 DM |
492 | |
493 | if (!kvm_arm_vcpu_sve_finalized(vcpu)) | |
494 | return -EPERM; | |
495 | ||
e1c9c983 DM |
496 | if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, |
497 | region.klen) || | |
498 | clear_user(uptr + region.klen, region.upad)) | |
499 | return -EFAULT; | |
500 | ||
501 | return 0; | |
502 | } | |
503 | ||
504 | static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
505 | { | |
52110aa9 | 506 | int ret; |
e1c9c983 DM |
507 | struct sve_state_reg_region region; |
508 | const char __user *uptr = (const char __user *)reg->addr; | |
509 | ||
9033bba4 DM |
510 | /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ |
511 | if (reg->id == KVM_REG_ARM64_SVE_VLS) | |
512 | return set_sve_vls(vcpu, reg); | |
513 | ||
52110aa9 DM |
514 | /* Try to interpret reg ID as an architectural SVE register... */ |
515 | ret = sve_reg_to_region(®ion, vcpu, reg); | |
516 | if (ret) | |
517 | return ret; | |
9033bba4 DM |
518 | |
519 | if (!kvm_arm_vcpu_sve_finalized(vcpu)) | |
520 | return -EPERM; | |
521 | ||
e1c9c983 DM |
522 | if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, |
523 | region.klen)) | |
524 | return -EFAULT; | |
525 | ||
526 | return 0; | |
527 | } | |
528 | ||
2f4a07c5 MZ |
529 | int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) |
530 | { | |
531 | return -EINVAL; | |
532 | } | |
533 | ||
534 | int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |
535 | { | |
536 | return -EINVAL; | |
537 | } | |
538 | ||
8c86dfe3 DM |
539 | static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, |
540 | u64 __user *uindices) | |
be25bbb3 DM |
541 | { |
542 | unsigned int i; | |
543 | int n = 0; | |
be25bbb3 DM |
544 | |
545 | for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { | |
df205b5c DM |
546 | u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; |
547 | int size = core_reg_size_from_offset(vcpu, i); | |
548 | ||
549 | if (size < 0) | |
550 | continue; | |
551 | ||
552 | switch (size) { | |
553 | case sizeof(__u32): | |
554 | reg |= KVM_REG_SIZE_U32; | |
555 | break; | |
556 | ||
557 | case sizeof(__u64): | |
558 | reg |= KVM_REG_SIZE_U64; | |
559 | break; | |
560 | ||
561 | case sizeof(__uint128_t): | |
562 | reg |= KVM_REG_SIZE_U128; | |
563 | break; | |
564 | ||
565 | default: | |
566 | WARN_ON(1); | |
8c86dfe3 | 567 | continue; |
df205b5c | 568 | } |
8c86dfe3 | 569 | |
be25bbb3 | 570 | if (uindices) { |
df205b5c | 571 | if (put_user(reg, uindices)) |
be25bbb3 DM |
572 | return -EFAULT; |
573 | uindices++; | |
574 | } | |
575 | ||
576 | n++; | |
577 | } | |
578 | ||
579 | return n; | |
580 | } | |
581 | ||
8c86dfe3 | 582 | static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) |
2f4a07c5 | 583 | { |
8c86dfe3 | 584 | return copy_core_reg_indices(vcpu, NULL); |
2f4a07c5 MZ |
585 | } |
586 | ||
1df08ba0 AB |
587 | /** |
588 | * ARM64 versions of the TIMER registers, always available on arm64 | |
589 | */ | |
590 | ||
591 | #define NUM_TIMER_REGS 3 | |
592 | ||
593 | static bool is_timer_reg(u64 index) | |
594 | { | |
595 | switch (index) { | |
596 | case KVM_REG_ARM_TIMER_CTL: | |
597 | case KVM_REG_ARM_TIMER_CNT: | |
598 | case KVM_REG_ARM_TIMER_CVAL: | |
599 | return true; | |
600 | } | |
601 | return false; | |
602 | } | |
603 | ||
604 | static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) | |
605 | { | |
606 | if (put_user(KVM_REG_ARM_TIMER_CTL, uindices)) | |
607 | return -EFAULT; | |
608 | uindices++; | |
609 | if (put_user(KVM_REG_ARM_TIMER_CNT, uindices)) | |
610 | return -EFAULT; | |
611 | uindices++; | |
612 | if (put_user(KVM_REG_ARM_TIMER_CVAL, uindices)) | |
613 | return -EFAULT; | |
614 | ||
615 | return 0; | |
616 | } | |
617 | ||
618 | static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
619 | { | |
620 | void __user *uaddr = (void __user *)(long)reg->addr; | |
621 | u64 val; | |
622 | int ret; | |
623 | ||
624 | ret = copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)); | |
625 | if (ret != 0) | |
bd218bce | 626 | return -EFAULT; |
1df08ba0 AB |
627 | |
628 | return kvm_arm_timer_set_reg(vcpu, reg->id, val); | |
629 | } | |
630 | ||
631 | static int get_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
632 | { | |
633 | void __user *uaddr = (void __user *)(long)reg->addr; | |
634 | u64 val; | |
635 | ||
636 | val = kvm_arm_timer_get_reg(vcpu, reg->id); | |
4cad67fc | 637 | return copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)) ? -EFAULT : 0; |
1df08ba0 AB |
638 | } |
639 | ||
8e3c54c8 DM |
640 | static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) |
641 | { | |
8e3c54c8 DM |
642 | const unsigned int slices = vcpu_sve_slices(vcpu); |
643 | ||
644 | if (!vcpu_has_sve(vcpu)) | |
645 | return 0; | |
646 | ||
9033bba4 DM |
647 | /* Policed by KVM_GET_REG_LIST: */ |
648 | WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); | |
649 | ||
650 | return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) | |
651 | + 1; /* KVM_REG_ARM64_SVE_VLS */ | |
8e3c54c8 DM |
652 | } |
653 | ||
654 | static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, | |
655 | u64 __user *uindices) | |
656 | { | |
8e3c54c8 DM |
657 | const unsigned int slices = vcpu_sve_slices(vcpu); |
658 | u64 reg; | |
659 | unsigned int i, n; | |
660 | int num_regs = 0; | |
661 | ||
662 | if (!vcpu_has_sve(vcpu)) | |
663 | return 0; | |
664 | ||
9033bba4 DM |
665 | /* Policed by KVM_GET_REG_LIST: */ |
666 | WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); | |
667 | ||
668 | /* | |
669 | * Enumerate this first, so that userspace can save/restore in | |
670 | * the order reported by KVM_GET_REG_LIST: | |
671 | */ | |
672 | reg = KVM_REG_ARM64_SVE_VLS; | |
673 | if (put_user(reg, uindices++)) | |
674 | return -EFAULT; | |
9033bba4 DM |
675 | ++num_regs; |
676 | ||
8e3c54c8 DM |
677 | for (i = 0; i < slices; i++) { |
678 | for (n = 0; n < SVE_NUM_ZREGS; n++) { | |
679 | reg = KVM_REG_ARM64_SVE_ZREG(n, i); | |
680 | if (put_user(reg, uindices++)) | |
681 | return -EFAULT; | |
8e3c54c8 DM |
682 | num_regs++; |
683 | } | |
684 | ||
685 | for (n = 0; n < SVE_NUM_PREGS; n++) { | |
686 | reg = KVM_REG_ARM64_SVE_PREG(n, i); | |
687 | if (put_user(reg, uindices++)) | |
688 | return -EFAULT; | |
8e3c54c8 DM |
689 | num_regs++; |
690 | } | |
691 | ||
692 | reg = KVM_REG_ARM64_SVE_FFR(i); | |
693 | if (put_user(reg, uindices++)) | |
694 | return -EFAULT; | |
8e3c54c8 DM |
695 | num_regs++; |
696 | } | |
697 | ||
698 | return num_regs; | |
699 | } | |
700 | ||
2f4a07c5 MZ |
701 | /** |
702 | * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG | |
703 | * | |
704 | * This is for all registers. | |
705 | */ | |
706 | unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) | |
707 | { | |
7aa92cf3 DM |
708 | unsigned long res = 0; |
709 | ||
8c86dfe3 | 710 | res += num_core_regs(vcpu); |
8e3c54c8 | 711 | res += num_sve_regs(vcpu); |
7aa92cf3 DM |
712 | res += kvm_arm_num_sys_reg_descs(vcpu); |
713 | res += kvm_arm_get_fw_num_regs(vcpu); | |
714 | res += NUM_TIMER_REGS; | |
715 | ||
716 | return res; | |
2f4a07c5 MZ |
717 | } |
718 | ||
719 | /** | |
720 | * kvm_arm_copy_reg_indices - get indices of all registers. | |
721 | * | |
edce2292 | 722 | * We do core registers right here, then we append system regs. |
2f4a07c5 MZ |
723 | */ |
724 | int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) | |
725 | { | |
1df08ba0 | 726 | int ret; |
2f4a07c5 | 727 | |
8c86dfe3 | 728 | ret = copy_core_reg_indices(vcpu, uindices); |
5d8d4af2 | 729 | if (ret < 0) |
be25bbb3 DM |
730 | return ret; |
731 | uindices += ret; | |
2f4a07c5 | 732 | |
8e3c54c8 | 733 | ret = copy_sve_reg_indices(vcpu, uindices); |
5d8d4af2 | 734 | if (ret < 0) |
8e3c54c8 DM |
735 | return ret; |
736 | uindices += ret; | |
737 | ||
85bd0ba1 | 738 | ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); |
5d8d4af2 | 739 | if (ret < 0) |
85bd0ba1 MZ |
740 | return ret; |
741 | uindices += kvm_arm_get_fw_num_regs(vcpu); | |
742 | ||
1df08ba0 | 743 | ret = copy_timer_indices(vcpu, uindices); |
5d8d4af2 | 744 | if (ret < 0) |
1df08ba0 AB |
745 | return ret; |
746 | uindices += NUM_TIMER_REGS; | |
747 | ||
2f4a07c5 MZ |
748 | return kvm_arm_copy_sys_reg_indices(vcpu, uindices); |
749 | } | |
750 | ||
751 | int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
752 | { | |
753 | /* We currently use nothing arch-specific in upper 32 bits */ | |
754 | if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) | |
755 | return -EINVAL; | |
756 | ||
e1c9c983 DM |
757 | switch (reg->id & KVM_REG_ARM_COPROC_MASK) { |
758 | case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); | |
05714cab RRA |
759 | case KVM_REG_ARM_FW: |
760 | case KVM_REG_ARM_FW_FEAT_BMAP: | |
761 | return kvm_arm_get_fw_reg(vcpu, reg); | |
e1c9c983 | 762 | case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); |
e1c9c983 | 763 | } |
85bd0ba1 | 764 | |
1df08ba0 AB |
765 | if (is_timer_reg(reg->id)) |
766 | return get_timer_reg(vcpu, reg); | |
767 | ||
2f4a07c5 MZ |
768 | return kvm_arm_sys_reg_get_reg(vcpu, reg); |
769 | } | |
770 | ||
771 | int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) | |
772 | { | |
773 | /* We currently use nothing arch-specific in upper 32 bits */ | |
774 | if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) | |
775 | return -EINVAL; | |
776 | ||
e1c9c983 DM |
777 | switch (reg->id & KVM_REG_ARM_COPROC_MASK) { |
778 | case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); | |
05714cab RRA |
779 | case KVM_REG_ARM_FW: |
780 | case KVM_REG_ARM_FW_FEAT_BMAP: | |
781 | return kvm_arm_set_fw_reg(vcpu, reg); | |
e1c9c983 | 782 | case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); |
e1c9c983 | 783 | } |
85bd0ba1 | 784 | |
1df08ba0 AB |
785 | if (is_timer_reg(reg->id)) |
786 | return set_timer_reg(vcpu, reg); | |
787 | ||
2f4a07c5 MZ |
788 | return kvm_arm_sys_reg_set_reg(vcpu, reg); |
789 | } | |
790 | ||
791 | int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, | |
792 | struct kvm_sregs *sregs) | |
793 | { | |
794 | return -EINVAL; | |
795 | } | |
796 | ||
797 | int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |
798 | struct kvm_sregs *sregs) | |
799 | { | |
800 | return -EINVAL; | |
801 | } | |
802 | ||
539aee0e JM |
803 | int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, |
804 | struct kvm_vcpu_events *events) | |
b7b27fac | 805 | { |
b7b27fac DG |
806 | events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE); |
807 | events->exception.serror_has_esr = cpus_have_const_cap(ARM64_HAS_RAS_EXTN); | |
808 | ||
809 | if (events->exception.serror_pending && events->exception.serror_has_esr) | |
810 | events->exception.serror_esr = vcpu_get_vsesr(vcpu); | |
811 | ||
da345174 CD |
812 | /* |
813 | * We never return a pending ext_dabt here because we deliver it to | |
814 | * the virtual CPU directly when setting the event and it's no longer | |
815 | * 'pending' at this point. | |
816 | */ | |
817 | ||
b7b27fac DG |
818 | return 0; |
819 | } | |
820 | ||
539aee0e JM |
821 | int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, |
822 | struct kvm_vcpu_events *events) | |
b7b27fac | 823 | { |
b7b27fac DG |
824 | bool serror_pending = events->exception.serror_pending; |
825 | bool has_esr = events->exception.serror_has_esr; | |
da345174 | 826 | bool ext_dabt_pending = events->exception.ext_dabt_pending; |
b7b27fac | 827 | |
b7b27fac DG |
828 | if (serror_pending && has_esr) { |
829 | if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) | |
830 | return -EINVAL; | |
831 | ||
832 | if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK)) | |
833 | kvm_set_sei_esr(vcpu, events->exception.serror_esr); | |
834 | else | |
835 | return -EINVAL; | |
836 | } else if (serror_pending) { | |
837 | kvm_inject_vabt(vcpu); | |
838 | } | |
839 | ||
da345174 CD |
840 | if (ext_dabt_pending) |
841 | kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); | |
842 | ||
b7b27fac DG |
843 | return 0; |
844 | } | |
845 | ||
6b7982fe | 846 | u32 __attribute_const__ kvm_target_cpu(void) |
2f4a07c5 MZ |
847 | { |
848 | unsigned long implementor = read_cpuid_implementor(); | |
849 | unsigned long part_number = read_cpuid_part_number(); | |
850 | ||
e28100bd AP |
851 | switch (implementor) { |
852 | case ARM_CPU_IMP_ARM: | |
853 | switch (part_number) { | |
854 | case ARM_CPU_PART_AEM_V8: | |
855 | return KVM_ARM_TARGET_AEM_V8; | |
856 | case ARM_CPU_PART_FOUNDATION: | |
857 | return KVM_ARM_TARGET_FOUNDATION_V8; | |
1252b331 MZ |
858 | case ARM_CPU_PART_CORTEX_A53: |
859 | return KVM_ARM_TARGET_CORTEX_A53; | |
e28100bd AP |
860 | case ARM_CPU_PART_CORTEX_A57: |
861 | return KVM_ARM_TARGET_CORTEX_A57; | |
f0725345 | 862 | } |
e28100bd AP |
863 | break; |
864 | case ARM_CPU_IMP_APM: | |
865 | switch (part_number) { | |
866 | case APM_CPU_PART_POTENZA: | |
867 | return KVM_ARM_TARGET_XGENE_POTENZA; | |
f0725345 | 868 | } |
e28100bd | 869 | break; |
f0725345 | 870 | } |
2f4a07c5 | 871 | |
bca556ac SP |
872 | /* Return a default generic target */ |
873 | return KVM_ARM_TARGET_GENERIC_V8; | |
2f4a07c5 MZ |
874 | } |
875 | ||
08e873cb | 876 | void kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) |
473bdc0e | 877 | { |
6b7982fe | 878 | u32 target = kvm_target_cpu(); |
473bdc0e | 879 | |
473bdc0e AP |
880 | memset(init, 0, sizeof(*init)); |
881 | ||
882 | /* | |
883 | * For now, we don't return any features. | |
884 | * In future, we might use features to return target | |
885 | * specific features available for the preferred | |
886 | * target type. | |
887 | */ | |
888 | init->target = (__u32)target; | |
473bdc0e AP |
889 | } |
890 | ||
2f4a07c5 MZ |
891 | int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) |
892 | { | |
893 | return -EINVAL; | |
894 | } | |
895 | ||
896 | int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) | |
897 | { | |
898 | return -EINVAL; | |
899 | } | |
900 | ||
901 | int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, | |
902 | struct kvm_translation *tr) | |
903 | { | |
904 | return -EINVAL; | |
905 | } | |
0e6f07f2 | 906 | |
0e6f07f2 AB |
907 | /** |
908 | * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging | |
909 | * @kvm: pointer to the KVM struct | |
910 | * @kvm_guest_debug: the ioctl data buffer | |
911 | * | |
912 | * This sets up and enables the VM for guest debugging. Userspace | |
913 | * passes in a control flag to enable different debug types and | |
914 | * potentially other architecture specific information in the rest of | |
915 | * the structure. | |
916 | */ | |
917 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, | |
918 | struct kvm_guest_debug *dbg) | |
919 | { | |
66b56562 CD |
920 | int ret = 0; |
921 | ||
eef8c85a AB |
922 | trace_kvm_set_guest_debug(vcpu, dbg->control); |
923 | ||
66b56562 CD |
924 | if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) { |
925 | ret = -EINVAL; | |
926 | goto out; | |
927 | } | |
0e6f07f2 AB |
928 | |
929 | if (dbg->control & KVM_GUESTDBG_ENABLE) { | |
930 | vcpu->guest_debug = dbg->control; | |
834bf887 AB |
931 | |
932 | /* Hardware assisted Break and Watch points */ | |
933 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { | |
934 | vcpu->arch.external_debug_state = dbg->arch; | |
935 | } | |
936 | ||
0e6f07f2 AB |
937 | } else { |
938 | /* If not enabled clear all flags */ | |
939 | vcpu->guest_debug = 0; | |
370531d1 | 940 | vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING); |
0e6f07f2 | 941 | } |
66b56562 CD |
942 | |
943 | out: | |
66b56562 | 944 | return ret; |
0e6f07f2 | 945 | } |
bb0c70bc SZ |
946 | |
947 | int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, | |
948 | struct kvm_device_attr *attr) | |
949 | { | |
950 | int ret; | |
951 | ||
952 | switch (attr->group) { | |
953 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
954 | ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); | |
955 | break; | |
99a1db7a CD |
956 | case KVM_ARM_VCPU_TIMER_CTRL: |
957 | ret = kvm_arm_timer_set_attr(vcpu, attr); | |
958 | break; | |
58772e9a SP |
959 | case KVM_ARM_VCPU_PVTIME_CTRL: |
960 | ret = kvm_arm_pvtime_set_attr(vcpu, attr); | |
961 | break; | |
bb0c70bc SZ |
962 | default: |
963 | ret = -ENXIO; | |
964 | break; | |
965 | } | |
966 | ||
967 | return ret; | |
968 | } | |
969 | ||
970 | int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, | |
971 | struct kvm_device_attr *attr) | |
972 | { | |
973 | int ret; | |
974 | ||
975 | switch (attr->group) { | |
976 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
977 | ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); | |
978 | break; | |
99a1db7a CD |
979 | case KVM_ARM_VCPU_TIMER_CTRL: |
980 | ret = kvm_arm_timer_get_attr(vcpu, attr); | |
981 | break; | |
58772e9a SP |
982 | case KVM_ARM_VCPU_PVTIME_CTRL: |
983 | ret = kvm_arm_pvtime_get_attr(vcpu, attr); | |
984 | break; | |
bb0c70bc SZ |
985 | default: |
986 | ret = -ENXIO; | |
987 | break; | |
988 | } | |
989 | ||
990 | return ret; | |
991 | } | |
992 | ||
993 | int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, | |
994 | struct kvm_device_attr *attr) | |
995 | { | |
996 | int ret; | |
997 | ||
998 | switch (attr->group) { | |
999 | case KVM_ARM_VCPU_PMU_V3_CTRL: | |
1000 | ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); | |
1001 | break; | |
99a1db7a CD |
1002 | case KVM_ARM_VCPU_TIMER_CTRL: |
1003 | ret = kvm_arm_timer_has_attr(vcpu, attr); | |
1004 | break; | |
58772e9a SP |
1005 | case KVM_ARM_VCPU_PVTIME_CTRL: |
1006 | ret = kvm_arm_pvtime_has_attr(vcpu, attr); | |
1007 | break; | |
bb0c70bc SZ |
1008 | default: |
1009 | ret = -ENXIO; | |
1010 | break; | |
1011 | } | |
1012 | ||
1013 | return ret; | |
1014 | } | |
f0376edb SP |
1015 | |
1016 | long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, | |
1017 | struct kvm_arm_copy_mte_tags *copy_tags) | |
1018 | { | |
1019 | gpa_t guest_ipa = copy_tags->guest_ipa; | |
1020 | size_t length = copy_tags->length; | |
1021 | void __user *tags = copy_tags->addr; | |
1022 | gpa_t gfn; | |
1023 | bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST); | |
1024 | int ret = 0; | |
1025 | ||
1026 | if (!kvm_has_mte(kvm)) | |
1027 | return -EINVAL; | |
1028 | ||
1029 | if (copy_tags->reserved[0] || copy_tags->reserved[1]) | |
1030 | return -EINVAL; | |
1031 | ||
1032 | if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST) | |
1033 | return -EINVAL; | |
1034 | ||
1035 | if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK) | |
1036 | return -EINVAL; | |
1037 | ||
1038 | gfn = gpa_to_gfn(guest_ipa); | |
1039 | ||
1040 | mutex_lock(&kvm->slots_lock); | |
1041 | ||
1042 | while (length > 0) { | |
1043 | kvm_pfn_t pfn = gfn_to_pfn_prot(kvm, gfn, write, NULL); | |
1044 | void *maddr; | |
1045 | unsigned long num_tags; | |
1046 | struct page *page; | |
1047 | ||
1048 | if (is_error_noslot_pfn(pfn)) { | |
1049 | ret = -EFAULT; | |
1050 | goto out; | |
1051 | } | |
1052 | ||
1053 | page = pfn_to_online_page(pfn); | |
1054 | if (!page) { | |
1055 | /* Reject ZONE_DEVICE memory */ | |
1056 | ret = -EFAULT; | |
1057 | goto out; | |
1058 | } | |
1059 | maddr = page_address(page); | |
1060 | ||
1061 | if (!write) { | |
1062 | if (test_bit(PG_mte_tagged, &page->flags)) | |
1063 | num_tags = mte_copy_tags_to_user(tags, maddr, | |
1064 | MTE_GRANULES_PER_PAGE); | |
1065 | else | |
1066 | /* No tags in memory, so write zeros */ | |
1067 | num_tags = MTE_GRANULES_PER_PAGE - | |
1068 | clear_user(tags, MTE_GRANULES_PER_PAGE); | |
1069 | kvm_release_pfn_clean(pfn); | |
1070 | } else { | |
1071 | num_tags = mte_copy_tags_from_user(maddr, tags, | |
1072 | MTE_GRANULES_PER_PAGE); | |
98db7259 MZ |
1073 | |
1074 | /* | |
1075 | * Set the flag after checking the write | |
1076 | * completed fully | |
1077 | */ | |
1078 | if (num_tags == MTE_GRANULES_PER_PAGE) | |
1079 | set_bit(PG_mte_tagged, &page->flags); | |
1080 | ||
f0376edb SP |
1081 | kvm_release_pfn_dirty(pfn); |
1082 | } | |
1083 | ||
1084 | if (num_tags != MTE_GRANULES_PER_PAGE) { | |
1085 | ret = -EFAULT; | |
1086 | goto out; | |
1087 | } | |
1088 | ||
f0376edb SP |
1089 | gfn++; |
1090 | tags += num_tags; | |
1091 | length -= PAGE_SIZE; | |
1092 | } | |
1093 | ||
1094 | out: | |
1095 | mutex_unlock(&kvm->slots_lock); | |
1096 | /* If some data has been copied report the number of bytes copied */ | |
1097 | if (length != copy_tags->length) | |
1098 | return copy_tags->length - length; | |
1099 | return ret; | |
1100 | } |