Commit | Line | Data |
---|---|---|
7a6629ef AJ |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * AArch64 code | |
4 | * | |
5 | * Copyright (C) 2018, Red Hat, Inc. | |
6 | */ | |
7 | ||
98e68344 | 8 | #include <linux/compiler.h> |
e3db7579 | 9 | #include <assert.h> |
98e68344 | 10 | |
0303ffdb | 11 | #include "guest_modes.h" |
7a6629ef | 12 | #include "kvm_util.h" |
0bec140f | 13 | #include "processor.h" |
2b7deea3 SC |
14 | #include "ucall_common.h" |
15 | ||
1a618203 | 16 | #include <linux/bitfield.h> |
10a0cc3b | 17 | #include <linux/sizes.h> |
7a6629ef | 18 | |
0bec140f | 19 | #define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000 |
7a6629ef | 20 | |
e3db7579 RK |
21 | static vm_vaddr_t exception_handlers; |
22 | ||
7a6629ef AJ |
23 | static uint64_t page_align(struct kvm_vm *vm, uint64_t v) |
24 | { | |
25 | return (v + vm->page_size) & ~(vm->page_size - 1); | |
26 | } | |
27 | ||
28 | static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva) | |
29 | { | |
30 | unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift; | |
31 | uint64_t mask = (1UL << (vm->va_bits - shift)) - 1; | |
32 | ||
33 | return (gva >> shift) & mask; | |
34 | } | |
35 | ||
36 | static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva) | |
37 | { | |
38 | unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift; | |
39 | uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; | |
40 | ||
41 | TEST_ASSERT(vm->pgtable_levels == 4, | |
42 | "Mode %d does not have 4 page table levels", vm->mode); | |
43 | ||
44 | return (gva >> shift) & mask; | |
45 | } | |
46 | ||
47 | static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva) | |
48 | { | |
49 | unsigned int shift = (vm->page_shift - 3) + vm->page_shift; | |
50 | uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; | |
51 | ||
52 | TEST_ASSERT(vm->pgtable_levels >= 3, | |
53 | "Mode %d does not have >= 3 page table levels", vm->mode); | |
54 | ||
55 | return (gva >> shift) & mask; | |
56 | } | |
57 | ||
58 | static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva) | |
59 | { | |
60 | uint64_t mask = (1UL << (vm->page_shift - 3)) - 1; | |
61 | return (gva >> vm->page_shift) & mask; | |
62 | } | |
63 | ||
10a0cc3b RR |
64 | static inline bool use_lpa2_pte_format(struct kvm_vm *vm) |
65 | { | |
66 | return (vm->page_size == SZ_4K || vm->page_size == SZ_16K) && | |
67 | (vm->pa_bits > 48 || vm->va_bits > 48); | |
68 | } | |
69 | ||
e659babf | 70 | static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs) |
7a6629ef | 71 | { |
e659babf RR |
72 | uint64_t pte; |
73 | ||
10a0cc3b RR |
74 | if (use_lpa2_pte_format(vm)) { |
75 | pte = pa & GENMASK(49, vm->page_shift); | |
76 | pte |= FIELD_GET(GENMASK(51, 50), pa) << 8; | |
77 | attrs &= ~GENMASK(9, 8); | |
78 | } else { | |
79 | pte = pa & GENMASK(47, vm->page_shift); | |
80 | if (vm->page_shift == 16) | |
81 | pte |= FIELD_GET(GENMASK(51, 48), pa) << 12; | |
82 | } | |
e659babf RR |
83 | pte |= attrs; |
84 | ||
85 | return pte; | |
86 | } | |
87 | ||
88 | static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte) | |
89 | { | |
90 | uint64_t pa; | |
91 | ||
10a0cc3b RR |
92 | if (use_lpa2_pte_format(vm)) { |
93 | pa = pte & GENMASK(49, vm->page_shift); | |
94 | pa |= FIELD_GET(GENMASK(9, 8), pte) << 50; | |
95 | } else { | |
96 | pa = pte & GENMASK(47, vm->page_shift); | |
97 | if (vm->page_shift == 16) | |
98 | pa |= FIELD_GET(GENMASK(15, 12), pte) << 48; | |
99 | } | |
e659babf RR |
100 | |
101 | return pa; | |
7a6629ef AJ |
102 | } |
103 | ||
104 | static uint64_t ptrs_per_pgd(struct kvm_vm *vm) | |
105 | { | |
106 | unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift; | |
107 | return 1 << (vm->va_bits - shift); | |
108 | } | |
109 | ||
98e68344 | 110 | static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm) |
7a6629ef AJ |
111 | { |
112 | return 1 << (vm->page_shift - 3); | |
113 | } | |
114 | ||
9931be3f | 115 | void virt_arch_pgd_alloc(struct kvm_vm *vm) |
7a6629ef | 116 | { |
5485e822 RK |
117 | size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size; |
118 | ||
119 | if (vm->pgd_created) | |
120 | return; | |
121 | ||
122 | vm->pgd = vm_phy_pages_alloc(vm, nr_pages, | |
1446e331 RK |
123 | KVM_GUEST_PAGE_TABLE_MIN_PADDR, |
124 | vm->memslots[MEM_REGION_PT]); | |
5485e822 | 125 | vm->pgd_created = true; |
7a6629ef AJ |
126 | } |
127 | ||
4307af73 SC |
128 | static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, |
129 | uint64_t flags) | |
7a6629ef AJ |
130 | { |
131 | uint8_t attr_idx = flags & 7; | |
132 | uint64_t *ptep; | |
133 | ||
134 | TEST_ASSERT((vaddr % vm->page_size) == 0, | |
135 | "Virtual address not on page boundary,\n" | |
136 | " vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size); | |
137 | TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, | |
138 | (vaddr >> vm->page_shift)), | |
139 | "Invalid virtual address, vaddr: 0x%lx", vaddr); | |
140 | TEST_ASSERT((paddr % vm->page_size) == 0, | |
141 | "Physical address not on page boundary,\n" | |
142 | " paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size); | |
143 | TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, | |
144 | "Physical address beyond beyond maximum supported,\n" | |
145 | " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", | |
146 | paddr, vm->max_gfn, vm->page_size); | |
147 | ||
148 | ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; | |
cce0c23d | 149 | if (!*ptep) |
e659babf | 150 | *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); |
7a6629ef AJ |
151 | |
152 | switch (vm->pgtable_levels) { | |
153 | case 4: | |
154 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; | |
cce0c23d | 155 | if (!*ptep) |
e659babf | 156 | *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); |
7a6629ef AJ |
157 | /* fall through */ |
158 | case 3: | |
159 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; | |
cce0c23d | 160 | if (!*ptep) |
e659babf | 161 | *ptep = addr_pte(vm, vm_alloc_page_table(vm), 3); |
7a6629ef AJ |
162 | /* fall through */ |
163 | case 2: | |
164 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8; | |
165 | break; | |
166 | default: | |
352be2c5 | 167 | TEST_FAIL("Page table levels must be 2, 3, or 4"); |
7a6629ef AJ |
168 | } |
169 | ||
e659babf | 170 | *ptep = addr_pte(vm, paddr, (attr_idx << 2) | (1 << 10) | 3); /* AF */ |
7a6629ef AJ |
171 | } |
172 | ||
9931be3f | 173 | void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) |
7a6629ef | 174 | { |
41f5189e | 175 | uint64_t attr_idx = MT_NORMAL; |
7a6629ef | 176 | |
4307af73 | 177 | _virt_pg_map(vm, vaddr, paddr, attr_idx); |
7a6629ef AJ |
178 | } |
179 | ||
228f324d | 180 | uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva) |
7a6629ef AJ |
181 | { |
182 | uint64_t *ptep; | |
183 | ||
184 | if (!vm->pgd_created) | |
185 | goto unmapped_gva; | |
186 | ||
187 | ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8; | |
188 | if (!ptep) | |
189 | goto unmapped_gva; | |
190 | ||
191 | switch (vm->pgtable_levels) { | |
192 | case 4: | |
193 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8; | |
194 | if (!ptep) | |
195 | goto unmapped_gva; | |
196 | /* fall through */ | |
197 | case 3: | |
198 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8; | |
199 | if (!ptep) | |
200 | goto unmapped_gva; | |
201 | /* fall through */ | |
202 | case 2: | |
203 | ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8; | |
204 | if (!ptep) | |
205 | goto unmapped_gva; | |
206 | break; | |
207 | default: | |
352be2c5 | 208 | TEST_FAIL("Page table levels must be 2, 3, or 4"); |
7a6629ef AJ |
209 | } |
210 | ||
228f324d | 211 | return ptep; |
7a6629ef AJ |
212 | |
213 | unmapped_gva: | |
352be2c5 | 214 | TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); |
228f324d RK |
215 | exit(EXIT_FAILURE); |
216 | } | |
217 | ||
218 | vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) | |
219 | { | |
220 | uint64_t *ptep = virt_get_pte_hva(vm, gva); | |
221 | ||
222 | return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1)); | |
7a6629ef AJ |
223 | } |
224 | ||
225 | static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level) | |
226 | { | |
3439d886 | 227 | #ifdef DEBUG |
7a6629ef AJ |
228 | static const char * const type[] = { "", "pud", "pmd", "pte" }; |
229 | uint64_t pte, *ptep; | |
230 | ||
231 | if (level == 4) | |
232 | return; | |
233 | ||
234 | for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) { | |
235 | ptep = addr_gpa2hva(vm, pte); | |
236 | if (!*ptep) | |
237 | continue; | |
f09ab268 | 238 | fprintf(stream, "%*s%s: %lx: %lx at %p\n", indent, "", type[level], pte, *ptep, ptep); |
7a6629ef AJ |
239 | pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1); |
240 | } | |
241 | #endif | |
242 | } | |
243 | ||
9931be3f | 244 | void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) |
7a6629ef AJ |
245 | { |
246 | int level = 4 - (vm->pgtable_levels - 1); | |
247 | uint64_t pgd, *ptep; | |
248 | ||
249 | if (!vm->pgd_created) | |
250 | return; | |
251 | ||
252 | for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) { | |
253 | ptep = addr_gpa2hva(vm, pgd); | |
254 | if (!*ptep) | |
255 | continue; | |
f09ab268 | 256 | fprintf(stream, "%*spgd: %lx: %lx at %p\n", indent, "", pgd, *ptep, ptep); |
7a6629ef AJ |
257 | pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level); |
258 | } | |
259 | } | |
0bec140f | 260 | |
768e9a61 | 261 | void aarch64_vcpu_setup(struct kvm_vcpu *vcpu, struct kvm_vcpu_init *init) |
0bec140f | 262 | { |
f5dd4ccf | 263 | struct kvm_vcpu_init default_init = { .target = -1, }; |
768e9a61 | 264 | struct kvm_vm *vm = vcpu->vm; |
e1707175 | 265 | uint64_t sctlr_el1, tcr_el1, ttbr0_el1; |
0bec140f | 266 | |
f5dd4ccf AJ |
267 | if (!init) |
268 | init = &default_init; | |
269 | ||
270 | if (init->target == -1) { | |
271 | struct kvm_vcpu_init preferred; | |
272 | vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred); | |
273 | init->target = preferred.target; | |
274 | } | |
275 | ||
768e9a61 | 276 | vcpu_ioctl(vcpu, KVM_ARM_VCPU_INIT, init); |
0bec140f AJ |
277 | |
278 | /* | |
279 | * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15 | |
280 | * registers, which the variable argument list macros do. | |
281 | */ | |
768e9a61 | 282 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20); |
0bec140f | 283 | |
768e9a61 SC |
284 | vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), &sctlr_el1); |
285 | vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), &tcr_el1); | |
0bec140f | 286 | |
2f41a61c | 287 | /* Configure base granule size */ |
0bec140f | 288 | switch (vm->mode) { |
567a9f1e | 289 | case VM_MODE_PXXV48_4K: |
352be2c5 WSM |
290 | TEST_FAIL("AArch64 does not support 4K sized pages " |
291 | "with ANY-bit physical address ranges"); | |
81d1cca0 | 292 | case VM_MODE_P52V48_64K: |
2f41a61c MZ |
293 | case VM_MODE_P48V48_64K: |
294 | case VM_MODE_P40V48_64K: | |
e7f58a6b | 295 | case VM_MODE_P36V48_64K: |
81d1cca0 | 296 | tcr_el1 |= 1ul << 14; /* TG0 = 64KB */ |
81d1cca0 | 297 | break; |
10a0cc3b | 298 | case VM_MODE_P52V48_16K: |
aa674de1 MZ |
299 | case VM_MODE_P48V48_16K: |
300 | case VM_MODE_P40V48_16K: | |
301 | case VM_MODE_P36V48_16K: | |
302 | case VM_MODE_P36V47_16K: | |
303 | tcr_el1 |= 2ul << 14; /* TG0 = 16KB */ | |
304 | break; | |
10a0cc3b | 305 | case VM_MODE_P52V48_4K: |
cdbd2428 | 306 | case VM_MODE_P48V48_4K: |
2f41a61c | 307 | case VM_MODE_P40V48_4K: |
e7f58a6b | 308 | case VM_MODE_P36V48_4K: |
cdbd2428 | 309 | tcr_el1 |= 0ul << 14; /* TG0 = 4KB */ |
cdbd2428 | 310 | break; |
2f41a61c MZ |
311 | default: |
312 | TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); | |
313 | } | |
314 | ||
e1707175 RR |
315 | ttbr0_el1 = vm->pgd & GENMASK(47, vm->page_shift); |
316 | ||
2f41a61c MZ |
317 | /* Configure output size */ |
318 | switch (vm->mode) { | |
10a0cc3b RR |
319 | case VM_MODE_P52V48_4K: |
320 | case VM_MODE_P52V48_16K: | |
2f41a61c MZ |
321 | case VM_MODE_P52V48_64K: |
322 | tcr_el1 |= 6ul << 32; /* IPS = 52 bits */ | |
e1707175 | 323 | ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2; |
2f41a61c MZ |
324 | break; |
325 | case VM_MODE_P48V48_4K: | |
aa674de1 | 326 | case VM_MODE_P48V48_16K: |
cdbd2428 | 327 | case VM_MODE_P48V48_64K: |
cdbd2428 AJ |
328 | tcr_el1 |= 5ul << 32; /* IPS = 48 bits */ |
329 | break; | |
e28934e6 | 330 | case VM_MODE_P40V48_4K: |
aa674de1 | 331 | case VM_MODE_P40V48_16K: |
e28934e6 | 332 | case VM_MODE_P40V48_64K: |
e28934e6 AJ |
333 | tcr_el1 |= 2ul << 32; /* IPS = 40 bits */ |
334 | break; | |
e7f58a6b | 335 | case VM_MODE_P36V48_4K: |
aa674de1 | 336 | case VM_MODE_P36V48_16K: |
e7f58a6b | 337 | case VM_MODE_P36V48_64K: |
aa674de1 | 338 | case VM_MODE_P36V47_16K: |
e7f58a6b MZ |
339 | tcr_el1 |= 1ul << 32; /* IPS = 36 bits */ |
340 | break; | |
0bec140f | 341 | default: |
352be2c5 | 342 | TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode); |
0bec140f AJ |
343 | } |
344 | ||
345 | sctlr_el1 |= (1 << 0) | (1 << 2) | (1 << 12) /* M | C | I */; | |
346 | /* TCR_EL1 |= IRGN0:WBWA | ORGN0:WBWA | SH0:Inner-Shareable */; | |
347 | tcr_el1 |= (1 << 8) | (1 << 10) | (3 << 12); | |
348 | tcr_el1 |= (64 - vm->va_bits) /* T0SZ */; | |
10a0cc3b RR |
349 | if (use_lpa2_pte_format(vm)) |
350 | tcr_el1 |= (1ul << 59) /* DS */; | |
0bec140f | 351 | |
768e9a61 SC |
352 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1); |
353 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1); | |
354 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1); | |
e1707175 | 355 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), ttbr0_el1); |
768e9a61 | 356 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpu->id); |
0bec140f AJ |
357 | } |
358 | ||
768e9a61 | 359 | void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, uint8_t indent) |
0bec140f AJ |
360 | { |
361 | uint64_t pstate, pc; | |
362 | ||
768e9a61 SC |
363 | vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pstate), &pstate); |
364 | vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc), &pc); | |
0bec140f | 365 | |
98e68344 | 366 | fprintf(stream, "%*spstate: 0x%.16lx pc: 0x%.16lx\n", |
cdbd2428 | 367 | indent, "", pstate, pc); |
0bec140f | 368 | } |
837ec79b | 369 | |
53a43dd4 SC |
370 | void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) |
371 | { | |
372 | vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code); | |
373 | } | |
374 | ||
375 | static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, | |
376 | struct kvm_vcpu_init *init) | |
837ec79b | 377 | { |
5485e822 RK |
378 | size_t stack_size; |
379 | uint64_t stack_vaddr; | |
f742d94f | 380 | struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id); |
837ec79b | 381 | |
5485e822 RK |
382 | stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size : |
383 | vm->page_size; | |
1446e331 RK |
384 | stack_vaddr = __vm_vaddr_alloc(vm, stack_size, |
385 | DEFAULT_ARM64_GUEST_STACK_VADDR_MIN, | |
386 | MEM_REGION_DATA); | |
5485e822 | 387 | |
768e9a61 | 388 | aarch64_vcpu_setup(vcpu, init); |
837ec79b | 389 | |
768e9a61 | 390 | vcpu_set_reg(vcpu, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size); |
53a43dd4 SC |
391 | return vcpu; |
392 | } | |
393 | ||
394 | struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id, | |
395 | struct kvm_vcpu_init *init, void *guest_code) | |
396 | { | |
397 | struct kvm_vcpu *vcpu = __aarch64_vcpu_add(vm, vcpu_id, init); | |
398 | ||
399 | vcpu_arch_set_entry_point(vcpu, guest_code); | |
1422efd6 SC |
400 | |
401 | return vcpu; | |
837ec79b PB |
402 | } |
403 | ||
53a43dd4 | 404 | struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id) |
fb89f4ea | 405 | { |
53a43dd4 | 406 | return __aarch64_vcpu_add(vm, vcpu_id, NULL); |
fb89f4ea | 407 | } |
9bbf2474 | 408 | |
768e9a61 | 409 | void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) |
9bbf2474 BG |
410 | { |
411 | va_list ap; | |
412 | int i; | |
413 | ||
414 | TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n" | |
95be17e4 | 415 | " num: %u", num); |
9bbf2474 BG |
416 | |
417 | va_start(ap, num); | |
418 | ||
419 | for (i = 0; i < num; i++) { | |
768e9a61 SC |
420 | vcpu_set_reg(vcpu, ARM64_CORE_REG(regs.regs[i]), |
421 | va_arg(ap, uint64_t)); | |
9bbf2474 BG |
422 | } |
423 | ||
424 | va_end(ap); | |
425 | } | |
29faeb96 | 426 | |
e3db7579 RK |
427 | void kvm_exit_unexpected_exception(int vector, uint64_t ec, bool valid_ec) |
428 | { | |
429 | ucall(UCALL_UNHANDLED, 3, vector, ec, valid_ec); | |
430 | while (1) | |
431 | ; | |
432 | } | |
433 | ||
768e9a61 | 434 | void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) |
29faeb96 | 435 | { |
e3db7579 RK |
436 | struct ucall uc; |
437 | ||
768e9a61 | 438 | if (get_ucall(vcpu, &uc) != UCALL_UNHANDLED) |
e3db7579 RK |
439 | return; |
440 | ||
441 | if (uc.args[2]) /* valid_ec */ { | |
442 | assert(VECTOR_IS_SYNC(uc.args[0])); | |
443 | TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)", | |
444 | uc.args[0], uc.args[1]); | |
445 | } else { | |
446 | assert(!VECTOR_IS_SYNC(uc.args[0])); | |
447 | TEST_FAIL("Unexpected exception (vector:0x%lx)", | |
448 | uc.args[0]); | |
449 | } | |
450 | } | |
451 | ||
452 | struct handlers { | |
453 | handler_fn exception_handlers[VECTOR_NUM][ESR_EC_NUM]; | |
454 | }; | |
455 | ||
768e9a61 | 456 | void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu) |
e3db7579 RK |
457 | { |
458 | extern char vectors; | |
459 | ||
768e9a61 | 460 | vcpu_set_reg(vcpu, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors); |
e3db7579 RK |
461 | } |
462 | ||
463 | void route_exception(struct ex_regs *regs, int vector) | |
464 | { | |
465 | struct handlers *handlers = (struct handlers *)exception_handlers; | |
466 | bool valid_ec; | |
467 | int ec = 0; | |
468 | ||
469 | switch (vector) { | |
470 | case VECTOR_SYNC_CURRENT: | |
471 | case VECTOR_SYNC_LOWER_64: | |
472 | ec = (read_sysreg(esr_el1) >> ESR_EC_SHIFT) & ESR_EC_MASK; | |
473 | valid_ec = true; | |
474 | break; | |
475 | case VECTOR_IRQ_CURRENT: | |
476 | case VECTOR_IRQ_LOWER_64: | |
477 | case VECTOR_FIQ_CURRENT: | |
478 | case VECTOR_FIQ_LOWER_64: | |
479 | case VECTOR_ERROR_CURRENT: | |
480 | case VECTOR_ERROR_LOWER_64: | |
481 | ec = 0; | |
482 | valid_ec = false; | |
483 | break; | |
484 | default: | |
485 | valid_ec = false; | |
486 | goto unexpected_exception; | |
487 | } | |
488 | ||
489 | if (handlers && handlers->exception_handlers[vector][ec]) | |
490 | return handlers->exception_handlers[vector][ec](regs); | |
491 | ||
492 | unexpected_exception: | |
493 | kvm_exit_unexpected_exception(vector, ec, valid_ec); | |
494 | } | |
495 | ||
496 | void vm_init_descriptor_tables(struct kvm_vm *vm) | |
497 | { | |
1446e331 RK |
498 | vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers), |
499 | vm->page_size, MEM_REGION_DATA); | |
e3db7579 RK |
500 | |
501 | *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; | |
502 | } | |
503 | ||
504 | void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec, | |
505 | void (*handler)(struct ex_regs *)) | |
506 | { | |
507 | struct handlers *handlers = addr_gva2hva(vm, vm->handlers); | |
508 | ||
509 | assert(VECTOR_IS_SYNC(vector)); | |
510 | assert(vector < VECTOR_NUM); | |
511 | assert(ec < ESR_EC_NUM); | |
512 | handlers->exception_handlers[vector][ec] = handler; | |
513 | } | |
514 | ||
515 | void vm_install_exception_handler(struct kvm_vm *vm, int vector, | |
516 | void (*handler)(struct ex_regs *)) | |
517 | { | |
518 | struct handlers *handlers = addr_gva2hva(vm, vm->handlers); | |
519 | ||
520 | assert(!VECTOR_IS_SYNC(vector)); | |
521 | assert(vector < VECTOR_NUM); | |
522 | handlers->exception_handlers[vector][0] = handler; | |
29faeb96 | 523 | } |
17229bdc RRA |
524 | |
525 | uint32_t guest_get_vcpuid(void) | |
526 | { | |
527 | return read_sysreg(tpidr_el1); | |
528 | } | |
cb7c4f36 | 529 | |
72324ac5 RR |
530 | static uint32_t max_ipa_for_page_size(uint32_t vm_ipa, uint32_t gran, |
531 | uint32_t not_sup_val, uint32_t ipa52_min_val) | |
532 | { | |
533 | if (gran == not_sup_val) | |
534 | return 0; | |
535 | else if (gran >= ipa52_min_val && vm_ipa >= 52) | |
536 | return 52; | |
537 | else | |
538 | return min(vm_ipa, 48U); | |
539 | } | |
540 | ||
541 | void aarch64_get_supported_page_sizes(uint32_t ipa, uint32_t *ipa4k, | |
542 | uint32_t *ipa16k, uint32_t *ipa64k) | |
0303ffdb MZ |
543 | { |
544 | struct kvm_vcpu_init preferred_init; | |
545 | int kvm_fd, vm_fd, vcpu_fd, err; | |
546 | uint64_t val; | |
72324ac5 | 547 | uint32_t gran; |
0303ffdb MZ |
548 | struct kvm_one_reg reg = { |
549 | .id = KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1), | |
550 | .addr = (uint64_t)&val, | |
551 | }; | |
552 | ||
553 | kvm_fd = open_kvm_dev_path_or_exit(); | |
fcba483e | 554 | vm_fd = __kvm_ioctl(kvm_fd, KVM_CREATE_VM, (void *)(unsigned long)ipa); |
f9725f89 | 555 | TEST_ASSERT(vm_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm_fd)); |
0303ffdb MZ |
556 | |
557 | vcpu_fd = ioctl(vm_fd, KVM_CREATE_VCPU, 0); | |
a78593fd | 558 | TEST_ASSERT(vcpu_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, vcpu_fd)); |
0303ffdb MZ |
559 | |
560 | err = ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init); | |
a78593fd | 561 | TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_PREFERRED_TARGET, err)); |
0303ffdb | 562 | err = ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init); |
a78593fd | 563 | TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_VCPU_INIT, err)); |
0303ffdb MZ |
564 | |
565 | err = ioctl(vcpu_fd, KVM_GET_ONE_REG, ®); | |
a78593fd | 566 | TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd)); |
0303ffdb | 567 | |
72324ac5 RR |
568 | gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN4), val); |
569 | *ipa4k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN4_NI, | |
570 | ID_AA64MMFR0_EL1_TGRAN4_52_BIT); | |
571 | ||
572 | gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN64), val); | |
573 | *ipa64k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN64_NI, | |
574 | ID_AA64MMFR0_EL1_TGRAN64_IMP); | |
575 | ||
576 | gran = FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_TGRAN16), val); | |
577 | *ipa16k = max_ipa_for_page_size(ipa, gran, ID_AA64MMFR0_EL1_TGRAN16_NI, | |
578 | ID_AA64MMFR0_EL1_TGRAN16_52_BIT); | |
0303ffdb MZ |
579 | |
580 | close(vcpu_fd); | |
581 | close(vm_fd); | |
582 | close(kvm_fd); | |
583 | } | |
584 | ||
fab19915 OU |
585 | #define __smccc_call(insn, function_id, arg0, arg1, arg2, arg3, arg4, arg5, \ |
586 | arg6, res) \ | |
587 | asm volatile("mov w0, %w[function_id]\n" \ | |
588 | "mov x1, %[arg0]\n" \ | |
589 | "mov x2, %[arg1]\n" \ | |
590 | "mov x3, %[arg2]\n" \ | |
591 | "mov x4, %[arg3]\n" \ | |
592 | "mov x5, %[arg4]\n" \ | |
593 | "mov x6, %[arg5]\n" \ | |
594 | "mov x7, %[arg6]\n" \ | |
595 | #insn "#0\n" \ | |
596 | "mov %[res0], x0\n" \ | |
597 | "mov %[res1], x1\n" \ | |
598 | "mov %[res2], x2\n" \ | |
599 | "mov %[res3], x3\n" \ | |
600 | : [res0] "=r"(res->a0), [res1] "=r"(res->a1), \ | |
601 | [res2] "=r"(res->a2), [res3] "=r"(res->a3) \ | |
602 | : [function_id] "r"(function_id), [arg0] "r"(arg0), \ | |
603 | [arg1] "r"(arg1), [arg2] "r"(arg2), [arg3] "r"(arg3), \ | |
604 | [arg4] "r"(arg4), [arg5] "r"(arg5), [arg6] "r"(arg6) \ | |
605 | : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7") | |
606 | ||
607 | ||
e918e2bc OU |
608 | void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1, |
609 | uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, | |
610 | uint64_t arg6, struct arm_smccc_res *res) | |
611 | { | |
fab19915 OU |
612 | __smccc_call(hvc, function_id, arg0, arg1, arg2, arg3, arg4, arg5, |
613 | arg6, res); | |
614 | } | |
615 | ||
616 | void smccc_smc(uint32_t function_id, uint64_t arg0, uint64_t arg1, | |
617 | uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, | |
618 | uint64_t arg6, struct arm_smccc_res *res) | |
619 | { | |
620 | __smccc_call(smc, function_id, arg0, arg1, arg2, arg3, arg4, arg5, | |
621 | arg6, res); | |
e918e2bc | 622 | } |
e1ab3124 VA |
623 | |
624 | void kvm_selftest_arch_init(void) | |
625 | { | |
626 | /* | |
627 | * arm64 doesn't have a true default mode, so start by computing the | |
628 | * available IPA space and page sizes early. | |
629 | */ | |
630 | guest_modes_append_default(); | |
631 | } | |
e8b9a055 OU |
632 | |
633 | void vm_vaddr_populate_bitmap(struct kvm_vm *vm) | |
634 | { | |
635 | /* | |
636 | * arm64 selftests use only TTBR0_EL1, meaning that the valid VA space | |
637 | * is [0, 2^(64 - TCR_EL1.T0SZ)). | |
638 | */ | |
639 | sparsebit_set_num(vm->vpages_valid, 0, | |
640 | (1ULL << vm->va_bits) >> vm->page_shift); | |
641 | } | |
ca1a1836 CL |
642 | |
643 | /* Helper to call wfi instruction. */ | |
644 | void wfi(void) | |
645 | { | |
646 | asm volatile("wfi"); | |
647 | } |