projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/kvm: Switch kvm pmd allocator to custom allocator
[linux-2.6-block.git] / arch / powerpc / kvm / book3s_64_mmu_radix.c
CommitLineData
9e04ba69
PM		 1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
5 *
6 * Copyright 2016 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
7 */
8
9#include <linux/types.h>
10#include <linux/string.h>
11#include <linux/kvm.h>
12#include <linux/kvm_host.h>
13
14#include <asm/kvm_ppc.h>
15#include <asm/kvm_book3s.h>
16#include <asm/page.h>
17#include <asm/mmu.h>
18#include <asm/pgtable.h>
19#include <asm/pgalloc.h>
94171b19 20#include <asm/pte-walk.h>
9e04ba69
PM		 21
22/*
23 * Supported radix tree geometry.
24 * Like p9, we support either 5 or 9 bits at the first (lowest) level,
25 * for a page size of 64k or 4k.
26 */
27static int p9_supported_radix_bits[4] = { 5, 9, 9, 13 };
28
29int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
30 struct kvmppc_pte *gpte, bool data, bool iswrite)
31{
32 struct kvm *kvm = vcpu->kvm;
33 u32 pid;
34 int ret, level, ps;
35 __be64 prte, rpte;
70cd4c10 36 unsigned long ptbl;
9e04ba69
PM		 37 unsigned long root, pte, index;
38 unsigned long rts, bits, offset;
39 unsigned long gpa;
40 unsigned long proc_tbl_size;
41
42 /* Work out effective PID */
43 switch (eaddr >> 62) {
44 case 0:
45 pid = vcpu->arch.pid;
46 break;
47 case 3:
48 pid = 0;
49 break;
50 default:
51 return -EINVAL;
52 }
53 proc_tbl_size = 1 << ((kvm->arch.process_table & PRTS_MASK) + 12);
54 if (pid * 16 >= proc_tbl_size)
55 return -EINVAL;
56
57 /* Read partition table to find root of tree for effective PID */
70cd4c10
PM		 58 ptbl = (kvm->arch.process_table & PRTB_MASK) + (pid * 16);
59 ret = kvm_read_guest(kvm, ptbl, &prte, sizeof(prte));
9e04ba69
PM		 60 if (ret)
61 return ret;
62
63 root = be64_to_cpu(prte);
64 rts = ((root & RTS1_MASK) >> (RTS1_SHIFT - 3)) |
65 ((root & RTS2_MASK) >> RTS2_SHIFT);
66 bits = root & RPDS_MASK;
67 root = root & RPDB_MASK;
68
69 /* P9 DD1 interprets RTS (radix tree size) differently */
70 offset = rts + 31;
71 if (cpu_has_feature(CPU_FTR_POWER9_DD1))
72 offset -= 3;
73
74 /* current implementations only support 52-bit space */
75 if (offset != 52)
76 return -EINVAL;
77
78 for (level = 3; level >= 0; --level) {
79 if (level && bits != p9_supported_radix_bits[level])
80 return -EINVAL;
81 if (level == 0 && !(bits == 5 || bits == 9))
82 return -EINVAL;
83 offset -= bits;
84 index = (eaddr >> offset) & ((1UL << bits) - 1);
85 /* check that low bits of page table base are zero */
86 if (root & ((1UL << (bits + 3)) - 1))
87 return -EINVAL;
88 ret = kvm_read_guest(kvm, root + index * 8,
89 &rpte, sizeof(rpte));
90 if (ret)
91 return ret;
92 pte = __be64_to_cpu(rpte);
93 if (!(pte & _PAGE_PRESENT))
94 return -ENOENT;
95 if (pte & _PAGE_PTE)
96 break;
97 bits = pte & 0x1f;
98 root = pte & 0x0fffffffffffff00ul;
99 }
100 /* need a leaf at lowest level; 512GB pages not supported */
101 if (level < 0 || level == 3)
102 return -EINVAL;
103
104 /* offset is now log base 2 of the page size */
105 gpa = pte & 0x01fffffffffff000ul;
106 if (gpa & ((1ul << offset) - 1))
107 return -EINVAL;
108 gpa += eaddr & ((1ul << offset) - 1);
109 for (ps = MMU_PAGE_4K; ps < MMU_PAGE_COUNT; ++ps)
110 if (offset == mmu_psize_defs[ps].shift)
111 break;
112 gpte->page_size = ps;
113
114 gpte->eaddr = eaddr;
115 gpte->raddr = gpa;
116
117 /* Work out permissions */
118 gpte->may_read = !!(pte & _PAGE_READ);
119 gpte->may_write = !!(pte & _PAGE_WRITE);
120 gpte->may_execute = !!(pte & _PAGE_EXEC);
121 if (kvmppc_get_msr(vcpu) & MSR_PR) {
122 if (pte & _PAGE_PRIVILEGED) {
123 gpte->may_read = 0;
124 gpte->may_write = 0;
125 gpte->may_execute = 0;
126 }
127 } else {
128 if (!(pte & _PAGE_PRIVILEGED)) {
129 /* Check AMR/IAMR to see if strict mode is in force */
130 if (vcpu->arch.amr & (1ul << 62))
131 gpte->may_read = 0;
132 if (vcpu->arch.amr & (1ul << 63))
133 gpte->may_write = 0;
134 if (vcpu->arch.iamr & (1ul << 62))
135 gpte->may_execute = 0;
136 }
137 }
138
139 return 0;
140}
141
5a319350
PM		 142#ifdef CONFIG_PPC_64K_PAGES
143#define MMU_BASE_PSIZE MMU_PAGE_64K
144#else
145#define MMU_BASE_PSIZE MMU_PAGE_4K
146#endif
147
148static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
149 unsigned int pshift)
150{
151 int psize = MMU_BASE_PSIZE;
152
58c5c276
PM		 153 if (pshift >= PUD_SHIFT)
154 psize = MMU_PAGE_1G;
155 else if (pshift >= PMD_SHIFT)
5a319350
PM		 156 psize = MMU_PAGE_2M;
157 addr &= ~0xfffUL;
158 addr |= mmu_psize_defs[psize].ap << 5;
159 asm volatile("ptesync": : :"memory");
160 asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
161 : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
a5d4b589
AK		 162 if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
163 asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
164 : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
5a319350
PM		 165 asm volatile("ptesync": : :"memory");
166}
167
c4c8a764
PM		 168static void kvmppc_radix_flush_pwc(struct kvm *kvm, unsigned long addr)
169{
170 unsigned long rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
171
172 asm volatile("ptesync": : :"memory");
173 /* RIC=1 PRS=0 R=1 IS=2 */
174 asm volatile(PPC_TLBIE_5(%0, %1, 1, 0, 1)
175 : : "r" (rb), "r" (kvm->arch.lpid) : "memory");
176 asm volatile("ptesync": : :"memory");
177}
178
8f7b79b8
PM		 179unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
180 unsigned long clr, unsigned long set,
181 unsigned long addr, unsigned int shift)
5a319350 182{
8f7b79b8
PM		 183 unsigned long old = 0;
184
5a319350
PM		 185 if (!(clr & _PAGE_PRESENT) && cpu_has_feature(CPU_FTR_POWER9_DD1) &&
186 pte_present(*ptep)) {
187 /* have to invalidate it first */
8f7b79b8 188 old = __radix_pte_update(ptep, _PAGE_PRESENT, 0);
5a319350
PM		 189 kvmppc_radix_tlbie_page(kvm, addr, shift);
190 set |= _PAGE_PRESENT;
8f7b79b8 191 old &= _PAGE_PRESENT;
5a319350 192 }
8f7b79b8 193 return __radix_pte_update(ptep, clr, set) | old;
5a319350
PM		 194}
195
196void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
197 pte_t *ptep, pte_t pte)
198{
199 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0);
200}
201
202static struct kmem_cache *kvm_pte_cache;
21828c99 203static struct kmem_cache *kvm_pmd_cache;
5a319350
PM		 204
205static pte_t *kvmppc_pte_alloc(void)
206{
207 return kmem_cache_alloc(kvm_pte_cache, GFP_KERNEL);
208}
209
210static void kvmppc_pte_free(pte_t *ptep)
211{
212 kmem_cache_free(kvm_pte_cache, ptep);
213}
214
c3856aeb
PM		 215/* Like pmd_huge() and pmd_large(), but works regardless of config options */
216static inline int pmd_is_leaf(pmd_t pmd)
217{
218 return !!(pmd_val(pmd) & _PAGE_PTE);
219}
220
21828c99
AK		 221static pmd_t *kvmppc_pmd_alloc(void)
222{
223 return kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL);
224}
225
226static void kvmppc_pmd_free(pmd_t *pmdp)
227{
228 kmem_cache_free(kvm_pmd_cache, pmdp);
229}
230
5a319350
PM		 231static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
232 unsigned int level, unsigned long mmu_seq)
233{
234 pgd_t *pgd;
235 pud_t *pud, *new_pud = NULL;
236 pmd_t *pmd, *new_pmd = NULL;
237 pte_t *ptep, *new_ptep = NULL;
8f7b79b8 238 unsigned long old;
5a319350
PM		 239 int ret;
240
241 /* Traverse the guest's 2nd-level tree, allocate new levels needed */
242 pgd = kvm->arch.pgtable + pgd_index(gpa);
243 pud = NULL;
244 if (pgd_present(*pgd))
245 pud = pud_offset(pgd, gpa);
246 else
247 new_pud = pud_alloc_one(kvm->mm, gpa);
248
249 pmd = NULL;
58c5c276 250 if (pud && pud_present(*pud) && !pud_huge(*pud))
5a319350 251 pmd = pmd_offset(pud, gpa);
58c5c276 252 else if (level <= 1)
21828c99 253 new_pmd = kvmppc_pmd_alloc();
5a319350 254
c3856aeb 255 if (level == 0 && !(pmd && pmd_present(*pmd) && !pmd_is_leaf(*pmd)))
5a319350
PM		 256 new_ptep = kvmppc_pte_alloc();
257
258 /* Check if we might have been invalidated; let the guest retry if so */
259 spin_lock(&kvm->mmu_lock);
260 ret = -EAGAIN;
261 if (mmu_notifier_retry(kvm, mmu_seq))
262 goto out_unlock;
263
264 /* Now traverse again under the lock and change the tree */
265 ret = -ENOMEM;
266 if (pgd_none(*pgd)) {
267 if (!new_pud)
268 goto out_unlock;
269 pgd_populate(kvm->mm, pgd, new_pud);
270 new_pud = NULL;
271 }
272 pud = pud_offset(pgd, gpa);
58c5c276
PM		 273 if (pud_huge(*pud)) {
274 unsigned long hgpa = gpa & PUD_MASK;
275
276 /*
277 * If we raced with another CPU which has just put
278 * a 1GB pte in after we saw a pmd page, try again.
279 */
280 if (level <= 1 && !new_pmd) {
281 ret = -EAGAIN;
282 goto out_unlock;
283 }
284 /* Check if we raced and someone else has set the same thing */
285 if (level == 2 && pud_raw(*pud) == pte_raw(pte)) {
286 ret = 0;
287 goto out_unlock;
288 }
289 /* Valid 1GB page here already, remove it */
290 old = kvmppc_radix_update_pte(kvm, (pte_t *)pud,
291 ~0UL, 0, hgpa, PUD_SHIFT);
292 kvmppc_radix_tlbie_page(kvm, hgpa, PUD_SHIFT);
293 if (old & _PAGE_DIRTY) {
294 unsigned long gfn = hgpa >> PAGE_SHIFT;
295 struct kvm_memory_slot *memslot;
296 memslot = gfn_to_memslot(kvm, gfn);
297 if (memslot && memslot->dirty_bitmap)
298 kvmppc_update_dirty_map(memslot,
299 gfn, PUD_SIZE);
300 }
301 }
302 if (level == 2) {
303 if (!pud_none(*pud)) {
304 /*
305 * There's a page table page here, but we wanted to
306 * install a large page, so remove and free the page
307 * table page. new_pmd will be NULL since level == 2.
308 */
309 new_pmd = pmd_offset(pud, 0);
310 pud_clear(pud);
311 kvmppc_radix_flush_pwc(kvm, gpa);
312 }
313 kvmppc_radix_set_pte_at(kvm, gpa, (pte_t *)pud, pte);
314 ret = 0;
315 goto out_unlock;
316 }
5a319350
PM		 317 if (pud_none(*pud)) {
318 if (!new_pmd)
319 goto out_unlock;
320 pud_populate(kvm->mm, pud, new_pmd);
321 new_pmd = NULL;
322 }
323 pmd = pmd_offset(pud, gpa);
c3856aeb
PM		 324 if (pmd_is_leaf(*pmd)) {
325 unsigned long lgpa = gpa & PMD_MASK;
326
327 /*
328 * If we raced with another CPU which has just put
329 * a 2MB pte in after we saw a pte page, try again.
330 */
331 if (level == 0 && !new_ptep) {
332 ret = -EAGAIN;
333 goto out_unlock;
334 }
c4c8a764
PM		 335 /* Check if we raced and someone else has set the same thing */
336 if (level == 1 && pmd_raw(*pmd) == pte_raw(pte)) {
337 ret = 0;
338 goto out_unlock;
339 }
c3856aeb
PM		 340 /* Valid 2MB page here already, remove it */
341 old = kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
342 ~0UL, 0, lgpa, PMD_SHIFT);
343 kvmppc_radix_tlbie_page(kvm, lgpa, PMD_SHIFT);
344 if (old & _PAGE_DIRTY) {
345 unsigned long gfn = lgpa >> PAGE_SHIFT;
346 struct kvm_memory_slot *memslot;
347 memslot = gfn_to_memslot(kvm, gfn);
348 if (memslot && memslot->dirty_bitmap)
349 kvmppc_update_dirty_map(memslot,
350 gfn, PMD_SIZE);
351 }
5a319350 352 }
58c5c276
PM		 353 if (level == 1) {
354 if (!pmd_none(*pmd)) {
355 /*
356 * There's a page table page here, but we wanted to
357 * install a large page, so remove and free the page
358 * table page. new_ptep will be NULL since level == 1.
359 */
360 new_ptep = pte_offset_kernel(pmd, 0);
361 pmd_clear(pmd);
362 kvmppc_radix_flush_pwc(kvm, gpa);
5a319350 363 }
5a319350 364 kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte);
58c5c276
PM		 365 ret = 0;
366 goto out_unlock;
5a319350 367 }
58c5c276
PM		 368 if (pmd_none(*pmd)) {
369 if (!new_ptep)
370 goto out_unlock;
371 pmd_populate(kvm->mm, pmd, new_ptep);
372 new_ptep = NULL;
373 }
374 ptep = pte_offset_kernel(pmd, gpa);
375 if (pte_present(*ptep)) {
376 /* Check if someone else set the same thing */
377 if (pte_raw(*ptep) == pte_raw(pte)) {
378 ret = 0;
379 goto out_unlock;
380 }
381 /* PTE was previously valid, so invalidate it */
382 old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
383 0, gpa, 0);
384 kvmppc_radix_tlbie_page(kvm, gpa, 0);
385 if (old & _PAGE_DIRTY)
386 mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
5a319350 387 }
58c5c276 388 kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
5a319350
PM		 389 ret = 0;
390
391 out_unlock:
392 spin_unlock(&kvm->mmu_lock);
393 if (new_pud)
394 pud_free(kvm->mm, new_pud);
395 if (new_pmd)
21828c99 396 kvmppc_pmd_free(new_pmd);
5a319350
PM		 397 if (new_ptep)
398 kvmppc_pte_free(new_ptep);
399 return ret;
400}
401
402int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
403 unsigned long ea, unsigned long dsisr)
404{
405 struct kvm *kvm = vcpu->kvm;
406 unsigned long mmu_seq, pte_size;
407 unsigned long gpa, gfn, hva, pfn;
408 struct kvm_memory_slot *memslot;
31c8b0d0
PM		 409 struct page *page = NULL;
410 long ret;
411 bool writing;
412 bool upgrade_write = false;
413 bool *upgrade_p = &upgrade_write;
5a319350
PM		 414 pte_t pte, *ptep;
415 unsigned long pgflags;
416 unsigned int shift, level;
417
418 /* Check for unusual errors */
419 if (dsisr & DSISR_UNSUPP_MMU) {
420 pr_err("KVM: Got unsupported MMU fault\n");
421 return -EFAULT;
422 }
423 if (dsisr & DSISR_BADACCESS) {
424 /* Reflect to the guest as DSI */
425 pr_err("KVM: Got radix HV page fault with DSISR=%lx\n", dsisr);
426 kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
427 return RESUME_GUEST;
428 }
429
430 /* Translate the logical address and get the page */
431 gpa = vcpu->arch.fault_gpa & ~0xfffUL;
432 gpa &= ~0xF000000000000000ul;
433 gfn = gpa >> PAGE_SHIFT;
870cfe77 434 if (!(dsisr & DSISR_PRTABLE_FAULT))
5a319350
PM		 435 gpa |= ea & 0xfff;
436 memslot = gfn_to_memslot(kvm, gfn);
437
438 /* No memslot means it's an emulated MMIO region */
439 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) {
870cfe77 440 if (dsisr & (DSISR_PRTABLE_FAULT | DSISR_BADACCESS |
5a319350
PM		 441 DSISR_SET_RC)) {
442 /*
443 * Bad address in guest page table tree, or other
444 * unusual error - reflect it to the guest as DSI.
445 */
446 kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
447 return RESUME_GUEST;
448 }
449 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
450 dsisr & DSISR_ISSTORE);
451 }
452
5a319350 453 writing = (dsisr & DSISR_ISSTORE) != 0;
31c8b0d0
PM		 454 if (memslot->flags & KVM_MEM_READONLY) {
455 if (writing) {
456 /* give the guest a DSI */
457 dsisr = DSISR_ISSTORE | DSISR_PROTFAULT;
458 kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
459 return RESUME_GUEST;
460 }
461 upgrade_p = NULL;
462 }
463
5a319350
PM		 464 if (dsisr & DSISR_SET_RC) {
465 /*
466 * Need to set an R or C bit in the 2nd-level tables;
f7caf712
PM		 467 * since we are just helping out the hardware here,
468 * it is sufficient to do what the hardware does.
5a319350 469 */
5a319350
PM		 470 pgflags = _PAGE_ACCESSED;
471 if (writing)
472 pgflags |= _PAGE_DIRTY;
f7caf712
PM		 473 /*
474 * We are walking the secondary page table here. We can do this
475 * without disabling irq.
476 */
477 spin_lock(&kvm->mmu_lock);
478 ptep = __find_linux_pte(kvm->arch.pgtable,
479 gpa, NULL, &shift);
480 if (ptep && pte_present(*ptep) &&
481 (!writing || pte_write(*ptep))) {
482 kvmppc_radix_update_pte(kvm, ptep, 0, pgflags,
483 gpa, shift);
484 dsisr &= ~DSISR_SET_RC;
5a319350 485 }
f7caf712
PM		 486 spin_unlock(&kvm->mmu_lock);
487 if (!(dsisr & (DSISR_BAD_FAULT_64S | DSISR_NOHPTE |
488 DSISR_PROTFAULT | DSISR_SET_RC)))
489 return RESUME_GUEST;
5a319350
PM		 490 }
491
31c8b0d0
PM		 492 /* used to check for invalidations in progress */
493 mmu_seq = kvm->mmu_notifier_seq;
494 smp_rmb();
495
496 /*
497 * Do a fast check first, since __gfn_to_pfn_memslot doesn't
498 * do it with !atomic && !async, which is how we call it.
499 * We always ask for write permission since the common case
500 * is that the page is writable.
501 */
502 hva = gfn_to_hva_memslot(memslot, gfn);
503 if (upgrade_p && __get_user_pages_fast(hva, 1, 1, &page) == 1) {
5a319350 504 pfn = page_to_pfn(page);
31c8b0d0
PM		 505 upgrade_write = true;
506 } else {
507 /* Call KVM generic code to do the slow-path check */
508 pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL,
509 writing, upgrade_p);
510 if (is_error_noslot_pfn(pfn))
511 return -EFAULT;
512 page = NULL;
513 if (pfn_valid(pfn)) {
514 page = pfn_to_page(pfn);
515 if (PageReserved(page))
516 page = NULL;
5a319350 517 }
31c8b0d0
PM		 518 }
519
520 /* See if we can insert a 1GB or 2MB large PTE here */
521 level = 0;
522 if (page && PageCompound(page)) {
523 pte_size = PAGE_SIZE << compound_order(compound_head(page));
524 if (pte_size >= PUD_SIZE &&
525 (gpa & (PUD_SIZE - PAGE_SIZE)) ==
526 (hva & (PUD_SIZE - PAGE_SIZE))) {
527 level = 2;
528 pfn &= ~((PUD_SIZE >> PAGE_SHIFT) - 1);
529 } else if (pte_size >= PMD_SIZE &&
530 (gpa & (PMD_SIZE - PAGE_SIZE)) ==
531 (hva & (PMD_SIZE - PAGE_SIZE))) {
532 level = 1;
533 pfn &= ~((PMD_SIZE >> PAGE_SHIFT) - 1);
5a319350
PM		 534 }
535 }
536
537 /*
538 * Compute the PTE value that we need to insert.
539 */
31c8b0d0
PM		 540 if (page) {
541 pgflags = _PAGE_READ | _PAGE_EXEC | _PAGE_PRESENT | _PAGE_PTE |
542 _PAGE_ACCESSED;
543 if (writing || upgrade_write)
544 pgflags |= _PAGE_WRITE | _PAGE_DIRTY;
545 pte = pfn_pte(pfn, __pgprot(pgflags));
546 } else {
5a319350 547 /*
31c8b0d0
PM		 548 * Read the PTE from the process' radix tree and use that
549 * so we get the attribute bits.
5a319350 550 */
31c8b0d0
PM		 551 local_irq_disable();
552 ptep = __find_linux_pte(vcpu->arch.pgdir, hva, NULL, &shift);
553 pte = *ptep;
554 local_irq_enable();
555 if (shift == PUD_SHIFT &&
556 (gpa & (PUD_SIZE - PAGE_SIZE)) ==
557 (hva & (PUD_SIZE - PAGE_SIZE))) {
558 level = 2;
559 } else if (shift == PMD_SHIFT &&
560 (gpa & (PMD_SIZE - PAGE_SIZE)) ==
561 (hva & (PMD_SIZE - PAGE_SIZE))) {
562 level = 1;
563 } else if (shift && shift != PAGE_SHIFT) {
564 /* Adjust PFN */
565 unsigned long mask = (1ul << shift) - PAGE_SIZE;
566 pte = __pte(pte_val(pte) | (hva & mask));
567 }
568 if (!(writing || upgrade_write))
569 pte = __pte(pte_val(pte) & ~ _PAGE_WRITE);
570 pte = __pte(pte_val(pte) | _PAGE_EXEC);
5a319350 571 }
5a319350 572
5a319350
PM		 573 /* Allocate space in the tree and write the PTE */
574 ret = kvmppc_create_pte(kvm, pte, gpa, level, mmu_seq);
5a319350
PM		 575
576 if (page) {
31c8b0d0 577 if (!ret && (pte_val(pte) & _PAGE_WRITE))
c3856aeb
PM		 578 set_page_dirty_lock(page);
579 put_page(page);
5a319350 580 }
c3856aeb
PM		 581
582 if (ret == 0 || ret == -EAGAIN)
583 ret = RESUME_GUEST;
5a319350
PM		 584 return ret;
585}
586
01756099
PM		 587/* Called with kvm->lock held */
588int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
589 unsigned long gfn)
590{
591 pte_t *ptep;
592 unsigned long gpa = gfn << PAGE_SHIFT;
593 unsigned int shift;
8f7b79b8 594 unsigned long old;
01756099 595
94171b19 596 ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
01756099 597 if (ptep && pte_present(*ptep)) {
8f7b79b8
PM		 598 old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT, 0,
599 gpa, shift);
01756099 600 kvmppc_radix_tlbie_page(kvm, gpa, shift);
e641a317
PM		 601 if ((old & _PAGE_DIRTY) && memslot->dirty_bitmap) {
602 unsigned long npages = 1;
603 if (shift)
604 npages = 1ul << (shift - PAGE_SHIFT);
605 kvmppc_update_dirty_map(memslot, gfn, npages);
8f7b79b8 606 }
01756099
PM		 607 }
608 return 0;
609}
610
611/* Called with kvm->lock held */
612int kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
613 unsigned long gfn)
614{
615 pte_t *ptep;
616 unsigned long gpa = gfn << PAGE_SHIFT;
617 unsigned int shift;
618 int ref = 0;
619
94171b19 620 ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
01756099
PM		 621 if (ptep && pte_present(*ptep) && pte_young(*ptep)) {
622 kvmppc_radix_update_pte(kvm, ptep, _PAGE_ACCESSED, 0,
623 gpa, shift);
624 /* XXX need to flush tlb here? */
625 ref = 1;
626 }
627 return ref;
628}
629
630/* Called with kvm->lock held */
631int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
632 unsigned long gfn)
633{
634 pte_t *ptep;
635 unsigned long gpa = gfn << PAGE_SHIFT;
636 unsigned int shift;
637 int ref = 0;
638
94171b19 639 ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
01756099
PM		 640 if (ptep && pte_present(*ptep) && pte_young(*ptep))
641 ref = 1;
642 return ref;
643}
644
8f7b79b8
PM		 645/* Returns the number of PAGE_SIZE pages that are dirty */
646static int kvm_radix_test_clear_dirty(struct kvm *kvm,
647 struct kvm_memory_slot *memslot, int pagenum)
648{
649 unsigned long gfn = memslot->base_gfn + pagenum;
650 unsigned long gpa = gfn << PAGE_SHIFT;
651 pte_t *ptep;
652 unsigned int shift;
653 int ret = 0;
654
94171b19 655 ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
8f7b79b8
PM		 656 if (ptep && pte_present(*ptep) && pte_dirty(*ptep)) {
657 ret = 1;
658 if (shift)
659 ret = 1 << (shift - PAGE_SHIFT);
660 kvmppc_radix_update_pte(kvm, ptep, _PAGE_DIRTY, 0,
661 gpa, shift);
662 kvmppc_radix_tlbie_page(kvm, gpa, shift);
663 }
664 return ret;
665}
666
667long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
668 struct kvm_memory_slot *memslot, unsigned long *map)
669{
670 unsigned long i, j;
8f7b79b8
PM		 671 int npages;
672
8f7b79b8
PM		 673 for (i = 0; i < memslot->npages; i = j) {
674 npages = kvm_radix_test_clear_dirty(kvm, memslot, i);
675
676 /*
677 * Note that if npages > 0 then i must be a multiple of npages,
678 * since huge pages are only used to back the guest at guest
679 * real addresses that are a multiple of their size.
680 * Since we have at most one PTE covering any given guest
681 * real address, if npages > 1 we can skip to i + npages.
682 */
683 j = i + 1;
e641a317
PM		 684 if (npages) {
685 set_dirty_bits(map, i, npages);
117647ff 686 j = i + npages;
e641a317 687 }
8f7b79b8
PM		 688 }
689 return 0;
690}
691
8cf4ecc0
PM		 692static void add_rmmu_ap_encoding(struct kvm_ppc_rmmu_info *info,
693 int psize, int *indexp)
694{
695 if (!mmu_psize_defs[psize].shift)
696 return;
697 info->ap_encodings[*indexp] = mmu_psize_defs[psize].shift |
698 (mmu_psize_defs[psize].ap << 29);
699 ++(*indexp);
700}
701
702int kvmhv_get_rmmu_info(struct kvm *kvm, struct kvm_ppc_rmmu_info *info)
703{
704 int i;
705
706 if (!radix_enabled())
707 return -EINVAL;
708 memset(info, 0, sizeof(*info));
709
710 /* 4k page size */
711 info->geometries[0].page_shift = 12;
712 info->geometries[0].level_bits[0] = 9;
713 for (i = 1; i < 4; ++i)
714 info->geometries[0].level_bits[i] = p9_supported_radix_bits[i];
715 /* 64k page size */
716 info->geometries[1].page_shift = 16;
717 for (i = 0; i < 4; ++i)
718 info->geometries[1].level_bits[i] = p9_supported_radix_bits[i];
719
720 i = 0;
721 add_rmmu_ap_encoding(info, MMU_PAGE_4K, &i);
722 add_rmmu_ap_encoding(info, MMU_PAGE_64K, &i);
723 add_rmmu_ap_encoding(info, MMU_PAGE_2M, &i);
724 add_rmmu_ap_encoding(info, MMU_PAGE_1G, &i);
725
726 return 0;
727}
728
729int kvmppc_init_vm_radix(struct kvm *kvm)
730{
731 kvm->arch.pgtable = pgd_alloc(kvm->mm);
732 if (!kvm->arch.pgtable)
733 return -ENOMEM;
734 return 0;
735}
736
5a319350
PM		 737void kvmppc_free_radix(struct kvm *kvm)
738{
739 unsigned long ig, iu, im;
740 pte_t *pte;
741 pmd_t *pmd;
742 pud_t *pud;
743 pgd_t *pgd;
744
745 if (!kvm->arch.pgtable)
746 return;
747 pgd = kvm->arch.pgtable;
748 for (ig = 0; ig < PTRS_PER_PGD; ++ig, ++pgd) {
749 if (!pgd_present(*pgd))
750 continue;
751 pud = pud_offset(pgd, 0);
752 for (iu = 0; iu < PTRS_PER_PUD; ++iu, ++pud) {
753 if (!pud_present(*pud))
754 continue;
58c5c276
PM		 755 if (pud_huge(*pud)) {
756 pud_clear(pud);
757 continue;
758 }
5a319350
PM		 759 pmd = pmd_offset(pud, 0);
760 for (im = 0; im < PTRS_PER_PMD; ++im, ++pmd) {
c3856aeb 761 if (pmd_is_leaf(*pmd)) {
5a319350
PM		 762 pmd_clear(pmd);
763 continue;
764 }
765 if (!pmd_present(*pmd))
766 continue;
767 pte = pte_offset_map(pmd, 0);
768 memset(pte, 0, sizeof(long) << PTE_INDEX_SIZE);
769 kvmppc_pte_free(pte);
770 pmd_clear(pmd);
771 }
21828c99 772 kvmppc_pmd_free(pmd_offset(pud, 0));
5a319350
PM		 773 pud_clear(pud);
774 }
775 pud_free(kvm->mm, pud_offset(pgd, 0));
776 pgd_clear(pgd);
777 }
778 pgd_free(kvm->mm, kvm->arch.pgtable);
18c3640c 779 kvm->arch.pgtable = NULL;
5a319350
PM		 780}
781
782static void pte_ctor(void *addr)
783{
21828c99
AK		 784 memset(addr, 0, RADIX_PTE_TABLE_SIZE);
785}
786
787static void pmd_ctor(void *addr)
788{
789 memset(addr, 0, RADIX_PMD_TABLE_SIZE);
5a319350
PM		 790}
791
792int kvmppc_radix_init(void)
793{
21828c99 794 unsigned long size = sizeof(void *) << RADIX_PTE_INDEX_SIZE;
5a319350
PM		 795
796 kvm_pte_cache = kmem_cache_create("kvm-pte", size, size, 0, pte_ctor);
797 if (!kvm_pte_cache)
798 return -ENOMEM;
21828c99
AK		 799
800 size = sizeof(void *) << RADIX_PMD_INDEX_SIZE;
801
802 kvm_pmd_cache = kmem_cache_create("kvm-pmd", size, size, 0, pmd_ctor);
803 if (!kvm_pmd_cache) {
804 kmem_cache_destroy(kvm_pte_cache);
805 return -ENOMEM;
806 }
807
5a319350
PM		 808 return 0;
809}
810
811void kvmppc_radix_exit(void)
812{
813 kmem_cache_destroy(kvm_pte_cache);
21828c99 814 kmem_cache_destroy(kvm_pmd_cache);
5a319350 815}
Linux 6.x block layer and io_uring tree(s)