2 * PPC64 (POWER4) Huge TLB Page Support for Kernel.
4 * Copyright (C) 2003 David Gibson, IBM Corporation.
6 * Based on the IA-32 version:
7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
10 #include <linux/init.h>
13 #include <linux/hugetlb.h>
14 #include <linux/pagemap.h>
15 #include <linux/slab.h>
16 #include <linux/err.h>
17 #include <linux/sysctl.h>
19 #include <asm/pgalloc.h>
21 #include <asm/tlbflush.h>
22 #include <asm/mmu_context.h>
23 #include <asm/machdep.h>
24 #include <asm/cputable.h>
27 #define PAGE_SHIFT_64K 16
28 #define PAGE_SHIFT_16M 24
29 #define PAGE_SHIFT_16G 34
31 #define NUM_LOW_AREAS (0x100000000UL >> SID_SHIFT)
32 #define NUM_HIGH_AREAS (PGTABLE_RANGE >> HTLB_AREA_SHIFT)
33 #define MAX_NUMBER_GPAGES 1024
35 /* Tracks the 16G pages after the device tree is scanned and before the
36 * huge_boot_pages list is ready. */
37 static unsigned long gpage_freearray[MAX_NUMBER_GPAGES];
38 static unsigned nr_gpages;
40 /* Array of valid huge page sizes - non-zero value(hugepte_shift) is
41 * stored for the huge page sizes that are valid.
43 unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */
45 #define hugepte_shift mmu_huge_psizes
46 #define PTRS_PER_HUGEPTE(psize) (1 << hugepte_shift[psize])
47 #define HUGEPTE_TABLE_SIZE(psize) (sizeof(pte_t) << hugepte_shift[psize])
49 #define HUGEPD_SHIFT(psize) (mmu_psize_to_shift(psize) \
50 + hugepte_shift[psize])
51 #define HUGEPD_SIZE(psize) (1UL << HUGEPD_SHIFT(psize))
52 #define HUGEPD_MASK(psize) (~(HUGEPD_SIZE(psize)-1))
54 /* Subtract one from array size because we don't need a cache for 4K since
55 * is not a huge page size */
56 #define HUGE_PGTABLE_INDEX(psize) (HUGEPTE_CACHE_NUM + psize - 1)
57 #define HUGEPTE_CACHE_NAME(psize) (huge_pgtable_cache_name[psize])
59 static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = {
60 [MMU_PAGE_64K] = "hugepte_cache_64K",
61 [MMU_PAGE_1M] = "hugepte_cache_1M",
62 [MMU_PAGE_16M] = "hugepte_cache_16M",
63 [MMU_PAGE_16G] = "hugepte_cache_16G",
66 /* Flag to mark huge PD pointers. This means pmd_bad() and pud_bad()
67 * will choke on pointers to hugepte tables, which is handy for
68 * catching screwups early. */
71 typedef struct { unsigned long pd; } hugepd_t;
73 #define hugepd_none(hpd) ((hpd).pd == 0)
75 static inline int shift_to_mmu_psize(unsigned int shift)
78 #ifndef CONFIG_PPC_64K_PAGES
90 static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
92 if (mmu_psize_defs[mmu_psize].shift)
93 return mmu_psize_defs[mmu_psize].shift;
97 static inline pte_t *hugepd_page(hugepd_t hpd)
99 BUG_ON(!(hpd.pd & HUGEPD_OK));
100 return (pte_t *)(hpd.pd & ~HUGEPD_OK);
103 static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr,
104 struct hstate *hstate)
106 unsigned int shift = huge_page_shift(hstate);
107 int psize = shift_to_mmu_psize(shift);
108 unsigned long idx = ((addr >> shift) & (PTRS_PER_HUGEPTE(psize)-1));
109 pte_t *dir = hugepd_page(*hpdp);
114 static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
115 unsigned long address, unsigned int psize)
117 pte_t *new = kmem_cache_zalloc(pgtable_cache[HUGE_PGTABLE_INDEX(psize)],
118 GFP_KERNEL|__GFP_REPEAT);
123 spin_lock(&mm->page_table_lock);
124 if (!hugepd_none(*hpdp))
125 kmem_cache_free(pgtable_cache[HUGE_PGTABLE_INDEX(psize)], new);
127 hpdp->pd = (unsigned long)new | HUGEPD_OK;
128 spin_unlock(&mm->page_table_lock);
133 static pud_t *hpud_offset(pgd_t *pgd, unsigned long addr, struct hstate *hstate)
135 if (huge_page_shift(hstate) < PUD_SHIFT)
136 return pud_offset(pgd, addr);
138 return (pud_t *) pgd;
140 static pud_t *hpud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long addr,
141 struct hstate *hstate)
143 if (huge_page_shift(hstate) < PUD_SHIFT)
144 return pud_alloc(mm, pgd, addr);
146 return (pud_t *) pgd;
148 static pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
150 if (huge_page_shift(hstate) < PMD_SHIFT)
151 return pmd_offset(pud, addr);
153 return (pmd_t *) pud;
155 static pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
156 struct hstate *hstate)
158 if (huge_page_shift(hstate) < PMD_SHIFT)
159 return pmd_alloc(mm, pud, addr);
161 return (pmd_t *) pud;
164 /* Build list of addresses of gigantic pages. This function is used in early
165 * boot before the buddy or bootmem allocator is setup.
167 void add_gpage(unsigned long addr, unsigned long page_size,
168 unsigned long number_of_pages)
172 while (number_of_pages > 0) {
173 gpage_freearray[nr_gpages] = addr;
180 /* Moves the gigantic page addresses from the temporary list to the
181 * huge_boot_pages list.
183 int alloc_bootmem_huge_page(struct hstate *hstate)
185 struct huge_bootmem_page *m;
188 m = phys_to_virt(gpage_freearray[--nr_gpages]);
189 gpage_freearray[nr_gpages] = 0;
190 list_add(&m->list, &huge_boot_pages);
196 /* Modelled after find_linux_pte() */
197 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
206 struct hstate *hstate;
207 psize = get_slice_psize(mm, addr);
208 shift = mmu_psize_to_shift(psize);
209 sz = ((1UL) << shift);
210 hstate = size_to_hstate(sz);
212 addr &= hstate->mask;
214 pg = pgd_offset(mm, addr);
215 if (!pgd_none(*pg)) {
216 pu = hpud_offset(pg, addr, hstate);
217 if (!pud_none(*pu)) {
218 pm = hpmd_offset(pu, addr, hstate);
220 return hugepte_offset((hugepd_t *)pm, addr,
228 pte_t *huge_pte_alloc(struct mm_struct *mm,
229 unsigned long addr, unsigned long sz)
234 hugepd_t *hpdp = NULL;
235 struct hstate *hstate;
237 hstate = size_to_hstate(sz);
239 psize = get_slice_psize(mm, addr);
240 BUG_ON(!mmu_huge_psizes[psize]);
242 addr &= hstate->mask;
244 pg = pgd_offset(mm, addr);
245 pu = hpud_alloc(mm, pg, addr, hstate);
248 pm = hpmd_alloc(mm, pu, addr, hstate);
250 hpdp = (hugepd_t *)pm;
256 if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, psize))
259 return hugepte_offset(hpdp, addr, hstate);
262 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
267 static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp,
270 pte_t *hugepte = hugepd_page(*hpdp);
274 pgtable_free_tlb(tlb, pgtable_free_cache(hugepte,
275 HUGEPTE_CACHE_NUM+psize-1,
279 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
280 unsigned long addr, unsigned long end,
281 unsigned long floor, unsigned long ceiling,
289 pmd = pmd_offset(pud, addr);
291 next = pmd_addr_end(addr, end);
294 free_hugepte_range(tlb, (hugepd_t *)pmd, psize);
295 } while (pmd++, addr = next, addr != end);
305 if (end - 1 > ceiling - 1)
308 pmd = pmd_offset(pud, start);
310 pmd_free_tlb(tlb, pmd, start);
313 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
314 unsigned long addr, unsigned long end,
315 unsigned long floor, unsigned long ceiling)
321 unsigned int psize = get_slice_psize(tlb->mm, addr);
322 shift = mmu_psize_to_shift(psize);
325 pud = pud_offset(pgd, addr);
327 next = pud_addr_end(addr, end);
328 if (shift < PMD_SHIFT) {
329 if (pud_none_or_clear_bad(pud))
331 hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
336 free_hugepte_range(tlb, (hugepd_t *)pud, psize);
338 } while (pud++, addr = next, addr != end);
344 ceiling &= PGDIR_MASK;
348 if (end - 1 > ceiling - 1)
351 pud = pud_offset(pgd, start);
353 pud_free_tlb(tlb, pud, start);
357 * This function frees user-level page tables of a process.
359 * Must be called with pagetable lock held.
361 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
362 unsigned long addr, unsigned long end,
363 unsigned long floor, unsigned long ceiling)
370 * Comments below take from the normal free_pgd_range(). They
371 * apply here too. The tests against HUGEPD_MASK below are
372 * essential, because we *don't* test for this at the bottom
373 * level. Without them we'll attempt to free a hugepte table
374 * when we unmap just part of it, even if there are other
375 * active mappings using it.
377 * The next few lines have given us lots of grief...
379 * Why are we testing HUGEPD* at this top level? Because
380 * often there will be no work to do at all, and we'd prefer
381 * not to go all the way down to the bottom just to discover
384 * Why all these "- 1"s? Because 0 represents both the bottom
385 * of the address space and the top of it (using -1 for the
386 * top wouldn't help much: the masks would do the wrong thing).
387 * The rule is that addr 0 and floor 0 refer to the bottom of
388 * the address space, but end 0 and ceiling 0 refer to the top
389 * Comparisons need to use "end - 1" and "ceiling - 1" (though
390 * that end 0 case should be mythical).
392 * Wherever addr is brought up or ceiling brought down, we
393 * must be careful to reject "the opposite 0" before it
394 * confuses the subsequent tests. But what about where end is
395 * brought down by HUGEPD_SIZE below? no, end can't go down to
398 * Whereas we round start (addr) and ceiling down, by different
399 * masks at different levels, in order to test whether a table
400 * now has no other vmas using it, so can be freed, we don't
401 * bother to round floor or end up - the tests don't need that.
403 unsigned int psize = get_slice_psize(tlb->mm, addr);
405 addr &= HUGEPD_MASK(psize);
407 addr += HUGEPD_SIZE(psize);
412 ceiling &= HUGEPD_MASK(psize);
416 if (end - 1 > ceiling - 1)
417 end -= HUGEPD_SIZE(psize);
422 pgd = pgd_offset(tlb->mm, addr);
424 psize = get_slice_psize(tlb->mm, addr);
425 BUG_ON(!mmu_huge_psizes[psize]);
426 next = pgd_addr_end(addr, end);
427 if (mmu_psize_to_shift(psize) < PUD_SHIFT) {
428 if (pgd_none_or_clear_bad(pgd))
430 hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
434 free_hugepte_range(tlb, (hugepd_t *)pgd, psize);
436 } while (pgd++, addr = next, addr != end);
439 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
440 pte_t *ptep, pte_t pte)
442 if (pte_present(*ptep)) {
443 /* We open-code pte_clear because we need to pass the right
444 * argument to hpte_need_flush (huge / !huge). Might not be
445 * necessary anymore if we make hpte_need_flush() get the
446 * page size from the slices
448 unsigned int psize = get_slice_psize(mm, addr);
449 unsigned int shift = mmu_psize_to_shift(psize);
450 unsigned long sz = ((1UL) << shift);
451 struct hstate *hstate = size_to_hstate(sz);
452 pte_update(mm, addr & hstate->mask, ptep, ~0UL, 1);
454 *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
457 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
460 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
465 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
469 unsigned int mmu_psize = get_slice_psize(mm, address);
471 /* Verify it is a huge page else bail. */
472 if (!mmu_huge_psizes[mmu_psize])
473 return ERR_PTR(-EINVAL);
475 ptep = huge_pte_offset(mm, address);
476 page = pte_page(*ptep);
478 unsigned int shift = mmu_psize_to_shift(mmu_psize);
479 unsigned long sz = ((1UL) << shift);
480 page += (address % sz) / PAGE_SIZE;
486 int pmd_huge(pmd_t pmd)
491 int pud_huge(pud_t pud)
497 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
498 pmd_t *pmd, int write)
505 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
506 unsigned long len, unsigned long pgoff,
509 struct hstate *hstate = hstate_file(file);
510 int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
512 if (!mmu_huge_psizes[mmu_psize])
514 return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
517 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
519 unsigned int psize = get_slice_psize(vma->vm_mm, vma->vm_start);
521 return 1UL << mmu_psize_to_shift(psize);
525 * Called by asm hashtable.S for doing lazy icache flush
527 static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
528 pte_t pte, int trap, unsigned long sz)
533 if (!pfn_valid(pte_pfn(pte)))
536 page = pte_page(pte);
539 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
541 for (i = 0; i < (sz / PAGE_SIZE); i++)
542 __flush_dcache_icache(page_address(page+i));
543 set_bit(PG_arch_1, &page->flags);
551 int hash_huge_page(struct mm_struct *mm, unsigned long access,
552 unsigned long ea, unsigned long vsid, int local,
556 unsigned long old_pte, new_pte;
557 unsigned long va, rflags, pa, sz;
560 int ssize = user_segment_size(ea);
561 unsigned int mmu_psize;
563 mmu_psize = get_slice_psize(mm, ea);
565 if (!mmu_huge_psizes[mmu_psize])
567 ptep = huge_pte_offset(mm, ea);
569 /* Search the Linux page table for a match with va */
570 va = hpt_va(ea, vsid, ssize);
573 * If no pte found or not present, send the problem up to
576 if (unlikely(!ptep || pte_none(*ptep)))
580 * Check the user's access rights to the page. If access should be
581 * prevented then send the problem up to do_page_fault.
583 if (unlikely(access & ~pte_val(*ptep)))
586 * At this point, we have a pte (old_pte) which can be used to build
587 * or update an HPTE. There are 2 cases:
589 * 1. There is a valid (present) pte with no associated HPTE (this is
590 * the most common case)
591 * 2. There is a valid (present) pte with an associated HPTE. The
592 * current values of the pp bits in the HPTE prevent access
593 * because we are doing software DIRTY bit management and the
594 * page is currently not DIRTY.
599 old_pte = pte_val(*ptep);
600 if (old_pte & _PAGE_BUSY)
602 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
603 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
606 rflags = 0x2 | (!(new_pte & _PAGE_RW));
607 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
608 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
609 shift = mmu_psize_to_shift(mmu_psize);
610 sz = ((1UL) << shift);
611 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
612 /* No CPU has hugepages but lacks no execute, so we
613 * don't need to worry about that case */
614 rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
617 /* Check if pte already has an hpte (case 2) */
618 if (unlikely(old_pte & _PAGE_HASHPTE)) {
619 /* There MIGHT be an HPTE for this pte */
620 unsigned long hash, slot;
622 hash = hpt_hash(va, shift, ssize);
623 if (old_pte & _PAGE_F_SECOND)
625 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
626 slot += (old_pte & _PAGE_F_GIX) >> 12;
628 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
630 old_pte &= ~_PAGE_HPTEFLAGS;
633 if (likely(!(old_pte & _PAGE_HASHPTE))) {
634 unsigned long hash = hpt_hash(va, shift, ssize);
635 unsigned long hpte_group;
637 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
640 hpte_group = ((hash & htab_hash_mask) *
641 HPTES_PER_GROUP) & ~0x7UL;
643 /* clear HPTE slot informations in new PTE */
644 #ifdef CONFIG_PPC_64K_PAGES
645 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
647 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
649 /* Add in WIMG bits */
650 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
651 _PAGE_COHERENT | _PAGE_GUARDED));
653 /* Insert into the hash table, primary slot */
654 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
657 /* Primary is full, try the secondary */
658 if (unlikely(slot == -1)) {
659 hpte_group = ((~hash & htab_hash_mask) *
660 HPTES_PER_GROUP) & ~0x7UL;
661 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
666 hpte_group = ((hash & htab_hash_mask) *
667 HPTES_PER_GROUP)&~0x7UL;
669 ppc_md.hpte_remove(hpte_group);
674 if (unlikely(slot == -2))
675 panic("hash_huge_page: pte_insert failed\n");
677 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
681 * No need to use ldarx/stdcx here
683 *ptep = __pte(new_pte & ~_PAGE_BUSY);
691 static void __init set_huge_psize(int psize)
693 /* Check that it is a page size supported by the hardware and
694 * that it fits within pagetable limits. */
695 if (mmu_psize_defs[psize].shift &&
696 mmu_psize_defs[psize].shift < SID_SHIFT_1T &&
697 (mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT ||
698 mmu_psize_defs[psize].shift == PAGE_SHIFT_64K ||
699 mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) {
700 /* Return if huge page size has already been setup or is the
701 * same as the base page size. */
702 if (mmu_huge_psizes[psize] ||
703 mmu_psize_defs[psize].shift == PAGE_SHIFT)
705 if (WARN_ON(HUGEPTE_CACHE_NAME(psize) == NULL))
707 hugetlb_add_hstate(mmu_psize_defs[psize].shift - PAGE_SHIFT);
709 switch (mmu_psize_defs[psize].shift) {
711 /* We only allow 64k hpages with 4k base page,
712 * which was checked above, and always put them
714 hugepte_shift[psize] = PMD_SHIFT;
717 /* 16M pages can be at two different levels
718 * of pagestables based on base page size */
719 if (PAGE_SHIFT == PAGE_SHIFT_64K)
720 hugepte_shift[psize] = PMD_SHIFT;
721 else /* 4k base page */
722 hugepte_shift[psize] = PUD_SHIFT;
725 /* 16G pages are always at PGD level */
726 hugepte_shift[psize] = PGDIR_SHIFT;
729 hugepte_shift[psize] -= mmu_psize_defs[psize].shift;
731 hugepte_shift[psize] = 0;
734 static int __init hugepage_setup_sz(char *str)
736 unsigned long long size;
740 size = memparse(str, &str);
743 mmu_psize = shift_to_mmu_psize(shift);
744 if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift)
745 set_huge_psize(mmu_psize);
747 printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
751 __setup("hugepagesz=", hugepage_setup_sz);
753 static int __init hugetlbpage_init(void)
757 if (!cpu_has_feature(CPU_FTR_16M_PAGE))
760 /* Add supported huge page sizes. Need to change HUGE_MAX_HSTATE
761 * and adjust PTE_NONCACHE_NUM if the number of supported huge page
764 set_huge_psize(MMU_PAGE_16M);
765 set_huge_psize(MMU_PAGE_16G);
767 /* Temporarily disable support for 64K huge pages when 64K SPU local
768 * store support is enabled as the current implementation conflicts.
770 #ifndef CONFIG_SPU_FS_64K_LS
771 set_huge_psize(MMU_PAGE_64K);
774 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
775 if (mmu_huge_psizes[psize]) {
776 pgtable_cache[HUGE_PGTABLE_INDEX(psize)] =
778 HUGEPTE_CACHE_NAME(psize),
779 HUGEPTE_TABLE_SIZE(psize),
780 HUGEPTE_TABLE_SIZE(psize),
783 if (!pgtable_cache[HUGE_PGTABLE_INDEX(psize)])
784 panic("hugetlbpage_init(): could not create %s"\
785 "\n", HUGEPTE_CACHE_NAME(psize));
792 module_init(hugetlbpage_init);