2 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
10 #include <linux/sched.h>
11 #include <linux/mm_types.h>
12 #include <linux/memblock.h>
13 #include <misc/cxl-base.h>
15 #include <asm/pgalloc.h>
17 #include <asm/trace.h>
18 #include <asm/powernv.h>
21 #include <trace/events/thp.h>
23 int (*register_process_table)(unsigned long base, unsigned long page_size,
24 unsigned long tbl_size);
26 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
28 * This is called when relaxing access to a hugepage. It's also called in the page
29 * fault path when we don't hit any of the major fault cases, ie, a minor
30 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have
31 * handled those two for us, we additionally deal with missing execute
32 * permission here on some processors
34 int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
35 pmd_t *pmdp, pmd_t entry, int dirty)
38 #ifdef CONFIG_DEBUG_VM
39 WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
40 assert_spin_locked(&vma->vm_mm->page_table_lock);
42 changed = !pmd_same(*(pmdp), entry);
44 __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp),
45 pmd_pte(entry), address);
46 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
51 int pmdp_test_and_clear_young(struct vm_area_struct *vma,
52 unsigned long address, pmd_t *pmdp)
54 return __pmdp_test_and_clear_young(vma->vm_mm, address, pmdp);
57 * set a new huge pmd. We should not be called for updating
58 * an existing pmd entry. That should go via pmd_hugepage_update.
60 void set_pmd_at(struct mm_struct *mm, unsigned long addr,
61 pmd_t *pmdp, pmd_t pmd)
63 #ifdef CONFIG_DEBUG_VM
64 WARN_ON(pte_present(pmd_pte(*pmdp)) && !pte_protnone(pmd_pte(*pmdp)));
65 assert_spin_locked(&mm->page_table_lock);
66 WARN_ON(!(pmd_trans_huge(pmd) || pmd_devmap(pmd)));
68 trace_hugepage_set_pmd(addr, pmd_val(pmd));
69 return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd));
72 static void do_nothing(void *unused)
77 * Serialize against find_current_mm_pte which does lock-less
78 * lookup in page tables with local interrupts disabled. For huge pages
79 * it casts pmd_t to pte_t. Since format of pte_t is different from
80 * pmd_t we want to prevent transit from pmd pointing to page table
81 * to pmd pointing to huge page (and back) while interrupts are disabled.
82 * We clear pmd to possibly replace it with page table pointer in
83 * different code paths. So make sure we wait for the parallel
84 * find_current_mm_pte to finish.
86 void serialize_against_pte_lookup(struct mm_struct *mm)
89 smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
93 * We use this to invalidate a pmdp entry before switching from a
94 * hugepte to regular pmd entry.
96 pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
99 unsigned long old_pmd;
101 old_pmd = pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
102 flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
104 * This ensures that generic code that rely on IRQ disabling
105 * to prevent a parallel THP split work as expected.
107 serialize_against_pte_lookup(vma->vm_mm);
108 return __pmd(old_pmd);
111 static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
113 return __pmd(pmd_val(pmd) | pgprot_val(pgprot));
116 pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
120 pmdv = (pfn << PAGE_SHIFT) & PTE_RPN_MASK;
121 return pmd_set_protbits(__pmd(pmdv), pgprot);
124 pmd_t mk_pmd(struct page *page, pgprot_t pgprot)
126 return pfn_pmd(page_to_pfn(page), pgprot);
129 pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
134 pmdv &= _HPAGE_CHG_MASK;
135 return pmd_set_protbits(__pmd(pmdv), newprot);
139 * This is called at the end of handling a user page fault, when the
140 * fault has been handled by updating a HUGE PMD entry in the linux page tables.
141 * We use it to preload an HPTE into the hash table corresponding to
142 * the updated linux HUGE PMD entry.
144 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
149 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
151 /* For use by kexec */
152 void mmu_cleanup_all(void)
155 radix__mmu_cleanup_all();
156 else if (mmu_hash_ops.hpte_clear_all)
157 mmu_hash_ops.hpte_clear_all();
160 #ifdef CONFIG_MEMORY_HOTPLUG
161 int __meminit create_section_mapping(unsigned long start, unsigned long end, int nid)
164 return radix__create_section_mapping(start, end, nid);
166 return hash__create_section_mapping(start, end, nid);
169 int __meminit remove_section_mapping(unsigned long start, unsigned long end)
172 return radix__remove_section_mapping(start, end);
174 return hash__remove_section_mapping(start, end);
176 #endif /* CONFIG_MEMORY_HOTPLUG */
178 void __init mmu_partition_table_init(void)
180 unsigned long patb_size = 1UL << PATB_SIZE_SHIFT;
183 BUILD_BUG_ON_MSG((PATB_SIZE_SHIFT > 36), "Partition table size too large.");
184 partition_tb = __va(memblock_alloc_base(patb_size, patb_size,
185 MEMBLOCK_ALLOC_ANYWHERE));
187 /* Initialize the Partition Table with no entries */
188 memset((void *)partition_tb, 0, patb_size);
191 * update partition table control register,
194 ptcr = __pa(partition_tb) | (PATB_SIZE_SHIFT - 12);
195 mtspr(SPRN_PTCR, ptcr);
196 powernv_set_nmmu_ptcr(ptcr);
199 void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
202 unsigned long old = be64_to_cpu(partition_tb[lpid].patb0);
204 partition_tb[lpid].patb0 = cpu_to_be64(dw0);
205 partition_tb[lpid].patb1 = cpu_to_be64(dw1);
208 * Global flush of TLBs and partition table caches for this lpid.
209 * The type of flush (hash or radix) depends on what the previous
210 * use of this partition ID was, not the new use.
212 asm volatile("ptesync" : : : "memory");
214 asm volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
215 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
216 asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
217 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
218 trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 1);
220 asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
221 "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
222 trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
224 /* do we need fixup here ?*/
225 asm volatile("eieio; tlbsync; ptesync" : : : "memory");
227 EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);