Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
6d779079 GS |
2 | #ifndef _ASM_POWERPC_HUGETLB_H |
3 | #define _ASM_POWERPC_HUGETLB_H | |
4 | ||
41151e77 | 5 | #ifdef CONFIG_HUGETLB_PAGE |
6d779079 | 6 | #include <asm/page.h> |
106c992a | 7 | #include <asm-generic/hugetlb.h> |
6d779079 | 8 | |
41151e77 | 9 | extern struct kmem_cache *hugepte_cache; |
41151e77 | 10 | |
cf9427b8 | 11 | #ifdef CONFIG_PPC_BOOK3S_64 |
48483760 | 12 | |
bee8b3b5 | 13 | #include <asm/book3s/64/hugetlb.h> |
cf9427b8 AK |
14 | /* |
15 | * This should work for other subarchs too. But right now we use the | |
16 | * new format only for 64bit book3s | |
17 | */ | |
18 | static inline pte_t *hugepd_page(hugepd_t hpd) | |
19 | { | |
20 | BUG_ON(!hugepd_ok(hpd)); | |
21 | /* | |
22 | * We have only four bits to encode, MMU page size | |
23 | */ | |
24 | BUILD_BUG_ON((MMU_PAGE_COUNT - 1) > 0xf); | |
20717e1f | 25 | return __va(hpd_val(hpd) & HUGEPD_ADDR_MASK); |
cf9427b8 AK |
26 | } |
27 | ||
28 | static inline unsigned int hugepd_mmu_psize(hugepd_t hpd) | |
29 | { | |
20717e1f | 30 | return (hpd_val(hpd) & HUGEPD_SHIFT_MASK) >> 2; |
cf9427b8 AK |
31 | } |
32 | ||
33 | static inline unsigned int hugepd_shift(hugepd_t hpd) | |
34 | { | |
35 | return mmu_psize_to_shift(hugepd_mmu_psize(hpd)); | |
36 | } | |
48483760 AK |
37 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, |
38 | unsigned long vmaddr) | |
39 | { | |
40 | if (radix_enabled()) | |
41 | return radix__flush_hugetlb_page(vma, vmaddr); | |
42 | } | |
cf9427b8 | 43 | |
48483760 AK |
44 | static inline void __local_flush_hugetlb_page(struct vm_area_struct *vma, |
45 | unsigned long vmaddr) | |
46 | { | |
47 | if (radix_enabled()) | |
48 | return radix__local_flush_hugetlb_page(vma, vmaddr); | |
49 | } | |
cf9427b8 AK |
50 | #else |
51 | ||
41151e77 BB |
52 | static inline pte_t *hugepd_page(hugepd_t hpd) |
53 | { | |
54 | BUG_ON(!hugepd_ok(hpd)); | |
4b914286 | 55 | #ifdef CONFIG_PPC_8xx |
20717e1f AK |
56 | return (pte_t *)__va(hpd_val(hpd) & |
57 | ~(_PMD_PAGE_MASK | _PMD_PRESENT_MASK)); | |
4b914286 | 58 | #else |
20717e1f AK |
59 | return (pte_t *)((hpd_val(hpd) & |
60 | ~HUGEPD_SHIFT_MASK) | PD_HUGE); | |
4b914286 | 61 | #endif |
41151e77 BB |
62 | } |
63 | ||
64 | static inline unsigned int hugepd_shift(hugepd_t hpd) | |
65 | { | |
4b914286 | 66 | #ifdef CONFIG_PPC_8xx |
20717e1f | 67 | return ((hpd_val(hpd) & _PMD_PAGE_MASK) >> 1) + 17; |
4b914286 | 68 | #else |
20717e1f | 69 | return hpd_val(hpd) & HUGEPD_SHIFT_MASK; |
4b914286 | 70 | #endif |
41151e77 BB |
71 | } |
72 | ||
cf9427b8 AK |
73 | #endif /* CONFIG_PPC_BOOK3S_64 */ |
74 | ||
75 | ||
b30e7590 | 76 | static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, |
41151e77 BB |
77 | unsigned pdshift) |
78 | { | |
79 | /* | |
881fde1d BB |
80 | * On FSL BookE, we have multiple higher-level table entries that |
81 | * point to the same hugepte. Just use the first one since they're all | |
41151e77 BB |
82 | * identical. So for that case, idx=0. |
83 | */ | |
84 | unsigned long idx = 0; | |
85 | ||
b30e7590 | 86 | pte_t *dir = hugepd_page(hpd); |
881fde1d | 87 | #ifndef CONFIG_PPC_FSL_BOOK3E |
b30e7590 | 88 | idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(hpd); |
41151e77 BB |
89 | #endif |
90 | ||
91 | return dir + idx; | |
92 | } | |
93 | ||
883a3e52 DG |
94 | pte_t *huge_pte_offset_and_shift(struct mm_struct *mm, |
95 | unsigned long addr, unsigned *shift); | |
96 | ||
0895ecda DG |
97 | void flush_dcache_icache_hugepage(struct page *page); |
98 | ||
ca5f1d16 | 99 | #if defined(CONFIG_PPC_MM_SLICES) |
6d779079 GS |
100 | int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, |
101 | unsigned long len); | |
41151e77 BB |
102 | #else |
103 | static inline int is_hugepage_only_range(struct mm_struct *mm, | |
104 | unsigned long addr, | |
105 | unsigned long len) | |
106 | { | |
107 | return 0; | |
108 | } | |
109 | #endif | |
110 | ||
d93e4d7d BB |
111 | void book3e_hugetlb_preload(struct vm_area_struct *vma, unsigned long ea, |
112 | pte_t pte); | |
4b914286 CL |
113 | #ifdef CONFIG_PPC_8xx |
114 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, | |
115 | unsigned long vmaddr) | |
116 | { | |
117 | flush_tlb_page(vma, vmaddr); | |
118 | } | |
119 | #else | |
41151e77 | 120 | void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); |
4b914286 | 121 | #endif |
6d779079 | 122 | |
42b77728 | 123 | void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, |
6d779079 GS |
124 | unsigned long end, unsigned long floor, |
125 | unsigned long ceiling); | |
126 | ||
3340289d MG |
127 | /* |
128 | * The version of vma_mmu_pagesize() in arch/powerpc/mm/hugetlbpage.c needs | |
129 | * to override the version in mm/hugetlb.c | |
130 | */ | |
131 | #define vma_mmu_pagesize vma_mmu_pagesize | |
132 | ||
6d779079 GS |
133 | /* |
134 | * If the arch doesn't supply something else, assume that hugepage | |
135 | * size aligned regions are ok without further preparation. | |
136 | */ | |
a5516438 AK |
137 | static inline int prepare_hugepage_range(struct file *file, |
138 | unsigned long addr, unsigned long len) | |
6d779079 | 139 | { |
0d9ea754 JT |
140 | struct hstate *h = hstate_file(file); |
141 | if (len & ~huge_page_mask(h)) | |
6d779079 | 142 | return -EINVAL; |
0d9ea754 | 143 | if (addr & ~huge_page_mask(h)) |
6d779079 GS |
144 | return -EINVAL; |
145 | return 0; | |
146 | } | |
147 | ||
0895ecda DG |
148 | static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, |
149 | pte_t *ptep, pte_t pte) | |
150 | { | |
151 | set_pte_at(mm, addr, ptep, pte); | |
152 | } | |
153 | ||
154 | static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, | |
155 | unsigned long addr, pte_t *ptep) | |
156 | { | |
41151e77 | 157 | #ifdef CONFIG_PPC64 |
88247e8d | 158 | return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); |
41151e77 BB |
159 | #else |
160 | return __pte(pte_update(ptep, ~0UL, 0)); | |
161 | #endif | |
0895ecda DG |
162 | } |
163 | ||
8fe627ec GS |
164 | static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, |
165 | unsigned long addr, pte_t *ptep) | |
166 | { | |
0895ecda DG |
167 | pte_t pte; |
168 | pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); | |
13dce033 | 169 | flush_hugetlb_page(vma, addr); |
8fe627ec GS |
170 | } |
171 | ||
7f2e9525 GS |
172 | static inline int huge_pte_none(pte_t pte) |
173 | { | |
174 | return pte_none(pte); | |
175 | } | |
176 | ||
177 | static inline pte_t huge_pte_wrprotect(pte_t pte) | |
178 | { | |
179 | return pte_wrprotect(pte); | |
180 | } | |
181 | ||
7f2e9525 GS |
182 | static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, |
183 | unsigned long addr, pte_t *ptep, | |
184 | pte_t pte, int dirty) | |
185 | { | |
1f6820b4 | 186 | #ifdef HUGETLB_NEED_PRELOAD |
97632e6f BB |
187 | /* |
188 | * The "return 1" forces a call of update_mmu_cache, which will write a | |
189 | * TLB entry. Without this, platforms that don't do a write of the TLB | |
190 | * entry in the TLB miss handler asm will fault ad infinitum. | |
191 | */ | |
192 | ptep_set_access_flags(vma, addr, ptep, pte, dirty); | |
193 | return 1; | |
194 | #else | |
7f2e9525 | 195 | return ptep_set_access_flags(vma, addr, ptep, pte, dirty); |
97632e6f | 196 | #endif |
7f2e9525 GS |
197 | } |
198 | ||
199 | static inline pte_t huge_ptep_get(pte_t *ptep) | |
200 | { | |
201 | return *ptep; | |
202 | } | |
203 | ||
5d3a551c WD |
204 | static inline void arch_clear_hugepage_flags(struct page *page) |
205 | { | |
206 | } | |
207 | ||
41151e77 | 208 | #else /* ! CONFIG_HUGETLB_PAGE */ |
41151e77 BB |
209 | static inline void flush_hugetlb_page(struct vm_area_struct *vma, |
210 | unsigned long vmaddr) | |
211 | { | |
212 | } | |
a6146888 | 213 | |
29409997 | 214 | #define hugepd_shift(x) 0 |
b30e7590 | 215 | static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, |
29409997 AK |
216 | unsigned pdshift) |
217 | { | |
218 | return 0; | |
219 | } | |
220 | #endif /* CONFIG_HUGETLB_PAGE */ | |
a6146888 | 221 | |
6d779079 | 222 | #endif /* _ASM_POWERPC_HUGETLB_H */ |