Commit | Line | Data |
---|---|---|
047ea784 PM |
1 | #ifndef _ASM_POWERPC_PGTABLE_H |
2 | #define _ASM_POWERPC_PGTABLE_H | |
88ced031 | 3 | #ifdef __KERNEL__ |
047ea784 | 4 | |
9c709f3b | 5 | #ifndef __ASSEMBLY__ |
c34a51ce | 6 | #include <linux/mmdebug.h> |
9c709f3b DG |
7 | #include <asm/processor.h> /* For TASK_SIZE */ |
8 | #include <asm/mmu.h> | |
9 | #include <asm/page.h> | |
8d30c14c | 10 | |
9c709f3b | 11 | struct mm_struct; |
8d30c14c | 12 | |
9c709f3b DG |
13 | #endif /* !__ASSEMBLY__ */ |
14 | ||
f88df14b DG |
15 | #if defined(CONFIG_PPC64) |
16 | # include <asm/pgtable-ppc64.h> | |
047ea784 | 17 | #else |
f88df14b | 18 | # include <asm/pgtable-ppc32.h> |
e28f7faf | 19 | #endif |
1da177e4 | 20 | |
cc3665a6 AK |
21 | /* |
22 | * We save the slot number & secondary bit in the second half of the | |
23 | * PTE page. We use the 8 bytes per each pte entry. | |
24 | */ | |
25 | #define PTE_PAGE_HIDX_OFFSET (PTRS_PER_PTE * 8) | |
26 | ||
1da177e4 | 27 | #ifndef __ASSEMBLY__ |
64b3d0e8 | 28 | |
78f1dbde AK |
29 | #include <asm/tlbflush.h> |
30 | ||
71087002 BH |
31 | /* Generic accessors to PTE bits */ |
32 | static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } | |
33 | static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } | |
34 | static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } | |
35 | static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE; } | |
36 | static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; } | |
71087002 BH |
37 | static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; } |
38 | static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); } | |
39 | ||
c34a51ce AK |
40 | #ifdef CONFIG_NUMA_BALANCING |
41 | ||
42 | static inline int pte_present(pte_t pte) | |
43 | { | |
44 | return pte_val(pte) & (_PAGE_PRESENT | _PAGE_NUMA); | |
45 | } | |
46 | ||
47 | #define pte_numa pte_numa | |
48 | static inline int pte_numa(pte_t pte) | |
49 | { | |
50 | return (pte_val(pte) & | |
51 | (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA; | |
52 | } | |
53 | ||
54 | #define pte_mknonnuma pte_mknonnuma | |
55 | static inline pte_t pte_mknonnuma(pte_t pte) | |
56 | { | |
57 | pte_val(pte) &= ~_PAGE_NUMA; | |
58 | pte_val(pte) |= _PAGE_PRESENT | _PAGE_ACCESSED; | |
59 | return pte; | |
60 | } | |
61 | ||
62 | #define pte_mknuma pte_mknuma | |
63 | static inline pte_t pte_mknuma(pte_t pte) | |
64 | { | |
65 | /* | |
66 | * We should not set _PAGE_NUMA on non present ptes. Also clear the | |
67 | * present bit so that hash_page will return 1 and we collect this | |
68 | * as numa fault. | |
69 | */ | |
70 | if (pte_present(pte)) { | |
71 | pte_val(pte) |= _PAGE_NUMA; | |
72 | pte_val(pte) &= ~_PAGE_PRESENT; | |
73 | } else | |
74 | VM_BUG_ON(1); | |
75 | return pte; | |
76 | } | |
77 | ||
56eecdb9 AK |
78 | #define ptep_set_numa ptep_set_numa |
79 | static inline void ptep_set_numa(struct mm_struct *mm, unsigned long addr, | |
80 | pte_t *ptep) | |
81 | { | |
82 | if ((pte_val(*ptep) & _PAGE_PRESENT) == 0) | |
83 | VM_BUG_ON(1); | |
84 | ||
85 | pte_update(mm, addr, ptep, _PAGE_PRESENT, _PAGE_NUMA, 0); | |
86 | return; | |
87 | } | |
88 | ||
c34a51ce AK |
89 | #define pmd_numa pmd_numa |
90 | static inline int pmd_numa(pmd_t pmd) | |
91 | { | |
92 | return pte_numa(pmd_pte(pmd)); | |
93 | } | |
94 | ||
56eecdb9 AK |
95 | #define pmdp_set_numa pmdp_set_numa |
96 | static inline void pmdp_set_numa(struct mm_struct *mm, unsigned long addr, | |
97 | pmd_t *pmdp) | |
98 | { | |
99 | if ((pmd_val(*pmdp) & _PAGE_PRESENT) == 0) | |
100 | VM_BUG_ON(1); | |
101 | ||
102 | pmd_hugepage_update(mm, addr, pmdp, _PAGE_PRESENT, _PAGE_NUMA); | |
103 | return; | |
104 | } | |
105 | ||
c34a51ce AK |
106 | #define pmd_mknonnuma pmd_mknonnuma |
107 | static inline pmd_t pmd_mknonnuma(pmd_t pmd) | |
108 | { | |
109 | return pte_pmd(pte_mknonnuma(pmd_pte(pmd))); | |
110 | } | |
111 | ||
112 | #define pmd_mknuma pmd_mknuma | |
113 | static inline pmd_t pmd_mknuma(pmd_t pmd) | |
114 | { | |
115 | return pte_pmd(pte_mknuma(pmd_pte(pmd))); | |
116 | } | |
117 | ||
118 | # else | |
119 | ||
120 | static inline int pte_present(pte_t pte) | |
121 | { | |
122 | return pte_val(pte) & _PAGE_PRESENT; | |
123 | } | |
124 | #endif /* CONFIG_NUMA_BALANCING */ | |
125 | ||
71087002 BH |
126 | /* Conversion functions: convert a page and protection to a page entry, |
127 | * and a page entry and page directory to the page they refer to. | |
128 | * | |
129 | * Even if PTEs can be unsigned long long, a PFN is always an unsigned | |
130 | * long for now. | |
131 | */ | |
132 | static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { | |
133 | return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | | |
134 | pgprot_val(pgprot)); } | |
135 | static inline unsigned long pte_pfn(pte_t pte) { | |
136 | return pte_val(pte) >> PTE_RPN_SHIFT; } | |
137 | ||
138 | /* Keep these as a macros to avoid include dependency mess */ | |
139 | #define pte_page(x) pfn_to_page(pte_pfn(x)) | |
140 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | |
141 | ||
142 | /* Generic modifiers for PTE bits */ | |
143 | static inline pte_t pte_wrprotect(pte_t pte) { | |
144 | pte_val(pte) &= ~(_PAGE_RW | _PAGE_HWWRITE); return pte; } | |
145 | static inline pte_t pte_mkclean(pte_t pte) { | |
146 | pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_HWWRITE); return pte; } | |
147 | static inline pte_t pte_mkold(pte_t pte) { | |
148 | pte_val(pte) &= ~_PAGE_ACCESSED; return pte; } | |
149 | static inline pte_t pte_mkwrite(pte_t pte) { | |
150 | pte_val(pte) |= _PAGE_RW; return pte; } | |
151 | static inline pte_t pte_mkdirty(pte_t pte) { | |
152 | pte_val(pte) |= _PAGE_DIRTY; return pte; } | |
153 | static inline pte_t pte_mkyoung(pte_t pte) { | |
154 | pte_val(pte) |= _PAGE_ACCESSED; return pte; } | |
155 | static inline pte_t pte_mkspecial(pte_t pte) { | |
156 | pte_val(pte) |= _PAGE_SPECIAL; return pte; } | |
157 | static inline pte_t pte_mkhuge(pte_t pte) { | |
158 | return pte; } | |
159 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) | |
160 | { | |
161 | pte_val(pte) = (pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot); | |
162 | return pte; | |
163 | } | |
164 | ||
165 | ||
8d30c14c BH |
166 | /* Insert a PTE, top-level function is out of line. It uses an inline |
167 | * low level function in the respective pgtable-* files | |
168 | */ | |
169 | extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, | |
170 | pte_t pte); | |
171 | ||
172 | /* This low level function performs the actual PTE insertion | |
173 | * Setting the PTE depends on the MMU type and other factors. It's | |
174 | * an horrible mess that I'm not going to try to clean up now but | |
175 | * I'm keeping it in one place rather than spread around | |
176 | */ | |
177 | static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, | |
178 | pte_t *ptep, pte_t pte, int percpu) | |
179 | { | |
180 | #if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT) | |
181 | /* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the | |
182 | * helper pte_update() which does an atomic update. We need to do that | |
183 | * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a | |
184 | * per-CPU PTE such as a kmap_atomic, we do a simple update preserving | |
185 | * the hash bits instead (ie, same as the non-SMP case) | |
186 | */ | |
187 | if (percpu) | |
188 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) | |
189 | | (pte_val(pte) & ~_PAGE_HASHPTE)); | |
190 | else | |
191 | pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte)); | |
192 | ||
1660e9d3 PM |
193 | #elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) |
194 | /* Second case is 32-bit with 64-bit PTE. In this case, we | |
8d30c14c BH |
195 | * can just store as long as we do the two halves in the right order |
196 | * with a barrier in between. This is possible because we take care, | |
197 | * in the hash code, to pre-invalidate if the PTE was already hashed, | |
198 | * which synchronizes us with any concurrent invalidation. | |
199 | * In the percpu case, we also fallback to the simple update preserving | |
200 | * the hash bits | |
201 | */ | |
202 | if (percpu) { | |
203 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) | |
204 | | (pte_val(pte) & ~_PAGE_HASHPTE)); | |
205 | return; | |
206 | } | |
207 | #if _PAGE_HASHPTE != 0 | |
208 | if (pte_val(*ptep) & _PAGE_HASHPTE) | |
209 | flush_hash_entry(mm, ptep, addr); | |
210 | #endif | |
211 | __asm__ __volatile__("\ | |
212 | stw%U0%X0 %2,%0\n\ | |
213 | eieio\n\ | |
214 | stw%U0%X0 %L2,%1" | |
215 | : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4)) | |
216 | : "r" (pte) : "memory"); | |
217 | ||
218 | #elif defined(CONFIG_PPC_STD_MMU_32) | |
219 | /* Third case is 32-bit hash table in UP mode, we need to preserve | |
220 | * the _PAGE_HASHPTE bit since we may not have invalidated the previous | |
221 | * translation in the hash yet (done in a subsequent flush_tlb_xxx()) | |
222 | * and see we need to keep track that this PTE needs invalidating | |
223 | */ | |
224 | *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE) | |
225 | | (pte_val(pte) & ~_PAGE_HASHPTE)); | |
226 | ||
227 | #else | |
228 | /* Anything else just stores the PTE normally. That covers all 64-bit | |
1660e9d3 | 229 | * cases, and 32-bit non-hash with 32-bit PTEs. |
8d30c14c BH |
230 | */ |
231 | *ptep = pte; | |
232 | #endif | |
233 | } | |
234 | ||
235 | ||
236 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS | |
237 | extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, | |
238 | pte_t *ptep, pte_t entry, int dirty); | |
239 | ||
64b3d0e8 BH |
240 | /* |
241 | * Macro to mark a page protection value as "uncacheable". | |
242 | */ | |
243 | ||
244 | #define _PAGE_CACHE_CTL (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \ | |
245 | _PAGE_WRITETHRU) | |
246 | ||
247 | #define pgprot_noncached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ | |
248 | _PAGE_NO_CACHE | _PAGE_GUARDED)) | |
249 | ||
250 | #define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ | |
251 | _PAGE_NO_CACHE)) | |
252 | ||
253 | #define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ | |
254 | _PAGE_COHERENT)) | |
255 | ||
256 | #define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ | |
257 | _PAGE_COHERENT | _PAGE_WRITETHRU)) | |
258 | ||
09c188c4 GT |
259 | #define pgprot_cached_noncoherent(prot) \ |
260 | (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) | |
261 | ||
fe3cc0d9 | 262 | #define pgprot_writecombine pgprot_noncached_wc |
64b3d0e8 BH |
263 | |
264 | struct file; | |
265 | extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, | |
266 | unsigned long size, pgprot_t vma_prot); | |
267 | #define __HAVE_PHYS_MEM_ACCESS_PROT | |
268 | ||
9c709f3b DG |
269 | /* |
270 | * ZERO_PAGE is a global shared page that is always zero: used | |
271 | * for zero-mapped memory areas etc.. | |
272 | */ | |
273 | extern unsigned long empty_zero_page[]; | |
274 | #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) | |
275 | ||
276 | extern pgd_t swapper_pg_dir[]; | |
277 | ||
278 | extern void paging_init(void); | |
279 | ||
280 | /* | |
281 | * kern_addr_valid is intended to indicate whether an address is a valid | |
282 | * kernel address. Most 32-bit archs define it as always true (like this) | |
283 | * but most 64-bit archs actually perform a test. What should we do here? | |
284 | */ | |
285 | #define kern_addr_valid(addr) (1) | |
286 | ||
1da177e4 | 287 | #include <asm-generic/pgtable.h> |
1e3519f8 BH |
288 | |
289 | ||
290 | /* | |
291 | * This gets called at the end of handling a page fault, when | |
292 | * the kernel has put a new PTE into the page table for the process. | |
293 | * We use it to ensure coherency between the i-cache and d-cache | |
294 | * for the page which has just been mapped in. | |
295 | * On machines which use an MMU hash table, we use this to put a | |
296 | * corresponding HPTE into the hash table ahead of time, instead of | |
297 | * waiting for the inevitable extra hash-table miss exception. | |
298 | */ | |
4b3073e1 | 299 | extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); |
1e3519f8 | 300 | |
a4fe3ce7 DG |
301 | extern int gup_hugepd(hugepd_t *hugepd, unsigned pdshift, unsigned long addr, |
302 | unsigned long end, int write, struct page **pages, int *nr); | |
303 | ||
e2b3d202 AK |
304 | extern int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, |
305 | unsigned long end, int write, struct page **pages, int *nr); | |
074c2eae AK |
306 | #ifndef CONFIG_TRANSPARENT_HUGEPAGE |
307 | #define pmd_large(pmd) 0 | |
308 | #define has_transparent_hugepage() 0 | |
309 | #endif | |
29409997 AK |
310 | pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, |
311 | unsigned *shift); | |
f5e3fe09 BB |
312 | |
313 | static inline pte_t *lookup_linux_ptep(pgd_t *pgdir, unsigned long hva, | |
314 | unsigned long *pte_sizep) | |
315 | { | |
316 | pte_t *ptep; | |
317 | unsigned long ps = *pte_sizep; | |
318 | unsigned int shift; | |
319 | ||
320 | ptep = find_linux_pte_or_hugepte(pgdir, hva, &shift); | |
321 | if (!ptep) | |
322 | return NULL; | |
323 | if (shift) | |
324 | *pte_sizep = 1ul << shift; | |
325 | else | |
326 | *pte_sizep = PAGE_SIZE; | |
327 | ||
328 | if (ps > *pte_sizep) | |
329 | return NULL; | |
330 | ||
331 | return ptep; | |
332 | } | |
1da177e4 LT |
333 | #endif /* __ASSEMBLY__ */ |
334 | ||
88ced031 | 335 | #endif /* __KERNEL__ */ |
047ea784 | 336 | #endif /* _ASM_POWERPC_PGTABLE_H */ |