Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
17ed9e31 AK |
2 | #ifndef _ASM_POWERPC_NOHASH_64_PGTABLE_H |
3 | #define _ASM_POWERPC_NOHASH_64_PGTABLE_H | |
f88df14b DG |
4 | /* |
5 | * This file contains the functions and defines necessary to modify and use | |
0c295d0e | 6 | * the ppc64 non-hashed page table. |
f88df14b DG |
7 | */ |
8 | ||
9ccba66d CL |
9 | #include <linux/sizes.h> |
10 | ||
17ed9e31 | 11 | #include <asm/nohash/64/pgtable-4k.h> |
074c2eae | 12 | #include <asm/barrier.h> |
ec0c464c | 13 | #include <asm/asm-const.h> |
f88df14b | 14 | |
f88df14b DG |
15 | /* |
16 | * Size of EA range mapped by our pagetables. | |
17 | */ | |
18 | #define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \ | |
17ed9e31 | 19 | PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT) |
3d5134ee | 20 | #define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE) |
f88df14b | 21 | |
f940f528 | 22 | #define PMD_CACHE_INDEX PMD_INDEX_SIZE |
fae22116 | 23 | #define PUD_CACHE_INDEX PUD_INDEX_SIZE |
fda0440d | 24 | |
f88df14b | 25 | /* |
57e2a99f | 26 | * Define the address range of the kernel non-linear virtual area |
f88df14b | 27 | */ |
128c1ea2 | 28 | #define KERN_VIRT_START ASM_CONST(0xc000100000000000) |
67550080 | 29 | #define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000) |
f88df14b DG |
30 | |
31 | /* | |
57e2a99f | 32 | * The vmalloc space starts at the beginning of that region, and |
0c295d0e | 33 | * occupies a quarter of it on Book3E |
32a74949 | 34 | * (we keep a quarter for the virtual memmap) |
57e2a99f BH |
35 | */ |
36 | #define VMALLOC_START KERN_VIRT_START | |
57e2a99f | 37 | #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 2) |
57e2a99f BH |
38 | #define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE) |
39 | ||
40 | /* | |
059c1893 | 41 | * The third quarter of the kernel virtual space is used for IO mappings, |
57e2a99f BH |
42 | * it's itself carved into the PIO region (ISA and PHB IO space) and |
43 | * the ioremap space | |
3d5134ee | 44 | * |
57e2a99f | 45 | * ISA_IO_BASE = KERN_IO_START, 64K reserved area |
3d5134ee | 46 | * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces |
059c1893 | 47 | * IOREMAP_BASE = ISA_IO_BASE + 2G to KERN_IO_START + KERN_IO_SIZE |
f88df14b | 48 | */ |
57e2a99f | 49 | #define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1)) |
059c1893 | 50 | #define KERN_IO_SIZE (KERN_VIRT_SIZE >> 2) |
3d5134ee | 51 | #define FULL_IO_SIZE 0x80000000ul |
57e2a99f BH |
52 | #define ISA_IO_BASE (KERN_IO_START) |
53 | #define ISA_IO_END (KERN_IO_START + 0x10000ul) | |
3d5134ee | 54 | #define PHB_IO_BASE (ISA_IO_END) |
57e2a99f | 55 | #define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE) |
3d5134ee | 56 | #define IOREMAP_BASE (PHB_IO_END) |
4a45b746 | 57 | #define IOREMAP_START (ioremap_bot) |
059c1893 | 58 | #define IOREMAP_END (KERN_IO_START + KERN_IO_SIZE - FIXADDR_SIZE) |
9ccba66d | 59 | #define FIXADDR_SIZE SZ_32M |
57e2a99f | 60 | |
d29eff7b | 61 | /* |
57e2a99f | 62 | * Defines the address of the vmemap area, in its own region on |
0c295d0e | 63 | * after the vmalloc space on Book3E |
d29eff7b | 64 | */ |
57e2a99f BH |
65 | #define VMEMMAP_BASE VMALLOC_END |
66 | #define VMEMMAP_END KERN_IO_START | |
cec08e7a BH |
67 | #define vmemmap ((struct page *)VMEMMAP_BASE) |
68 | ||
d29eff7b | 69 | |
f88df14b | 70 | /* |
c605782b | 71 | * Include the PTE bits definitions |
f88df14b | 72 | */ |
3e731858 | 73 | #include <asm/nohash/pte-e500.h> |
6c5d2d3f | 74 | |
6c5d2d3f CL |
75 | #define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) |
76 | ||
77 | /* | |
78 | * _PAGE_CHG_MASK masks of bits that are to be preserved across | |
79 | * pgprot changes. | |
80 | */ | |
81 | #define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SPECIAL) | |
82 | ||
83 | #define H_PAGE_4K_PFN 0 | |
c605782b | 84 | |
f88df14b | 85 | #ifndef __ASSEMBLY__ |
f88df14b DG |
86 | /* pte_clear moved to later in this file */ |
87 | ||
aa9cd505 CL |
88 | static inline pte_t pte_mkwrite(pte_t pte) |
89 | { | |
90 | return __pte(pte_val(pte) | _PAGE_RW); | |
91 | } | |
92 | ||
93 | static inline pte_t pte_mkdirty(pte_t pte) | |
94 | { | |
95 | return __pte(pte_val(pte) | _PAGE_DIRTY); | |
96 | } | |
97 | ||
98 | static inline pte_t pte_mkyoung(pte_t pte) | |
99 | { | |
100 | return __pte(pte_val(pte) | _PAGE_ACCESSED); | |
101 | } | |
102 | ||
103 | static inline pte_t pte_wrprotect(pte_t pte) | |
104 | { | |
105 | return __pte(pte_val(pte) & ~_PAGE_RW); | |
106 | } | |
107 | ||
f88df14b DG |
108 | #define PMD_BAD_BITS (PTE_TABLE_SIZE-1) |
109 | #define PUD_BAD_BITS (PMD_TABLE_SIZE-1) | |
110 | ||
f281b5d5 AK |
111 | static inline void pmd_set(pmd_t *pmdp, unsigned long val) |
112 | { | |
113 | *pmdp = __pmd(val); | |
114 | } | |
115 | ||
116 | static inline void pmd_clear(pmd_t *pmdp) | |
117 | { | |
118 | *pmdp = __pmd(0); | |
119 | } | |
120 | ||
e34aa03c AK |
121 | static inline pte_t pmd_pte(pmd_t pmd) |
122 | { | |
123 | return __pte(pmd_val(pmd)); | |
124 | } | |
125 | ||
f88df14b DG |
126 | #define pmd_none(pmd) (!pmd_val(pmd)) |
127 | #define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \ | |
128 | || (pmd_val(pmd) & PMD_BAD_BITS)) | |
06743521 | 129 | #define pmd_present(pmd) (!pmd_none(pmd)) |
f88df14b | 130 | #define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) |
074c2eae | 131 | extern struct page *pmd_page(pmd_t pmd); |
9e996c21 | 132 | #define pmd_pfn(pmd) (page_to_pfn(pmd_page(pmd))) |
f88df14b | 133 | |
f281b5d5 AK |
134 | static inline void pud_set(pud_t *pudp, unsigned long val) |
135 | { | |
136 | *pudp = __pud(val); | |
137 | } | |
138 | ||
139 | static inline void pud_clear(pud_t *pudp) | |
140 | { | |
141 | *pudp = __pud(0); | |
142 | } | |
143 | ||
f88df14b DG |
144 | #define pud_none(pud) (!pud_val(pud)) |
145 | #define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \ | |
146 | || (pud_val(pud) & PUD_BAD_BITS)) | |
147 | #define pud_present(pud) (pud_val(pud) != 0) | |
9cf6fa24 AK |
148 | |
149 | static inline pmd_t *pud_pgtable(pud_t pud) | |
150 | { | |
151 | return (pmd_t *)(pud_val(pud) & ~PUD_MASKED_BITS); | |
152 | } | |
f88df14b | 153 | |
06743521 AK |
154 | extern struct page *pud_page(pud_t pud); |
155 | ||
156 | static inline pte_t pud_pte(pud_t pud) | |
157 | { | |
158 | return __pte(pud_val(pud)); | |
159 | } | |
160 | ||
161 | static inline pud_t pte_pud(pte_t pte) | |
162 | { | |
163 | return __pud(pte_val(pte)); | |
164 | } | |
165 | #define pud_write(pud) pte_write(pud_pte(pud)) | |
2fb47060 | 166 | #define p4d_write(pgd) pte_write(p4d_pte(p4d)) |
f88df14b | 167 | |
2fb47060 | 168 | static inline void p4d_set(p4d_t *p4dp, unsigned long val) |
f281b5d5 | 169 | { |
2fb47060 | 170 | *p4dp = __p4d(val); |
f281b5d5 AK |
171 | } |
172 | ||
f88df14b DG |
173 | /* Atomic PTE updates */ |
174 | static inline unsigned long pte_update(struct mm_struct *mm, | |
175 | unsigned long addr, | |
176 | pte_t *ptep, unsigned long clr, | |
88247e8d | 177 | unsigned long set, |
f88df14b DG |
178 | int huge) |
179 | { | |
a033a487 | 180 | unsigned long old = pte_val(*ptep); |
88247e8d | 181 | *ptep = __pte((old & ~clr) | set); |
fadaac67 | 182 | |
8d30c14c BH |
183 | /* huge pages use the old page table lock */ |
184 | if (!huge) | |
185 | assert_pte_locked(mm, addr); | |
186 | ||
f88df14b DG |
187 | return old; |
188 | } | |
189 | ||
45201c87 CL |
190 | static inline int pte_young(pte_t pte) |
191 | { | |
192 | return pte_val(pte) & _PAGE_ACCESSED; | |
193 | } | |
194 | ||
f88df14b DG |
195 | static inline int __ptep_test_and_clear_young(struct mm_struct *mm, |
196 | unsigned long addr, pte_t *ptep) | |
197 | { | |
198 | unsigned long old; | |
199 | ||
45201c87 | 200 | if (pte_young(*ptep)) |
f88df14b | 201 | return 0; |
88247e8d | 202 | old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); |
f88df14b DG |
203 | return (old & _PAGE_ACCESSED) != 0; |
204 | } | |
205 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG | |
206 | #define ptep_test_and_clear_young(__vma, __addr, __ptep) \ | |
207 | ({ \ | |
208 | int __r; \ | |
209 | __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ | |
210 | __r; \ | |
211 | }) | |
212 | ||
f88df14b DG |
213 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
214 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, | |
215 | pte_t *ptep) | |
216 | { | |
f88df14b | 217 | |
2a2c29c1 SP |
218 | if ((pte_val(*ptep) & _PAGE_RW) == 0) |
219 | return; | |
220 | ||
88247e8d | 221 | pte_update(mm, addr, ptep, _PAGE_RW, 0, 0); |
f88df14b DG |
222 | } |
223 | ||
8e581d43 | 224 | #define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT |
016b33c4 AW |
225 | static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, |
226 | unsigned long addr, pte_t *ptep) | |
227 | { | |
86df8642 DG |
228 | if ((pte_val(*ptep) & _PAGE_RW) == 0) |
229 | return; | |
2a2c29c1 | 230 | |
88247e8d | 231 | pte_update(mm, addr, ptep, _PAGE_RW, 0, 1); |
016b33c4 | 232 | } |
f88df14b | 233 | |
f88df14b DG |
234 | #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH |
235 | #define ptep_clear_flush_young(__vma, __address, __ptep) \ | |
236 | ({ \ | |
237 | int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \ | |
238 | __ptep); \ | |
239 | __young; \ | |
240 | }) | |
241 | ||
f88df14b DG |
242 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR |
243 | static inline pte_t ptep_get_and_clear(struct mm_struct *mm, | |
244 | unsigned long addr, pte_t *ptep) | |
245 | { | |
88247e8d | 246 | unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0); |
f88df14b DG |
247 | return __pte(old); |
248 | } | |
249 | ||
250 | static inline void pte_clear(struct mm_struct *mm, unsigned long addr, | |
251 | pte_t * ptep) | |
252 | { | |
88247e8d | 253 | pte_update(mm, addr, ptep, ~0UL, 0, 0); |
f88df14b DG |
254 | } |
255 | ||
f88df14b | 256 | |
0c295d0e | 257 | /* Set the dirty and/or accessed bits atomically in a linux PTE */ |
e4c1112c | 258 | static inline void __ptep_set_access_flags(struct vm_area_struct *vma, |
b3603e17 | 259 | pte_t *ptep, pte_t entry, |
e4c1112c AK |
260 | unsigned long address, |
261 | int psize) | |
f88df14b DG |
262 | { |
263 | unsigned long bits = pte_val(entry) & | |
ea3cc330 | 264 | (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); |
a033a487 | 265 | |
a033a487 BH |
266 | unsigned long old = pte_val(*ptep); |
267 | *ptep = __pte(old | bits); | |
bd5050e3 AK |
268 | |
269 | flush_tlb_page(vma, address); | |
f88df14b | 270 | } |
f88df14b | 271 | |
f88df14b | 272 | #define pte_ERROR(e) \ |
a7696b36 | 273 | pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) |
f88df14b | 274 | #define pmd_ERROR(e) \ |
a7696b36 | 275 | pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) |
f88df14b | 276 | #define pgd_ERROR(e) \ |
a7696b36 | 277 | pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) |
f88df14b | 278 | |
2bba2ffb DH |
279 | /* |
280 | * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that | |
281 | * are !pte_none() && !pte_present(). | |
282 | * | |
283 | * Format of swap PTEs: | |
284 | * | |
285 | * 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 3 3 | |
286 | * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 | |
287 | * <-------------------------- offset ---------------------------- | |
288 | * | |
289 | * 3 3 3 3 3 3 3 3 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 6 6 6 6 | |
290 | * 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 | |
291 | * --------------> <----------- zero ------------> E < type -> 0 0 | |
292 | * | |
293 | * E is the exclusive marker that is not stored in swap entries. | |
294 | */ | |
e1483158 AK |
295 | #define MAX_SWAPFILES_CHECK() do { \ |
296 | BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \ | |
e1483158 | 297 | } while (0) |
3159f943 | 298 | |
e1483158 | 299 | #define SWP_TYPE_BITS 5 |
2bba2ffb | 300 | #define __swp_type(x) (((x).val >> 2) \ |
e1483158 AK |
301 | & ((1UL << SWP_TYPE_BITS) - 1)) |
302 | #define __swp_offset(x) ((x).val >> PTE_RPN_SHIFT) | |
303 | #define __swp_entry(type, offset) ((swp_entry_t) { \ | |
2bba2ffb | 304 | (((type) & 0x1f) << 2) \ |
e1483158 AK |
305 | | ((offset) << PTE_RPN_SHIFT) }) |
306 | ||
307 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) | |
308 | #define __swp_entry_to_pte(x) __pte((x).val) | |
f88df14b | 309 | |
2bba2ffb DH |
310 | /* We borrow MSB 56 (LSB 7) to store the exclusive marker in swap PTEs. */ |
311 | #define _PAGE_SWP_EXCLUSIVE 0x80 | |
312 | ||
c766ee72 | 313 | int map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot); |
aec98260 | 314 | void unmap_kernel_page(unsigned long va); |
31a14fae AK |
315 | extern int __meminit vmemmap_create_mapping(unsigned long start, |
316 | unsigned long page_size, | |
317 | unsigned long phys); | |
318 | extern void vmemmap_remove_mapping(unsigned long start, | |
319 | unsigned long page_size); | |
29562a9d CL |
320 | void __patch_exception(int exc, unsigned long addr); |
321 | #define patch_exception(exc, name) do { \ | |
322 | extern unsigned int name; \ | |
323 | __patch_exception((exc), (unsigned long)&name); \ | |
324 | } while (0) | |
325 | ||
f88df14b DG |
326 | #endif /* __ASSEMBLY__ */ |
327 | ||
17ed9e31 | 328 | #endif /* _ASM_POWERPC_NOHASH_64_PGTABLE_H */ |