projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mm: remove kern_addr_valid() completely
[linux-block.git] / arch / arm64 / include / asm / pgtable.h
CommitLineData
caab277b 1/* SPDX-License-Identifier: GPL-2.0-only */
4f04d8f0
CM		 2/*
3 * Copyright (C) 2012 ARM Ltd.
4f04d8f0
CM		 4 */
5#ifndef __ASM_PGTABLE_H
6#define __ASM_PGTABLE_H
7
2f4b829c 8#include <asm/bug.h>
4f04d8f0
CM		 9#include <asm/proc-fns.h>
10
11#include <asm/memory.h>
34bfeea4 12#include <asm/mte.h>
4f04d8f0 13#include <asm/pgtable-hwdef.h>
3eca86e7 14#include <asm/pgtable-prot.h>
3403e56b 15#include <asm/tlbflush.h>
4f04d8f0
CM		 16
17/*
3e1907d5 18 * VMALLOC range.
08375198 19 *
f9040773 20 * VMALLOC_START: beginning of the kernel vmalloc space
a5315819 21 * VMALLOC_END: extends to the available space below vmemmap, PCI I/O space
3e1907d5 22 * and fixed mappings
4f04d8f0 23 */
f9040773 24#define VMALLOC_START (MODULES_END)
9ad7c6d5 25#define VMALLOC_END (VMEMMAP_START - SZ_256M)
4f04d8f0 26
7bc1a0f9
AB		 27#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
28
4f04d8f0 29#ifndef __ASSEMBLY__
2f4b829c 30
3bbf7157 31#include <asm/cmpxchg.h>
961faac1 32#include <asm/fixmap.h>
2f4b829c 33#include <linux/mmdebug.h>
86c9e812
WD		 34#include <linux/mm_types.h>
35#include <linux/sched.h>
42b25471 36#include <linux/page_table_check.h>
2f4b829c 37
a7ac1cfa
ZY		 38#ifdef CONFIG_TRANSPARENT_HUGEPAGE
39#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
40
41/* Set stride and tlb_level in flush_*_tlb_range */
42#define flush_pmd_tlb_range(vma, addr, end) \
43 __flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
44#define flush_pud_tlb_range(vma, addr, end) \
45 __flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
46#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
47
d0637c50
BS		 48static inline bool arch_thp_swp_supported(void)
49{
50 return !system_supports_mte();
51}
52#define arch_thp_swp_supported arch_thp_swp_supported
53
6a1bdb17
WD		 54/*
55 * Outside of a few very special situations (e.g. hibernation), we always
56 * use broadcast TLB invalidation instructions, therefore a spurious page
57 * fault on one CPU which has been handled concurrently by another CPU
58 * does not need to perform additional invalidation.
59 */
60#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
61
4f04d8f0
CM		 62/*
63 * ZERO_PAGE is a global shared page that is always zero: used
64 * for zero-mapped memory areas etc..
65 */
5227cfa7 66extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
2077be67 67#define ZERO_PAGE(vaddr) phys_to_page(__pa_symbol(empty_zero_page))
4f04d8f0 68
2cf660eb
GS		 69#define pte_ERROR(e) \
70 pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
7078db46 71
75387b92
KM		 72/*
73 * Macros to convert between a physical address and its placement in a
74 * page table entry, taking care of 52-bit addresses.
75 */
76#ifdef CONFIG_ARM64_PA_BITS_52
c7c386fb
AB		 77static inline phys_addr_t __pte_to_phys(pte_t pte)
78{
79 return (pte_val(pte) & PTE_ADDR_LOW) |
80 ((pte_val(pte) & PTE_ADDR_HIGH) << 36);
81}
82static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
83{
84 return (phys | (phys >> 36)) & PTE_ADDR_MASK;
85}
75387b92
KM		 86#else
87#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
88#define __phys_to_pte_val(phys) (phys)
89#endif
4f04d8f0 90
75387b92
KM		 91#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT)
92#define pfn_pte(pfn,prot) \
93 __pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
4f04d8f0
CM		 94
95#define pte_none(pte) (!pte_val(pte))
96#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
97#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
7078db46 98
4f04d8f0
CM		 99/*
100 * The following only work if pte_present(). Undefined behaviour otherwise.
101 */
84fe6826 102#define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)))
84fe6826
SC		 103#define pte_young(pte) (!!(pte_val(pte) & PTE_AF))
104#define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL))
105#define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE))
42b25471 106#define pte_user(pte) (!!(pte_val(pte) & PTE_USER))
ec663d96 107#define pte_user_exec(pte) (!(pte_val(pte) & PTE_UXN))
93ef666a 108#define pte_cont(pte) (!!(pte_val(pte) & PTE_CONT))
73b20c84 109#define pte_devmap(pte) (!!(pte_val(pte) & PTE_DEVMAP))
34bfeea4
CM		 110#define pte_tagged(pte) ((pte_val(pte) & PTE_ATTRINDX_MASK) == \
111 PTE_ATTRINDX(MT_NORMAL_TAGGED))
4f04d8f0 112
d27cfa1f
AB		 113#define pte_cont_addr_end(addr, end) \
114({ unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK; \
115 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
116})
117
118#define pmd_cont_addr_end(addr, end) \
119({ unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK; \
120 (__boundary - 1 < (end) - 1) ? __boundary : (end); \
121})
122
b847415c 123#define pte_hw_dirty(pte) (pte_write(pte) && !(pte_val(pte) & PTE_RDONLY))
2f4b829c
CM		 124#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY))
125#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte))
126
766ffb69 127#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID))
18107f8a
VM		 128/*
129 * Execute-only user mappings do not have the PTE_USER bit set. All valid
130 * kernel mappings have the PTE_UXN bit set.
131 */
ec663d96 132#define pte_valid_not_user(pte) \
18107f8a 133 ((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
76c714be
WD		 134/*
135 * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
136 * so that we don't erroneously return false for pages that have been
137 * remapped as PROT_NONE but are yet to be flushed from the TLB.
07509e10
WD		 138 * Note that we can't make any assumptions based on the state of the access
139 * flag, since ptep_clear_flush_young() elides a DSB when invalidating the
140 * TLB.
76c714be
WD		 141 */
142#define pte_accessible(mm, pte) \
07509e10 143 (mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
4f04d8f0 144
6218f96c 145/*
18107f8a
VM		 146 * p??_access_permitted() is true for valid user mappings (PTE_USER
147 * bit set, subject to the write permission check). For execute-only
148 * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits
149 * not set) must return false. PROT_NONE mappings do not have the
150 * PTE_VALID bit set.
6218f96c
CM		 151 */
152#define pte_access_permitted(pte, write) \
18107f8a 153 (((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))
6218f96c
CM		 154#define pmd_access_permitted(pmd, write) \
155 (pte_access_permitted(pmd_pte(pmd), (write)))
156#define pud_access_permitted(pud, write) \
157 (pte_access_permitted(pud_pte(pud), (write)))
158
b6d4f280 159static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
44b6dfc5 160{
b6d4f280 161 pte_val(pte) &= ~pgprot_val(prot);
44b6dfc5
SC		 162 return pte;
163}
164
b6d4f280 165static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
44b6dfc5 166{
b6d4f280 167 pte_val(pte) |= pgprot_val(prot);
44b6dfc5
SC		 168 return pte;
169}
170
b65399f6
AK		 171static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
172{
173 pmd_val(pmd) &= ~pgprot_val(prot);
174 return pmd;
175}
176
177static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
178{
179 pmd_val(pmd) |= pgprot_val(prot);
180 return pmd;
181}
182
b6d4f280
LA		 183static inline pte_t pte_mkwrite(pte_t pte)
184{
73e86cb0
CM		 185 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
186 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
187 return pte;
b6d4f280
LA		 188}
189
44b6dfc5
SC		 190static inline pte_t pte_mkclean(pte_t pte)
191{
8781bcbc
SC		 192 pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
193 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
194
195 return pte;
44b6dfc5
SC		 196}
197
198static inline pte_t pte_mkdirty(pte_t pte)
199{
8781bcbc
SC		 200 pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
201
202 if (pte_write(pte))
203 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
204
205 return pte;
44b6dfc5
SC		 206}
207
ff1712f9
WD		 208static inline pte_t pte_wrprotect(pte_t pte)
209{
210 /*
211 * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
212 * clear), set the PTE_DIRTY bit.
213 */
214 if (pte_hw_dirty(pte))
215 pte = pte_mkdirty(pte);
216
217 pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
218 pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
219 return pte;
220}
221
44b6dfc5
SC		 222static inline pte_t pte_mkold(pte_t pte)
223{
b6d4f280 224 return clear_pte_bit(pte, __pgprot(PTE_AF));
44b6dfc5
SC		 225}
226
227static inline pte_t pte_mkyoung(pte_t pte)
228{
b6d4f280 229 return set_pte_bit(pte, __pgprot(PTE_AF));
44b6dfc5
SC		 230}
231
232static inline pte_t pte_mkspecial(pte_t pte)
233{
b6d4f280 234 return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
44b6dfc5 235}
4f04d8f0 236
93ef666a
JL		 237static inline pte_t pte_mkcont(pte_t pte)
238{
66b3923a
DW		 239 pte = set_pte_bit(pte, __pgprot(PTE_CONT));
240 return set_pte_bit(pte, __pgprot(PTE_TYPE_PAGE));
93ef666a
JL		 241}
242
243static inline pte_t pte_mknoncont(pte_t pte)
244{
245 return clear_pte_bit(pte, __pgprot(PTE_CONT));
246}
247
5ebe3a44
JM		 248static inline pte_t pte_mkpresent(pte_t pte)
249{
250 return set_pte_bit(pte, __pgprot(PTE_VALID));
251}
252
66b3923a
DW		 253static inline pmd_t pmd_mkcont(pmd_t pmd)
254{
255 return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
256}
257
73b20c84
RM		 258static inline pte_t pte_mkdevmap(pte_t pte)
259{
30e23538 260 return set_pte_bit(pte, __pgprot(PTE_DEVMAP | PTE_SPECIAL));
73b20c84
RM		 261}
262
4f04d8f0
CM		 263static inline void set_pte(pte_t *ptep, pte_t pte)
264{
20a004e7 265 WRITE_ONCE(*ptep, pte);
7f0b1bf0
CM		 266
267 /*
268 * Only if the new pte is valid and kernel, otherwise TLB maintenance
269 * or update_mmu_cache() have the necessary barriers.
270 */
d0b7a302 271 if (pte_valid_not_user(pte)) {
7f0b1bf0 272 dsb(ishst);
d0b7a302
WD		 273 isb();
274 }
4f04d8f0
CM		 275}
276
907e21c1 277extern void __sync_icache_dcache(pte_t pteval);
4f04d8f0 278
2f4b829c
CM		 279/*
280 * PTE bits configuration in the presence of hardware Dirty Bit Management
281 * (PTE_WRITE == PTE_DBM):
282 *
283 * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw)
284 * 0 0 | 1 0 0
285 * 0 1 | 1 1 0
286 * 1 0 | 1 0 1
287 * 1 1 | 0 1 x
288 *
289 * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via
290 * the page fault mechanism. Checking the dirty status of a pte becomes:
291 *
b847415c 292 * PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
2f4b829c 293 */
9b604722
MR		 294
295static inline void __check_racy_pte_update(struct mm_struct *mm, pte_t *ptep,
296 pte_t pte)
4f04d8f0 297{
20a004e7
WD		 298 pte_t old_pte;
299
9b604722
MR		 300 if (!IS_ENABLED(CONFIG_DEBUG_VM))
301 return;
302
303 old_pte = READ_ONCE(*ptep);
304
305 if (!pte_valid(old_pte) || !pte_valid(pte))
306 return;
307 if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
308 return;
02522463 309
2f4b829c 310 /*
9b604722
MR		 311 * Check for potential race with hardware updates of the pte
312 * (ptep_set_access_flags safely changes valid ptes without going
313 * through an invalid entry).
2f4b829c 314 */
9b604722
MR		 315 VM_WARN_ONCE(!pte_young(pte),
316 "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
317 __func__, pte_val(old_pte), pte_val(pte));
318 VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
319 "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
320 __func__, pte_val(old_pte), pte_val(pte));
321}
322
42b25471
KW		 323static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
324 pte_t *ptep, pte_t pte)
9b604722
MR		 325{
326 if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
327 __sync_icache_dcache(pte);
328
69e3b846
SP		 329 /*
330 * If the PTE would provide user space access to the tags associated
331 * with it then ensure that the MTE tags are synchronised. Although
332 * pte_access_permitted() returns false for exec only mappings, they
333 * don't expose tags (instruction fetches don't check tags).
334 */
335 if (system_supports_mte() && pte_access_permitted(pte, false) &&
336 !pte_special(pte)) {
337 pte_t old_pte = READ_ONCE(*ptep);
338 /*
339 * We only need to synchronise if the new PTE has tags enabled
340 * or if swapping in (in which case another mapping may have
341 * set tags in the past even if this PTE isn't tagged).
342 * (!pte_none() && !pte_present()) is an open coded version of
343 * is_swap_pte()
344 */
345 if (pte_tagged(pte) || (!pte_none(old_pte) && !pte_present(old_pte)))
346 mte_sync_tags(old_pte, pte);
347 }
34bfeea4 348
9b604722 349 __check_racy_pte_update(mm, ptep, pte);
2f4b829c 350
4f04d8f0
CM		 351 set_pte(ptep, pte);
352}
353
42b25471
KW		 354static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
355 pte_t *ptep, pte_t pte)
356{
357 page_table_check_pte_set(mm, addr, ptep, pte);
358 return __set_pte_at(mm, addr, ptep, pte);
359}
360
4f04d8f0
CM		 361/*
362 * Huge pte definitions.
363 */
084bd298
SC		 364#define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
365
366/*
367 * Hugetlb definitions.
368 */
66b3923a 369#define HUGE_MAX_HSTATE 4
084bd298
SC		 370#define HPAGE_SHIFT PMD_SHIFT
371#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
372#define HPAGE_MASK (~(HPAGE_SIZE - 1))
373#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
4f04d8f0 374
75387b92
KM		 375static inline pte_t pgd_pte(pgd_t pgd)
376{
377 return __pte(pgd_val(pgd));
378}
379
e9f63768
MR		 380static inline pte_t p4d_pte(p4d_t p4d)
381{
382 return __pte(p4d_val(p4d));
383}
384
29e56940
SC		 385static inline pte_t pud_pte(pud_t pud)
386{
387 return __pte(pud_val(pud));
388}
389
eb3f0624
PA		 390static inline pud_t pte_pud(pte_t pte)
391{
392 return __pud(pte_val(pte));
393}
394
29e56940
SC		 395static inline pmd_t pud_pmd(pud_t pud)
396{
397 return __pmd(pud_val(pud));
398}
399
9c7e535f
SC		 400static inline pte_t pmd_pte(pmd_t pmd)
401{
402 return __pte(pmd_val(pmd));
403}
af074848 404
9c7e535f
SC		 405static inline pmd_t pte_pmd(pte_t pte)
406{
407 return __pmd(pte_val(pte));
408}
af074848 409
f7f0097a 410static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
8ce837ce 411{
f7f0097a
AK		 412 return __pgprot((pgprot_val(prot) & ~PUD_TABLE_BIT) | PUD_TYPE_SECT);
413}
414
415static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
8ce837ce 416{
f7f0097a 417 return __pgprot((pgprot_val(prot) & ~PMD_TABLE_BIT) | PMD_TYPE_SECT);
8ce837ce
AB		 418}
419
570ef363
DH		 420#define __HAVE_ARCH_PTE_SWP_EXCLUSIVE
421static inline pte_t pte_swp_mkexclusive(pte_t pte)
422{
423 return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
424}
425
426static inline int pte_swp_exclusive(pte_t pte)
427{
428 return pte_val(pte) & PTE_SWP_EXCLUSIVE;
429}
430
431static inline pte_t pte_swp_clear_exclusive(pte_t pte)
432{
433 return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
434}
435
893dea9c
KW		 436/*
437 * Select all bits except the pfn
438 */
439static inline pgprot_t pte_pgprot(pte_t pte)
440{
441 unsigned long pfn = pte_pfn(pte);
442
443 return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
444}
445
56166230
GK		 446#ifdef CONFIG_NUMA_BALANCING
447/*
ca5999fd 448 * See the comment in include/linux/pgtable.h
56166230
GK		 449 */
450static inline int pte_protnone(pte_t pte)
451{
452 return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE;
453}
454
455static inline int pmd_protnone(pmd_t pmd)
456{
457 return pte_protnone(pmd_pte(pmd));
458}
459#endif
460
b65399f6
AK		 461#define pmd_present_invalid(pmd) (!!(pmd_val(pmd) & PMD_PRESENT_INVALID))
462
463static inline int pmd_present(pmd_t pmd)
464{
465 return pte_present(pmd_pte(pmd)) || pmd_present_invalid(pmd);
466}
467
af074848
SC		 468/*
469 * THP definitions.
470 */
af074848
SC		 471
472#ifdef CONFIG_TRANSPARENT_HUGEPAGE
b65399f6
AK		 473static inline int pmd_trans_huge(pmd_t pmd)
474{
475 return pmd_val(pmd) && pmd_present(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
476}
29e56940 477#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
af074848 478
c164e038 479#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
9c7e535f 480#define pmd_young(pmd) pte_young(pmd_pte(pmd))
0795edaf 481#define pmd_valid(pmd) pte_valid(pmd_pte(pmd))
42b25471
KW		 482#define pmd_user(pmd) pte_user(pmd_pte(pmd))
483#define pmd_user_exec(pmd) pte_user_exec(pmd_pte(pmd))
d55863db 484#define pmd_cont(pmd) pte_cont(pmd_pte(pmd))
9c7e535f 485#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
9c7e535f
SC		 486#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
487#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
ab4db1f2 488#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
9c7e535f
SC		 489#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
490#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
b65399f6
AK		 491
492static inline pmd_t pmd_mkinvalid(pmd_t pmd)
493{
494 pmd = set_pmd_bit(pmd, __pgprot(PMD_PRESENT_INVALID));
495 pmd = clear_pmd_bit(pmd, __pgprot(PMD_SECT_VALID));
496
497 return pmd;
498}
af074848 499
0dbd3b18
SP		 500#define pmd_thp_or_huge(pmd) (pmd_huge(pmd) || pmd_trans_huge(pmd))
501
9c7e535f 502#define pmd_write(pmd) pte_write(pmd_pte(pmd))
af074848
SC		 503
504#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
505
73b20c84
RM		 506#ifdef CONFIG_TRANSPARENT_HUGEPAGE
507#define pmd_devmap(pmd) pte_devmap(pmd_pte(pmd))
508#endif
30e23538
JH		 509static inline pmd_t pmd_mkdevmap(pmd_t pmd)
510{
511 return pte_pmd(set_pte_bit(pmd_pte(pmd), __pgprot(PTE_DEVMAP)));
512}
73b20c84 513
75387b92
KM		 514#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd))
515#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys)
516#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
517#define pfn_pmd(pfn,prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
af074848
SC		 518#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
519
35a63966 520#define pud_young(pud) pte_young(pud_pte(pud))
eb3f0624 521#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud)))
29e56940 522#define pud_write(pud) pte_write(pud_pte(pud))
75387b92 523
b8e0ba7c
PA		 524#define pud_mkhuge(pud) (__pud(pud_val(pud) & ~PUD_TABLE_BIT))
525
75387b92
KM		 526#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud))
527#define __phys_to_pud_val(phys) __phys_to_pte_val(phys)
528#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
529#define pfn_pud(pfn,prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
af074848 530
42b25471
KW		 531static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
532 pmd_t *pmdp, pmd_t pmd)
533{
534 page_table_check_pmd_set(mm, addr, pmdp, pmd);
535 return __set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd));
536}
537
538static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
539 pud_t *pudp, pud_t pud)
540{
541 page_table_check_pud_set(mm, addr, pudp, pud);
542 return __set_pte_at(mm, addr, (pte_t *)pudp, pud_pte(pud));
543}
af074848 544
e9f63768
MR		 545#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d))
546#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys)
547
75387b92
KM		 548#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd))
549#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys)
550
a501e324
CM		 551#define __pgprot_modify(prot,mask,bits) \
552 __pgprot((pgprot_val(prot) & ~(mask)) | (bits))
553
cca98e9f 554#define pgprot_nx(prot) \
034aa9cd 555 __pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN)
cca98e9f 556
4f04d8f0
CM		 557/*
558 * Mark the prot value as uncacheable and unbufferable.
559 */
560#define pgprot_noncached(prot) \
de2db743 561 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
4f04d8f0 562#define pgprot_writecombine(prot) \
de2db743 563 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
d1e6dc91
LD		 564#define pgprot_device(prot) \
565 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
d15dfd31
CM		 566#define pgprot_tagged(prot) \
567 __pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED))
568#define pgprot_mhp pgprot_tagged
3e4e1d3f
CH		 569/*
570 * DMA allocations for non-coherent devices use what the Arm architecture calls
571 * "Normal non-cacheable" memory, which permits speculation, unaligned accesses
572 * and merging of writes. This is different from "Device-nGnR[nE]" memory which
573 * is intended for MMIO and thus forbids speculation, preserves access size,
574 * requires strict alignment and can also force write responses to come from the
575 * endpoint.
576 */
419e2f18
CH		 577#define pgprot_dmacoherent(prot) \
578 __pgprot_modify(prot, PTE_ATTRINDX_MASK, \
579 PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
580
4f04d8f0
CM		 581#define __HAVE_PHYS_MEM_ACCESS_PROT
582struct file;
583extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
584 unsigned long size, pgprot_t vma_prot);
585
586#define pmd_none(pmd) (!pmd_val(pmd))
4f04d8f0 587
36311607
MZ		 588#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
589 PMD_TYPE_TABLE)
590#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
591 PMD_TYPE_SECT)
23bc8f69 592#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd))
e377ab82 593#define pmd_bad(pmd) (!pmd_table(pmd))
36311607 594
d55863db
PZ		 595#define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
596#define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
597
cac4b8cd 598#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
7d4e2dcf
QC		 599static inline bool pud_sect(pud_t pud) { return false; }
600static inline bool pud_table(pud_t pud) { return true; }
206a2a73
SC		 601#else
602#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
603 PUD_TYPE_SECT)
523d6e9f 604#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
605 PUD_TYPE_TABLE)
206a2a73 606#endif
36311607 607
2330b7ca
JY		 608extern pgd_t init_pg_dir[PTRS_PER_PGD];
609extern pgd_t init_pg_end[];
610extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
611extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
9d2d75ed 612extern pgd_t idmap_pg_end[];
2330b7ca 613extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
833be850 614extern pgd_t reserved_pg_dir[PTRS_PER_PGD];
2330b7ca
JY		 615
616extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
617
618static inline bool in_swapper_pgdir(void *addr)
619{
620 return ((unsigned long)addr & PAGE_MASK) ==
621 ((unsigned long)swapper_pg_dir & PAGE_MASK);
622}
623
4f04d8f0
CM		 624static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
625{
e9ed821b
JM		 626#ifdef __PAGETABLE_PMD_FOLDED
627 if (in_swapper_pgdir(pmdp)) {
2330b7ca
JY		 628 set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
629 return;
630 }
e9ed821b 631#endif /* __PAGETABLE_PMD_FOLDED */
2330b7ca 632
20a004e7 633 WRITE_ONCE(*pmdp, pmd);
0795edaf 634
d0b7a302 635 if (pmd_valid(pmd)) {
0795edaf 636 dsb(ishst);
d0b7a302
WD		 637 isb();
638 }
4f04d8f0
CM		 639}
640
641static inline void pmd_clear(pmd_t *pmdp)
642{
643 set_pmd(pmdp, __pmd(0));
644}
645
dca56dca 646static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
4f04d8f0 647{
75387b92 648 return __pmd_to_phys(pmd);
4f04d8f0
CM		 649}
650
974b9b2c
MR		 651static inline unsigned long pmd_page_vaddr(pmd_t pmd)
652{
653 return (unsigned long)__va(pmd_page_paddr(pmd));
654}
74dd022f 655
053520f7 656/* Find an entry in the third-level page table. */
f069faba 657#define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
053520f7 658
961faac1
MR		 659#define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr))
660#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr))
661#define pte_clear_fixmap() clear_fixmap(FIX_PTE)
662
68ecabd0 663#define pmd_page(pmd) phys_to_page(__pmd_to_phys(pmd))
4f04d8f0 664
6533945a
AB		 665/* use ONLY for statically allocated translation tables */
666#define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
667
4f04d8f0
CM		 668/*
669 * Conversion functions: convert a page and protection to a page entry,
670 * and a page entry and page directory to the page they refer to.
671 */
672#define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot)
673
9f25e6ad 674#if CONFIG_PGTABLE_LEVELS > 2
4f04d8f0 675
2cf660eb
GS		 676#define pmd_ERROR(e) \
677 pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
7078db46 678
4f04d8f0 679#define pud_none(pud) (!pud_val(pud))
e377ab82 680#define pud_bad(pud) (!pud_table(pud))
f02ab08a 681#define pud_present(pud) pte_present(pud_pte(pud))
23bc8f69 682#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud))
0795edaf 683#define pud_valid(pud) pte_valid(pud_pte(pud))
42b25471
KW		 684#define pud_user(pud) pte_user(pud_pte(pud))
685
4f04d8f0
CM		 686
687static inline void set_pud(pud_t *pudp, pud_t pud)
688{
e9ed821b
JM		 689#ifdef __PAGETABLE_PUD_FOLDED
690 if (in_swapper_pgdir(pudp)) {
2330b7ca
JY		 691 set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
692 return;
693 }
e9ed821b 694#endif /* __PAGETABLE_PUD_FOLDED */
2330b7ca 695
20a004e7 696 WRITE_ONCE(*pudp, pud);
0795edaf 697
d0b7a302 698 if (pud_valid(pud)) {
0795edaf 699 dsb(ishst);
d0b7a302
WD		 700 isb();
701 }
4f04d8f0
CM		 702}
703
704static inline void pud_clear(pud_t *pudp)
705{
706 set_pud(pudp, __pud(0));
707}
708
dca56dca 709static inline phys_addr_t pud_page_paddr(pud_t pud)
4f04d8f0 710{
75387b92 711 return __pud_to_phys(pud);
4f04d8f0
CM		 712}
713
9cf6fa24 714static inline pmd_t *pud_pgtable(pud_t pud)
974b9b2c 715{
9cf6fa24 716 return (pmd_t *)__va(pud_page_paddr(pud));
974b9b2c 717}
7078db46 718
974b9b2c 719/* Find an entry in the second-level page table. */
20a004e7 720#define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
7078db46 721
961faac1
MR		 722#define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
723#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr))
724#define pmd_clear_fixmap() clear_fixmap(FIX_PMD)
7078db46 725
68ecabd0 726#define pud_page(pud) phys_to_page(__pud_to_phys(pud))
29e56940 727
6533945a
AB		 728/* use ONLY for statically allocated translation tables */
729#define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
730
dca56dca
MR		 731#else
732
733#define pud_page_paddr(pud) ({ BUILD_BUG(); 0; })
734
961faac1
MR		 735/* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
736#define pmd_set_fixmap(addr) NULL
737#define pmd_set_fixmap_offset(pudp, addr) ((pmd_t *)pudp)
738#define pmd_clear_fixmap()
739
6533945a
AB		 740#define pmd_offset_kimg(dir,addr) ((pmd_t *)dir)
741
9f25e6ad 742#endif /* CONFIG_PGTABLE_LEVELS > 2 */
4f04d8f0 743
9f25e6ad 744#if CONFIG_PGTABLE_LEVELS > 3
c79b954b 745
2cf660eb
GS		 746#define pud_ERROR(e) \
747 pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
7078db46 748
e9f63768
MR		 749#define p4d_none(p4d) (!p4d_val(p4d))
750#define p4d_bad(p4d) (!(p4d_val(p4d) & 2))
751#define p4d_present(p4d) (p4d_val(p4d))
c79b954b 752
e9f63768 753static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
c79b954b 754{
e9f63768
MR		 755 if (in_swapper_pgdir(p4dp)) {
756 set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
2330b7ca
JY		 757 return;
758 }
759
e9f63768 760 WRITE_ONCE(*p4dp, p4d);
c79b954b 761 dsb(ishst);
eb6a4dcc 762 isb();
c79b954b
JL		 763}
764
e9f63768 765static inline void p4d_clear(p4d_t *p4dp)
c79b954b 766{
e9f63768 767 set_p4d(p4dp, __p4d(0));
c79b954b
JL		 768}
769
e9f63768 770static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
c79b954b 771{
e9f63768 772 return __p4d_to_phys(p4d);
c79b954b
JL		 773}
774
dc4875f0 775static inline pud_t *p4d_pgtable(p4d_t p4d)
974b9b2c 776{
dc4875f0 777 return (pud_t *)__va(p4d_page_paddr(p4d));
974b9b2c 778}
7078db46 779
5845e703 780/* Find an entry in the first-level page table. */
e9f63768 781#define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
7078db46 782
961faac1 783#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr))
e9f63768 784#define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr))
961faac1 785#define pud_clear_fixmap() clear_fixmap(FIX_PUD)
7078db46 786
e9f63768 787#define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
5d96e0cb 788
6533945a
AB		 789/* use ONLY for statically allocated translation tables */
790#define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
791
dca56dca
MR		 792#else
793
e9f63768 794#define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0;})
dca56dca
MR		 795#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0;})
796
961faac1
MR		 797/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
798#define pud_set_fixmap(addr) NULL
799#define pud_set_fixmap_offset(pgdp, addr) ((pud_t *)pgdp)
800#define pud_clear_fixmap()
801
6533945a
AB		 802#define pud_offset_kimg(dir,addr) ((pud_t *)dir)
803
9f25e6ad 804#endif /* CONFIG_PGTABLE_LEVELS > 3 */
c79b954b 805
2cf660eb
GS		 806#define pgd_ERROR(e) \
807 pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
7078db46 808
961faac1
MR		 809#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
810#define pgd_clear_fixmap() clear_fixmap(FIX_PGD)
811
4f04d8f0
CM		 812static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
813{
9f341931
CM		 814 /*
815 * Normal and Normal-Tagged are two different memory types and indices
816 * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
817 */
a6fadf7e 818 const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
9f341931
CM		 819 PTE_PROT_NONE | PTE_VALID | PTE_WRITE | PTE_GP |
820 PTE_ATTRINDX_MASK;
2f4b829c
CM		 821 /* preserve the hardware dirty information */
822 if (pte_hw_dirty(pte))
62d96c71 823 pte = pte_mkdirty(pte);
4f04d8f0
CM		 824 pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
825 return pte;
826}
827
9c7e535f
SC		 828static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
829{
830 return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
831}
832
66dbd6e6
CM		 833#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
834extern int ptep_set_access_flags(struct vm_area_struct *vma,
835 unsigned long address, pte_t *ptep,
836 pte_t entry, int dirty);
837
282aa705
CM		 838#ifdef CONFIG_TRANSPARENT_HUGEPAGE
839#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
840static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
841 unsigned long address, pmd_t *pmdp,
842 pmd_t entry, int dirty)
843{
844 return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
845}
73b20c84
RM		 846
847static inline int pud_devmap(pud_t pud)
848{
849 return 0;
850}
851
852static inline int pgd_devmap(pgd_t pgd)
853{
854 return 0;
855}
282aa705
CM		 856#endif
857
ed928a34
TT		 858#ifdef CONFIG_PAGE_TABLE_CHECK
859static inline bool pte_user_accessible_page(pte_t pte)
860{
861 return pte_present(pte) && (pte_user(pte) || pte_user_exec(pte));
862}
863
864static inline bool pmd_user_accessible_page(pmd_t pmd)
865{
866 return pmd_present(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
867}
868
869static inline bool pud_user_accessible_page(pud_t pud)
870{
871 return pud_present(pud) && pud_user(pud);
872}
873#endif
874
2f4b829c
CM		 875/*
876 * Atomic pte/pmd modifications.
877 */
878#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
06485053 879static inline int __ptep_test_and_clear_young(pte_t *ptep)
2f4b829c 880{
3bbf7157 881 pte_t old_pte, pte;
2f4b829c 882
3bbf7157
CM		 883 pte = READ_ONCE(*ptep);
884 do {
885 old_pte = pte;
886 pte = pte_mkold(pte);
887 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
888 pte_val(old_pte), pte_val(pte));
889 } while (pte_val(pte) != pte_val(old_pte));
2f4b829c 890
3bbf7157 891 return pte_young(pte);
2f4b829c
CM		 892}
893
06485053
CM		 894static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
895 unsigned long address,
896 pte_t *ptep)
897{
898 return __ptep_test_and_clear_young(ptep);
899}
900
3403e56b
AVB		 901#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
902static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
903 unsigned long address, pte_t *ptep)
904{
905 int young = ptep_test_and_clear_young(vma, address, ptep);
906
907 if (young) {
908 /*
909 * We can elide the trailing DSB here since the worst that can
910 * happen is that a CPU continues to use the young entry in its
911 * TLB and we mistakenly reclaim the associated page. The
912 * window for such an event is bounded by the next
913 * context-switch, which provides a DSB to complete the TLB
914 * invalidation.
915 */
916 flush_tlb_page_nosync(vma, address);
917 }
918
919 return young;
920}
921
2f4b829c
CM		 922#ifdef CONFIG_TRANSPARENT_HUGEPAGE
923#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
924static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
925 unsigned long address,
926 pmd_t *pmdp)
927{
928 return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
929}
930#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
931
932#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
933static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
934 unsigned long address, pte_t *ptep)
935{
42b25471
KW		 936 pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
937
938 page_table_check_pte_clear(mm, address, pte);
939
940 return pte;
2f4b829c
CM		 941}
942
943#ifdef CONFIG_TRANSPARENT_HUGEPAGE
911f56ee
CM		 944#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
945static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
946 unsigned long address, pmd_t *pmdp)
2f4b829c 947{
42b25471
KW		 948 pmd_t pmd = __pmd(xchg_relaxed(&pmd_val(*pmdp), 0));
949
950 page_table_check_pmd_clear(mm, address, pmd);
951
952 return pmd;
2f4b829c
CM		 953}
954#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
955
956/*
8781bcbc
SC		 957 * ptep_set_wrprotect - mark read-only while trasferring potential hardware
958 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
2f4b829c
CM		 959 */
960#define __HAVE_ARCH_PTEP_SET_WRPROTECT
961static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
962{
3bbf7157
CM		 963 pte_t old_pte, pte;
964
965 pte = READ_ONCE(*ptep);
966 do {
967 old_pte = pte;
3bbf7157
CM		 968 pte = pte_wrprotect(pte);
969 pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
970 pte_val(old_pte), pte_val(pte));
971 } while (pte_val(pte) != pte_val(old_pte));
2f4b829c
CM		 972}
973
974#ifdef CONFIG_TRANSPARENT_HUGEPAGE
975#define __HAVE_ARCH_PMDP_SET_WRPROTECT
976static inline void pmdp_set_wrprotect(struct mm_struct *mm,
977 unsigned long address, pmd_t *pmdp)
978{
979 ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
980}
1d78a62c
CM		 981
982#define pmdp_establish pmdp_establish
983static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
984 unsigned long address, pmd_t *pmdp, pmd_t pmd)
985{
42b25471 986 page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd);
1d78a62c
CM		 987 return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
988}
2f4b829c 989#endif
2f4b829c 990
4f04d8f0
CM		 991/*
992 * Encode and decode a swap entry:
3676f9ef 993 * bits 0-1: present (must be zero)
570ef363
DH		 994 * bits 2: remember PG_anon_exclusive
995 * bits 3-7: swap type
9b3e661e 996 * bits 8-57: swap offset
fdc69e7d 997 * bit 58: PTE_PROT_NONE (must be zero)
4f04d8f0 998 */
570ef363
DH		 999#define __SWP_TYPE_SHIFT 3
1000#define __SWP_TYPE_BITS 5
9b3e661e 1001#define __SWP_OFFSET_BITS 50
4f04d8f0
CM		 1002#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
1003#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
3676f9ef 1004#define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1)
4f04d8f0
CM		 1005
1006#define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
3676f9ef 1007#define __swp_offset(x) (((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
4f04d8f0
CM		 1008#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
1009
1010#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
1011#define __swp_entry_to_pte(swp) ((pte_t) { (swp).val })
1012
53fa117b
AK		 1013#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1014#define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) })
1015#define __swp_entry_to_pmd(swp) __pmd((swp).val)
1016#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
1017
4f04d8f0
CM		 1018/*
1019 * Ensure that there are not more swap files than can be encoded in the kernel
aad9061b 1020 * PTEs.
4f04d8f0
CM		 1021 */
1022#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
1023
36943aba
SP		 1024#ifdef CONFIG_ARM64_MTE
1025
1026#define __HAVE_ARCH_PREPARE_TO_SWAP
1027static inline int arch_prepare_to_swap(struct page *page)
1028{
1029 if (system_supports_mte())
1030 return mte_save_tags(page);
1031 return 0;
1032}
1033
1034#define __HAVE_ARCH_SWAP_INVALIDATE
1035static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
1036{
1037 if (system_supports_mte())
1038 mte_invalidate_tags(type, offset);
1039}
1040
1041static inline void arch_swap_invalidate_area(int type)
1042{
1043 if (system_supports_mte())
1044 mte_invalidate_tags_area(type);
1045}
1046
1047#define __HAVE_ARCH_SWAP_RESTORE
da08e9b7 1048static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio)
36943aba 1049{
da08e9b7
MWO		 1050 if (system_supports_mte() && mte_restore_tags(entry, &folio->page))
1051 set_bit(PG_mte_tagged, &folio->flags);
36943aba
SP		 1052}
1053
1054#endif /* CONFIG_ARM64_MTE */
1055
cba3574f
WD		 1056/*
1057 * On AArch64, the cache coherency is handled via the set_pte_at() function.
1058 */
1059static inline void update_mmu_cache(struct vm_area_struct *vma,
1060 unsigned long addr, pte_t *ptep)
1061{
1062 /*
120798d2
WD		 1063 * We don't do anything here, so there's a very small chance of
1064 * us retaking a user fault which we just fixed up. The alternative
1065 * is doing a dsb(ishst), but that penalises the fastpath.
cba3574f 1066 */
cba3574f
WD		 1067}
1068
1069#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
1070
529c4b05
KM		 1071#ifdef CONFIG_ARM64_PA_BITS_52
1072#define phys_to_ttbr(addr) (((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
1073#else
1074#define phys_to_ttbr(addr) (addr)
1075#endif
1076
6af31226
JH		 1077/*
1078 * On arm64 without hardware Access Flag, copying from user will fail because
1079 * the pte is old and cannot be marked young. So we always end up with zeroed
1080 * page after fork() + CoW for pfn mappings. We don't always have a
1081 * hardware-managed access flag on arm64.
1082 */
e1fd09e3 1083#define arch_has_hw_pte_young cpu_has_hw_af
0388f9c7
WD		 1084
1085/*
1086 * Experimentally, it's cheap to set the access flag in hardware and we
1087 * benefit from prefaulting mappings as 'old' to start with.
1088 */
e1fd09e3 1089#define arch_wants_old_prefaulted_pte cpu_has_hw_af
6af31226 1090
f8b46c4b
AK		 1091static inline bool pud_sect_supported(void)
1092{
1093 return PAGE_SIZE == SZ_4K;
1094}
1095
18107f8a 1096
4f04d8f0
CM		 1097#endif /* !__ASSEMBLY__ */
1098
1099#endif /* __ASM_PGTABLE_H */
Linux 6.x block layer and io_uring tree(s)