projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/mm/radix: Update pte update sequence for pte clear case
[linux-2.6-block.git] / arch / powerpc / include / asm / book3s / 64 / pgtable.h
CommitLineData
3dfcb315
AK		 1#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
2#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
2e873519
AK		 3
4/*
5 * Common bits between hash and Radix page table
6 */
7#define _PAGE_BIT_SWAP_TYPE 0
8
6b8cb66a
CL		 9#define _PAGE_RO 0
10
2e873519
AK		 11#define _PAGE_EXEC 0x00001 /* execute permission */
12#define _PAGE_WRITE 0x00002 /* write access allowed */
13#define _PAGE_READ 0x00004 /* read access allowed */
14#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE)
15#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)
16#define _PAGE_PRIVILEGED 0x00008 /* kernel access only */
17#define _PAGE_SAO 0x00010 /* Strong access order */
18#define _PAGE_NON_IDEMPOTENT 0x00020 /* non idempotent memory */
19#define _PAGE_TOLERANT 0x00030 /* tolerant memory, cache inhibited */
20#define _PAGE_DIRTY 0x00080 /* C: page changed */
21#define _PAGE_ACCESSED 0x00100 /* R: page referenced */
3dfcb315 22/*
2e873519 23 * Software bits
3dfcb315 24 */
69dfbaeb
AK		 25#define _RPAGE_SW0 0x2000000000000000UL
26#define _RPAGE_SW1 0x00800
27#define _RPAGE_SW2 0x00400
28#define _RPAGE_SW3 0x00200
049d567a
AK		 29#define _RPAGE_RSV1 0x1000000000000000UL
30#define _RPAGE_RSV2 0x0800000000000000UL
31#define _RPAGE_RSV3 0x0400000000000000UL
32#define _RPAGE_RSV4 0x0200000000000000UL
33
2e873519 34#ifdef CONFIG_MEM_SOFT_DIRTY
69dfbaeb 35#define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */
2e873519
AK		 36#else
37#define _PAGE_SOFT_DIRTY 0x00000
38#endif
69dfbaeb 39#define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */
2e873519 40
049d567a
AK		 41/*
42 * For P9 DD1 only, we need to track whether the pte's huge.
43 */
44#define _PAGE_LARGE _RPAGE_RSV1
45
2e873519
AK		 46
47#define _PAGE_PTE (1ul << 62) /* distinguishes PTEs from pointers */
48#define _PAGE_PRESENT (1ul << 63) /* pte contains a translation */
49/*
50 * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE
51 * Instead of fixing all of them, add an alternate define which
52 * maps CI pte mapping.
53 */
54#define _PAGE_NO_CACHE _PAGE_TOLERANT
55/*
56 * We support 57 bit real address in pte. Clear everything above 57, and
57 * every thing below PAGE_SHIFT;
58 */
59#define PTE_RPN_MASK (((1UL << 57) - 1) & (PAGE_MASK))
60/*
61 * set of bits not changed in pmd_modify. Even though we have hash specific bits
62 * in here, on radix we expect them to be zero.
63 */
64#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
65 _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \
66 _PAGE_SOFT_DIRTY)
67/*
68 * user access blocked by key
69 */
70#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY)
71#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ)
72#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \
73 _PAGE_RW | _PAGE_EXEC)
74/*
75 * No page size encoding in the linux PTE
76 */
77#define _PAGE_PSIZE 0
78/*
79 * _PAGE_CHG_MASK masks of bits that are to be preserved across
80 * pgprot changes
81 */
82#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
83 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
84 _PAGE_SOFT_DIRTY)
85/*
86 * Mask of bits returned by pte_pgprot()
87 */
88#define PAGE_PROT_BITS (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT | \
89 H_PAGE_4K_PFN | _PAGE_PRIVILEGED | _PAGE_ACCESSED | \
90 _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_EXEC | \
91 _PAGE_SOFT_DIRTY)
3dfcb315 92/*
2e873519
AK		 93 * We define 2 sets of base prot bits, one for basic pages (ie,
94 * cacheable kernel and user pages) and one for non cacheable
95 * pages. We always set _PAGE_COHERENT when SMP is enabled or
96 * the processor might need it for DMA coherency.
3dfcb315 97 */
2e873519
AK		 98#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE)
99#define _PAGE_BASE (_PAGE_BASE_NC)
100
101/* Permission masks used to generate the __P and __S table,
102 *
103 * Note:__pgprot is defined in arch/powerpc/include/asm/page.h
104 *
105 * Write permissions imply read permissions for now (we could make write-only
106 * pages on BookE but we don't bother for now). Execute permission control is
107 * possible on platforms that define _PAGE_EXEC
108 *
109 * Note due to the way vm flags are laid out, the bits are XWR
110 */
111#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED)
112#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW)
113#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC)
114#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ)
115#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
116#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ)
117#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
118
119#define __P000 PAGE_NONE
120#define __P001 PAGE_READONLY
121#define __P010 PAGE_COPY
122#define __P011 PAGE_COPY
123#define __P100 PAGE_READONLY_X
124#define __P101 PAGE_READONLY_X
125#define __P110 PAGE_COPY_X
126#define __P111 PAGE_COPY_X
127
128#define __S000 PAGE_NONE
129#define __S001 PAGE_READONLY
130#define __S010 PAGE_SHARED
131#define __S011 PAGE_SHARED
132#define __S100 PAGE_READONLY_X
133#define __S101 PAGE_READONLY_X
134#define __S110 PAGE_SHARED_X
135#define __S111 PAGE_SHARED_X
136
137/* Permission masks used for kernel mappings */
138#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW)
139#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
140 _PAGE_TOLERANT)
141#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
142 _PAGE_NON_IDEMPOTENT)
143#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX)
144#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO)
145#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)
146
147/*
148 * Protection used for kernel text. We want the debuggers to be able to
149 * set breakpoints anywhere, so don't write protect the kernel text
150 * on platforms where such control is possible.
151 */
152#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \
153 defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE)
154#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
155#else
156#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
157#endif
158
159/* Make modules code happy. We don't set RO yet */
160#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
161#define PAGE_AGP (PAGE_KERNEL_NC)
3dfcb315 162
dd1842a2
AK		 163#ifndef __ASSEMBLY__
164/*
165 * page table defines
166 */
167extern unsigned long __pte_index_size;
168extern unsigned long __pmd_index_size;
169extern unsigned long __pud_index_size;
170extern unsigned long __pgd_index_size;
171extern unsigned long __pmd_cache_index;
172#define PTE_INDEX_SIZE __pte_index_size
173#define PMD_INDEX_SIZE __pmd_index_size
174#define PUD_INDEX_SIZE __pud_index_size
175#define PGD_INDEX_SIZE __pgd_index_size
176#define PMD_CACHE_INDEX __pmd_cache_index
177/*
178 * Because of use of pte fragments and THP, size of page table
179 * are not always derived out of index size above.
180 */
181extern unsigned long __pte_table_size;
182extern unsigned long __pmd_table_size;
183extern unsigned long __pud_table_size;
184extern unsigned long __pgd_table_size;
185#define PTE_TABLE_SIZE __pte_table_size
186#define PMD_TABLE_SIZE __pmd_table_size
187#define PUD_TABLE_SIZE __pud_table_size
188#define PGD_TABLE_SIZE __pgd_table_size
a2f41eb9
AK		 189
190extern unsigned long __pmd_val_bits;
191extern unsigned long __pud_val_bits;
192extern unsigned long __pgd_val_bits;
193#define PMD_VAL_BITS __pmd_val_bits
194#define PUD_VAL_BITS __pud_val_bits
195#define PGD_VAL_BITS __pgd_val_bits
5ed7ecd0
AK		 196
197extern unsigned long __pte_frag_nr;
198#define PTE_FRAG_NR __pte_frag_nr
199extern unsigned long __pte_frag_size_shift;
200#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
201#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
dd1842a2
AK		 202/*
203 * Pgtable size used by swapper, init in asm code
dd1842a2 204 */
a2f41eb9 205#define MAX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE)
dd1842a2
AK		 206
207#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
208#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
209#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
210#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
211
212/* PMD_SHIFT determines what a second-level page table entry can map */
213#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
214#define PMD_SIZE (1UL << PMD_SHIFT)
215#define PMD_MASK (~(PMD_SIZE-1))
216
217/* PUD_SHIFT determines what a third-level page table entry can map */
218#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
219#define PUD_SIZE (1UL << PUD_SHIFT)
220#define PUD_MASK (~(PUD_SIZE-1))
221
222/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
223#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
224#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
225#define PGDIR_MASK (~(PGDIR_SIZE-1))
226
227/* Bits to mask out from a PMD to get to the PTE page */
228#define PMD_MASKED_BITS 0xc0000000000000ffUL
229/* Bits to mask out from a PUD to get to the PMD page */
230#define PUD_MASKED_BITS 0xc0000000000000ffUL
231/* Bits to mask out from a PGD to get to the PUD page */
232#define PGD_MASKED_BITS 0xc0000000000000ffUL
d6a9996e
AK		 233
234extern unsigned long __vmalloc_start;
235extern unsigned long __vmalloc_end;
236#define VMALLOC_START __vmalloc_start
237#define VMALLOC_END __vmalloc_end
238
239extern unsigned long __kernel_virt_start;
240extern unsigned long __kernel_virt_size;
241#define KERN_VIRT_START __kernel_virt_start
242#define KERN_VIRT_SIZE __kernel_virt_size
243extern struct page *vmemmap;
244extern unsigned long ioremap_bot;
bfa37087 245extern unsigned long pci_io_base;
dd1842a2 246#endif /* __ASSEMBLY__ */
3dfcb315 247
ab537dca 248#include <asm/book3s/64/hash.h>
b0b5e9b1 249#include <asm/book3s/64/radix.h>
3dfcb315 250
a9252aae
AK		 251#ifdef CONFIG_PPC_64K_PAGES
252#include <asm/book3s/64/pgtable-64k.h>
253#else
254#include <asm/book3s/64/pgtable-4k.h>
255#endif
256
3dfcb315 257#include <asm/barrier.h>
3dfcb315
AK		 258/*
259 * The second half of the kernel virtual space is used for IO mappings,
260 * it's itself carved into the PIO region (ISA and PHB IO space) and
261 * the ioremap space
262 *
263 * ISA_IO_BASE = KERN_IO_START, 64K reserved area
264 * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces
265 * IOREMAP_BASE = ISA_IO_BASE + 2G to VMALLOC_START + PGTABLE_RANGE
266 */
267#define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1))
268#define FULL_IO_SIZE 0x80000000ul
269#define ISA_IO_BASE (KERN_IO_START)
270#define ISA_IO_END (KERN_IO_START + 0x10000ul)
271#define PHB_IO_BASE (ISA_IO_END)
272#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
273#define IOREMAP_BASE (PHB_IO_END)
274#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE)
275
b0412ea9 276/* Advertise special mapping type for AGP */
b0412ea9
AK		 277#define HAVE_PAGE_AGP
278
279/* Advertise support for _PAGE_SPECIAL */
280#define __HAVE_ARCH_PTE_SPECIAL
281
3dfcb315
AK		 282#ifndef __ASSEMBLY__
283
284/*
285 * This is the default implementation of various PTE accessors, it's
286 * used in all cases except Book3S with 64K pages where we have a
287 * concept of sub-pages
288 */
289#ifndef __real_pte
290
3dfcb315
AK		 291#define __real_pte(e,p) ((real_pte_t){(e)})
292#define __rpte_to_pte(r) ((r).pte)
945537df 293#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
3dfcb315
AK		 294
295#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
296 do { \
297 index = 0; \
298 shift = mmu_psize_defs[psize].shift; \
299
300#define pte_iterate_hashed_end() } while(0)
301
302/*
303 * We expect this to be called only for user addresses or kernel virtual
304 * addresses other than the linear mapping.
305 */
306#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
307
308#endif /* __real_pte */
309
ac94ac79
AK		 310static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr,
311 pte_t *ptep, unsigned long clr,
312 unsigned long set, int huge)
313{
314 if (radix_enabled())
315 return radix__pte_update(mm, addr, ptep, clr, set, huge);
316 return hash__pte_update(mm, addr, ptep, clr, set, huge);
317}
13f829a5
AK		 318/*
319 * For hash even if we have _PAGE_ACCESSED = 0, we do a pte_update.
320 * We currently remove entries from the hashtable regardless of whether
321 * the entry was young or dirty.
322 *
323 * We should be more intelligent about this but for the moment we override
324 * these functions and force a tlb flush unconditionally
325 * For radix: H_PAGE_HASHPTE should be zero. Hence we can use the same
326 * function for both hash and radix.
327 */
328static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
329 unsigned long addr, pte_t *ptep)
330{
331 unsigned long old;
332
66c570f5 333 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
13f829a5
AK		 334 return 0;
335 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
336 return (old & _PAGE_ACCESSED) != 0;
337}
338
339#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
340#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
341({ \
342 int __r; \
343 __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
344 __r; \
345})
346
347#define __HAVE_ARCH_PTEP_SET_WRPROTECT
348static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
349 pte_t *ptep)
350{
66c570f5 351 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
13f829a5
AK		 352 return;
353
354 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
355}
356
357static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
358 unsigned long addr, pte_t *ptep)
359{
66c570f5 360 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0)
13f829a5
AK		 361 return;
362
363 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
364}
365
366#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
367static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
368 unsigned long addr, pte_t *ptep)
369{
370 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
371 return __pte(old);
372}
373
374static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
375 pte_t * ptep)
376{
377 pte_update(mm, addr, ptep, ~0UL, 0, 0);
378}
66c570f5
AK		 379
380static inline int pte_write(pte_t pte)
381{
382 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
383}
384
385static inline int pte_dirty(pte_t pte)
386{
387 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
388}
389
390static inline int pte_young(pte_t pte)
391{
392 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));
393}
394
395static inline int pte_special(pte_t pte)
396{
397 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));
398}
399
13f829a5
AK		 400static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
401
402#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
403static inline bool pte_soft_dirty(pte_t pte)
404{
66c570f5 405 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));
13f829a5 406}
66c570f5 407
13f829a5
AK		 408static inline pte_t pte_mksoft_dirty(pte_t pte)
409{
410 return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
411}
412
413static inline pte_t pte_clear_soft_dirty(pte_t pte)
414{
415 return __pte(pte_val(pte) & ~_PAGE_SOFT_DIRTY);
416}
417#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
418
419#ifdef CONFIG_NUMA_BALANCING
420/*
421 * These work without NUMA balancing but the kernel does not care. See the
422 * comment in include/asm-generic/pgtable.h . On powerpc, this will only
423 * work for user pages and always return true for kernel pages.
424 */
425static inline int pte_protnone(pte_t pte)
426{
66c570f5
AK		 427 return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED)) ==
428 cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED);
13f829a5
AK		 429}
430#endif /* CONFIG_NUMA_BALANCING */
431
432static inline int pte_present(pte_t pte)
433{
66c570f5 434 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
13f829a5
AK		 435}
436/*
437 * Conversion functions: convert a page and protection to a page entry,
438 * and a page entry and page directory to the page they refer to.
439 *
440 * Even if PTEs can be unsigned long long, a PFN is always an unsigned
441 * long for now.
442 */
443static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
444{
445 return __pte((((pte_basic_t)(pfn) << PAGE_SHIFT) & PTE_RPN_MASK) |
446 pgprot_val(pgprot));
447}
448
449static inline unsigned long pte_pfn(pte_t pte)
450{
451 return (pte_val(pte) & PTE_RPN_MASK) >> PAGE_SHIFT;
452}
453
454/* Generic modifiers for PTE bits */
455static inline pte_t pte_wrprotect(pte_t pte)
456{
457 return __pte(pte_val(pte) & ~_PAGE_WRITE);
458}
459
460static inline pte_t pte_mkclean(pte_t pte)
461{
462 return __pte(pte_val(pte) & ~_PAGE_DIRTY);
463}
464
465static inline pte_t pte_mkold(pte_t pte)
466{
467 return __pte(pte_val(pte) & ~_PAGE_ACCESSED);
468}
469
470static inline pte_t pte_mkwrite(pte_t pte)
471{
472 /*
473 * write implies read, hence set both
474 */
475 return __pte(pte_val(pte) | _PAGE_RW);
476}
477
478static inline pte_t pte_mkdirty(pte_t pte)
479{
480 return __pte(pte_val(pte) | _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
481}
482
483static inline pte_t pte_mkyoung(pte_t pte)
484{
485 return __pte(pte_val(pte) | _PAGE_ACCESSED);
486}
487
488static inline pte_t pte_mkspecial(pte_t pte)
489{
490 return __pte(pte_val(pte) | _PAGE_SPECIAL);
491}
492
493static inline pte_t pte_mkhuge(pte_t pte)
494{
495 return pte;
496}
497
498static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
499{
500 /* FIXME!! check whether this need to be a conditional */
501 return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot));
502}
503
34fbadd8
AK		 504static inline bool pte_user(pte_t pte)
505{
66c570f5 506 return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));
34fbadd8
AK		 507}
508
509/* Encode and de-code a swap entry */
510#define MAX_SWAPFILES_CHECK() do { \
511 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
512 /* \
513 * Don't have overlapping bits with _PAGE_HPTEFLAGS \
514 * We filter HPTEFLAGS on set_pte. \
515 */ \
516 BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
517 BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \
518 } while (0)
519/*
520 * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT;
521 */
522#define SWP_TYPE_BITS 5
523#define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
524 & ((1UL << SWP_TYPE_BITS) - 1))
525#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT)
526#define __swp_entry(type, offset) ((swp_entry_t) { \
527 ((type) << _PAGE_BIT_SWAP_TYPE) \
528 | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)})
529/*
530 * swp_entry_t must be independent of pte bits. We build a swp_entry_t from
531 * swap type and offset we get from swap and convert that to pte to find a
532 * matching pte in linux page table.
533 * Clear bits not found in swap entries here.
534 */
535#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE })
536#define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE)
537
538#ifdef CONFIG_MEM_SOFT_DIRTY
539#define _PAGE_SWP_SOFT_DIRTY (1UL << (SWP_TYPE_BITS + _PAGE_BIT_SWAP_TYPE))
540#else
541#define _PAGE_SWP_SOFT_DIRTY 0UL
542#endif /* CONFIG_MEM_SOFT_DIRTY */
543
544#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
545static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
546{
547 return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY);
548}
66c570f5 549
34fbadd8
AK		 550static inline bool pte_swp_soft_dirty(pte_t pte)
551{
66c570f5 552 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
34fbadd8 553}
66c570f5 554
34fbadd8
AK		 555static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
556{
557 return __pte(pte_val(pte) & ~_PAGE_SWP_SOFT_DIRTY);
558}
559#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
560
561static inline bool check_pte_access(unsigned long access, unsigned long ptev)
562{
563 /*
564 * This check for _PAGE_RWX and _PAGE_PRESENT bits
565 */
566 if (access & ~ptev)
567 return false;
568 /*
569 * This check for access to privilege space
570 */
571 if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED))
572 return false;
573
574 return true;
575}
ac94ac79
AK		 576/*
577 * Generic functions with hash/radix callbacks
578 */
579
c6d1a767 580static inline void __ptep_set_access_flags(struct mm_struct *mm,
b3603e17
AK		 581 pte_t *ptep, pte_t entry,
582 unsigned long address)
ac94ac79
AK		 583{
584 if (radix_enabled())
b3603e17 585 return radix__ptep_set_access_flags(mm, ptep, entry, address);
ac94ac79
AK		 586 return hash__ptep_set_access_flags(ptep, entry);
587}
588
589#define __HAVE_ARCH_PTE_SAME
590static inline int pte_same(pte_t pte_a, pte_t pte_b)
591{
592 if (radix_enabled())
593 return radix__pte_same(pte_a, pte_b);
594 return hash__pte_same(pte_a, pte_b);
595}
596
597static inline int pte_none(pte_t pte)
598{
599 if (radix_enabled())
600 return radix__pte_none(pte);
601 return hash__pte_none(pte);
602}
603
604static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
605 pte_t *ptep, pte_t pte, int percpu)
606{
607 if (radix_enabled())
608 return radix__set_pte_at(mm, addr, ptep, pte, percpu);
609 return hash__set_pte_at(mm, addr, ptep, pte, percpu);
610}
34fbadd8 611
13f829a5
AK		 612#define _PAGE_CACHE_CTL (_PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
613
614#define pgprot_noncached pgprot_noncached
615static inline pgprot_t pgprot_noncached(pgprot_t prot)
616{
617 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
618 _PAGE_NON_IDEMPOTENT);
619}
620
621#define pgprot_noncached_wc pgprot_noncached_wc
622static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
623{
624 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
625 _PAGE_TOLERANT);
626}
627
628#define pgprot_cached pgprot_cached
629static inline pgprot_t pgprot_cached(pgprot_t prot)
630{
631 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL));
632}
633
634#define pgprot_writecombine pgprot_writecombine
635static inline pgprot_t pgprot_writecombine(pgprot_t prot)
636{
637 return pgprot_noncached_wc(prot);
638}
639/*
640 * check a pte mapping have cache inhibited property
641 */
642static inline bool pte_ci(pte_t pte)
643{
644 unsigned long pte_v = pte_val(pte);
645
646 if (((pte_v & _PAGE_CACHE_CTL) == _PAGE_TOLERANT) ||
647 ((pte_v & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT))
648 return true;
649 return false;
650}
651
f281b5d5
AK		 652static inline void pmd_set(pmd_t *pmdp, unsigned long val)
653{
654 *pmdp = __pmd(val);
655}
656
657static inline void pmd_clear(pmd_t *pmdp)
658{
659 *pmdp = __pmd(0);
660}
661
66c570f5
AK		 662static inline int pmd_none(pmd_t pmd)
663{
664 return !pmd_raw(pmd);
665}
666
667static inline int pmd_present(pmd_t pmd)
668{
669
670 return !pmd_none(pmd);
671}
3dfcb315 672
ac94ac79
AK		 673static inline int pmd_bad(pmd_t pmd)
674{
675 if (radix_enabled())
676 return radix__pmd_bad(pmd);
677 return hash__pmd_bad(pmd);
678}
679
f281b5d5
AK		 680static inline void pud_set(pud_t *pudp, unsigned long val)
681{
682 *pudp = __pud(val);
683}
684
685static inline void pud_clear(pud_t *pudp)
686{
687 *pudp = __pud(0);
688}
689
66c570f5
AK		 690static inline int pud_none(pud_t pud)
691{
692 return !pud_raw(pud);
693}
694
695static inline int pud_present(pud_t pud)
696{
697 return !pud_none(pud);
698}
3dfcb315
AK		 699
700extern struct page *pud_page(pud_t pud);
371352ca 701extern struct page *pmd_page(pmd_t pmd);
3dfcb315
AK		 702static inline pte_t pud_pte(pud_t pud)
703{
66c570f5 704 return __pte_raw(pud_raw(pud));
3dfcb315
AK		 705}
706
707static inline pud_t pte_pud(pte_t pte)
708{
66c570f5 709 return __pud_raw(pte_raw(pte));
3dfcb315
AK		 710}
711#define pud_write(pud) pte_write(pud_pte(pud))
ac94ac79
AK		 712
713static inline int pud_bad(pud_t pud)
714{
715 if (radix_enabled())
716 return radix__pud_bad(pud);
717 return hash__pud_bad(pud);
718}
719
720
3dfcb315 721#define pgd_write(pgd) pte_write(pgd_pte(pgd))
f281b5d5
AK		 722static inline void pgd_set(pgd_t *pgdp, unsigned long val)
723{
724 *pgdp = __pgd(val);
725}
3dfcb315 726
368ced78
AK		 727static inline void pgd_clear(pgd_t *pgdp)
728{
729 *pgdp = __pgd(0);
730}
731
66c570f5
AK		 732static inline int pgd_none(pgd_t pgd)
733{
734 return !pgd_raw(pgd);
735}
736
737static inline int pgd_present(pgd_t pgd)
738{
739 return !pgd_none(pgd);
740}
368ced78
AK		 741
742static inline pte_t pgd_pte(pgd_t pgd)
743{
66c570f5 744 return __pte_raw(pgd_raw(pgd));
368ced78
AK		 745}
746
747static inline pgd_t pte_pgd(pte_t pte)
748{
66c570f5 749 return __pgd_raw(pte_raw(pte));
368ced78
AK		 750}
751
ac94ac79
AK		 752static inline int pgd_bad(pgd_t pgd)
753{
754 if (radix_enabled())
755 return radix__pgd_bad(pgd);
756 return hash__pgd_bad(pgd);
757}
758
368ced78
AK		 759extern struct page *pgd_page(pgd_t pgd);
760
aba480e1
AK		 761/* Pointers in the page table tree are physical addresses */
762#define __pgtable_ptr_val(ptr) __pa(ptr)
763
764#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)
765#define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS)
766#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS)
767
768#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
769#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))
770#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
771#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))
772
3dfcb315
AK		 773/*
774 * Find an entry in a page-table-directory. We combine the address region
775 * (the high order N bits) and the pgd portion of the address.
776 */
3dfcb315
AK		 777
778#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
779
368ced78
AK		 780#define pud_offset(pgdp, addr) \
781 (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr))
3dfcb315 782#define pmd_offset(pudp,addr) \
371352ca 783 (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))
3dfcb315 784#define pte_offset_kernel(dir,addr) \
371352ca 785 (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr))
3dfcb315
AK		 786
787#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
788#define pte_unmap(pte) do { } while(0)
789
790/* to find an entry in a kernel page-table-directory */
791/* This now only contains the vmalloc pages */
792#define pgd_offset_k(address) pgd_offset(&init_mm, address)
3dfcb315
AK		 793
794#define pte_ERROR(e) \
795 pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
796#define pmd_ERROR(e) \
797 pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
368ced78
AK		 798#define pud_ERROR(e) \
799 pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
3dfcb315
AK		 800#define pgd_ERROR(e) \
801 pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
802
31a14fae
AK		 803static inline int map_kernel_page(unsigned long ea, unsigned long pa,
804 unsigned long flags)
7207f436 805{
d9225ad9
AK		 806 if (radix_enabled()) {
807#if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM)
808 unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift;
809 WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE");
810#endif
811 return radix__map_kernel_page(ea, pa, __pgprot(flags), PAGE_SIZE);
812 }
31a14fae 813 return hash__map_kernel_page(ea, pa, flags);
7207f436 814}
31a14fae
AK		 815
816static inline int __meminit vmemmap_create_mapping(unsigned long start,
817 unsigned long page_size,
818 unsigned long phys)
7207f436 819{
d9225ad9
AK		 820 if (radix_enabled())
821 return radix__vmemmap_create_mapping(start, page_size, phys);
31a14fae 822 return hash__vmemmap_create_mapping(start, page_size, phys);
7207f436 823}
31a14fae
AK		 824
825#ifdef CONFIG_MEMORY_HOTPLUG
826static inline void vmemmap_remove_mapping(unsigned long start,
827 unsigned long page_size)
7207f436 828{
d9225ad9
AK		 829 if (radix_enabled())
830 return radix__vmemmap_remove_mapping(start, page_size);
31a14fae 831 return hash__vmemmap_remove_mapping(start, page_size);
7207f436 832}
31a14fae 833#endif
3dfcb315
AK		 834struct page *realmode_pfn_to_page(unsigned long pfn);
835
3dfcb315
AK		 836static inline pte_t pmd_pte(pmd_t pmd)
837{
66c570f5 838 return __pte_raw(pmd_raw(pmd));
3dfcb315
AK		 839}
840
841static inline pmd_t pte_pmd(pte_t pte)
842{
66c570f5 843 return __pmd_raw(pte_raw(pte));
3dfcb315
AK		 844}
845
846static inline pte_t *pmdp_ptep(pmd_t *pmd)
847{
848 return (pte_t *)pmd;
849}
3dfcb315
AK		 850#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
851#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
852#define pmd_young(pmd) pte_young(pmd_pte(pmd))
853#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
854#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
855#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
d5d6a443 856#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
3dfcb315
AK		 857#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
858#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
7207f436
LD		 859
860#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
861#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
862#define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)))
863#define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)))
864#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
865
1ca72129
AK		 866#ifdef CONFIG_NUMA_BALANCING
867static inline int pmd_protnone(pmd_t pmd)
868{
869 return pte_protnone(pmd_pte(pmd));
870}
871#endif /* CONFIG_NUMA_BALANCING */
3dfcb315
AK		 872
873#define __HAVE_ARCH_PMD_WRITE
874#define pmd_write(pmd) pte_write(pmd_pte(pmd))
875
6a1ea362
AK		 876#ifdef CONFIG_TRANSPARENT_HUGEPAGE
877extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
878extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
879extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
880extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
881 pmd_t *pmdp, pmd_t pmd);
882extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
883 pmd_t *pmd);
3df33f12
AK		 884extern int hash__has_transparent_hugepage(void);
885static inline int has_transparent_hugepage(void)
886{
bde3eb62
AK		 887 if (radix_enabled())
888 return radix__has_transparent_hugepage();
3df33f12
AK		 889 return hash__has_transparent_hugepage();
890}
c04a5880 891#define has_transparent_hugepage has_transparent_hugepage
6a1ea362 892
3df33f12
AK		 893static inline unsigned long
894pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
895 unsigned long clr, unsigned long set)
3dfcb315 896{
bde3eb62
AK		 897 if (radix_enabled())
898 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set);
3df33f12
AK		 899 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set);
900}
901
902static inline int pmd_large(pmd_t pmd)
903{
66c570f5 904 return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
3df33f12
AK		 905}
906
907static inline pmd_t pmd_mknotpresent(pmd_t pmd)
908{
909 return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
910}
911/*
912 * For radix we should always find H_PAGE_HASHPTE zero. Hence
913 * the below will work for radix too
914 */
915static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
916 unsigned long addr, pmd_t *pmdp)
917{
918 unsigned long old;
919
66c570f5 920 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
3df33f12
AK		 921 return 0;
922 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
923 return ((old & _PAGE_ACCESSED) != 0);
924}
925
926#define __HAVE_ARCH_PMDP_SET_WRPROTECT
927static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
928 pmd_t *pmdp)
929{
930
66c570f5 931 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_WRITE)) == 0)
3df33f12
AK		 932 return;
933
934 pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
3dfcb315
AK		 935}
936
ab624762
AK		 937static inline int pmd_trans_huge(pmd_t pmd)
938{
939 if (radix_enabled())
940 return radix__pmd_trans_huge(pmd);
941 return hash__pmd_trans_huge(pmd);
942}
943
944#define __HAVE_ARCH_PMD_SAME
945static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
946{
947 if (radix_enabled())
948 return radix__pmd_same(pmd_a, pmd_b);
949 return hash__pmd_same(pmd_a, pmd_b);
950}
951
3dfcb315
AK		 952static inline pmd_t pmd_mkhuge(pmd_t pmd)
953{
ab624762
AK		 954 if (radix_enabled())
955 return radix__pmd_mkhuge(pmd);
956 return hash__pmd_mkhuge(pmd);
3dfcb315
AK		 957}
958
3dfcb315
AK		 959#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
960extern int pmdp_set_access_flags(struct vm_area_struct *vma,
961 unsigned long address, pmd_t *pmdp,
962 pmd_t entry, int dirty);
963
3dfcb315
AK		 964#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
965extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
966 unsigned long address, pmd_t *pmdp);
3dfcb315
AK		 967
968#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
3df33f12
AK		 969static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
970 unsigned long addr, pmd_t *pmdp)
971{
bde3eb62
AK		 972 if (radix_enabled())
973 return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
3df33f12
AK		 974 return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
975}
3dfcb315 976
3df33f12
AK		 977static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
978 unsigned long address, pmd_t *pmdp)
979{
bde3eb62
AK		 980 if (radix_enabled())
981 return radix__pmdp_collapse_flush(vma, address, pmdp);
3df33f12
AK		 982 return hash__pmdp_collapse_flush(vma, address, pmdp);
983}
3dfcb315
AK		 984#define pmdp_collapse_flush pmdp_collapse_flush
985
986#define __HAVE_ARCH_PGTABLE_DEPOSIT
3df33f12
AK		 987static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
988 pmd_t *pmdp, pgtable_t pgtable)
989{
bde3eb62
AK		 990 if (radix_enabled())
991 return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
3df33f12
AK		 992 return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
993}
994
3dfcb315 995#define __HAVE_ARCH_PGTABLE_WITHDRAW
3df33f12
AK		 996static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
997 pmd_t *pmdp)
998{
bde3eb62
AK		 999 if (radix_enabled())
1000 return radix__pgtable_trans_huge_withdraw(mm, pmdp);
3df33f12
AK		 1001 return hash__pgtable_trans_huge_withdraw(mm, pmdp);
1002}
3dfcb315
AK		 1003
1004#define __HAVE_ARCH_PMDP_INVALIDATE
1005extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
1006 pmd_t *pmdp);
1007
c777e2a8 1008#define __HAVE_ARCH_PMDP_HUGE_SPLIT_PREPARE
3df33f12
AK		 1009static inline void pmdp_huge_split_prepare(struct vm_area_struct *vma,
1010 unsigned long address, pmd_t *pmdp)
1011{
bde3eb62
AK		 1012 if (radix_enabled())
1013 return radix__pmdp_huge_split_prepare(vma, address, pmdp);
3df33f12
AK		 1014 return hash__pmdp_huge_split_prepare(vma, address, pmdp);
1015}
c777e2a8 1016
3dfcb315
AK		 1017#define pmd_move_must_withdraw pmd_move_must_withdraw
1018struct spinlock;
1019static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
1dd38b6c
AK		 1020 struct spinlock *old_pmd_ptl,
1021 struct vm_area_struct *vma)
3dfcb315 1022{
bde3eb62
AK		 1023 if (radix_enabled())
1024 return false;
3dfcb315
AK		 1025 /*
1026 * Archs like ppc64 use pgtable to store per pmd
1027 * specific information. So when we switch the pmd,
1028 * we should also withdraw and deposit the pgtable
1029 */
1030 return true;
1031}
953c66c2
AK		 1032
1033
1034#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit
1035static inline bool arch_needs_pgtable_deposit(void)
1036{
1037 if (radix_enabled())
1038 return false;
1039 return true;
1040}
1041
6a1ea362 1042#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3dfcb315
AK		 1043#endif /* __ASSEMBLY__ */
1044#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
Linux 6.x block layer and io_uring tree(s)