projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/mm: Lower the max real address to 53 bits
[linux-2.6-block.git] / arch / powerpc / include / asm / book3s / 64 / pgtable.h
CommitLineData
3dfcb315
AK		 1#ifndef _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
2#define _ASM_POWERPC_BOOK3S_64_PGTABLE_H_
2e873519 3
9849a569
KS		 4#include <asm-generic/5level-fixup.h>
5
c137a275
AK		 6#ifndef __ASSEMBLY__
7#include <linux/mmdebug.h>
8#endif
9849a569 9
2e873519
AK		 10/*
11 * Common bits between hash and Radix page table
12 */
13#define _PAGE_BIT_SWAP_TYPE 0
14
6b8cb66a
CL		 15#define _PAGE_RO 0
16
2e873519
AK		 17#define _PAGE_EXEC 0x00001 /* execute permission */
18#define _PAGE_WRITE 0x00002 /* write access allowed */
19#define _PAGE_READ 0x00004 /* read access allowed */
20#define _PAGE_RW (_PAGE_READ | _PAGE_WRITE)
21#define _PAGE_RWX (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)
22#define _PAGE_PRIVILEGED 0x00008 /* kernel access only */
23#define _PAGE_SAO 0x00010 /* Strong access order */
24#define _PAGE_NON_IDEMPOTENT 0x00020 /* non idempotent memory */
25#define _PAGE_TOLERANT 0x00030 /* tolerant memory, cache inhibited */
26#define _PAGE_DIRTY 0x00080 /* C: page changed */
27#define _PAGE_ACCESSED 0x00100 /* R: page referenced */
3dfcb315 28/*
2e873519 29 * Software bits
3dfcb315 30 */
69dfbaeb
AK		 31#define _RPAGE_SW0 0x2000000000000000UL
32#define _RPAGE_SW1 0x00800
33#define _RPAGE_SW2 0x00400
34#define _RPAGE_SW3 0x00200
049d567a
AK		 35#define _RPAGE_RSV1 0x1000000000000000UL
36#define _RPAGE_RSV2 0x0800000000000000UL
37#define _RPAGE_RSV3 0x0400000000000000UL
38#define _RPAGE_RSV4 0x0200000000000000UL
32789d38
AK		 39#define _RPAGE_RPN0 0x01000
40#define _RPAGE_RPN1 0x02000
049d567a 41
2f18d533
AK		 42/* Max physical address bit as per radix table */
43#define _RPAGE_PA_MAX 57
44
45/*
46 * Max physical address bit we will use for now.
47 *
48 * This is mostly a hardware limitation and for now Power9 has
49 * a 51 bit limit.
50 *
51 * This is different from the number of physical bit required to address
52 * the last byte of memory. That is defined by MAX_PHYSMEM_BITS.
53 * MAX_PHYSMEM_BITS is a linux limitation imposed by the maximum
54 * number of sections we can support (SECTIONS_SHIFT).
55 *
56 * This is different from Radix page table limitation above and
57 * should always be less than that. The limit is done such that
58 * we can overload the bits between _RPAGE_PA_MAX and _PAGE_PA_MAX
59 * for hash linux page table specific bits.
60 *
61 * In order to be compatible with future hardware generations we keep
62 * some offsets and limit this for now to 53
63 */
64#define _PAGE_PA_MAX 53
65
69dfbaeb 66#define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */
69dfbaeb 67#define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */
2e873519 68
f5bd0fdc
AK		 69#define _PAGE_PTE 0x4000000000000000UL /* distinguishes PTEs from pointers */
70#define _PAGE_PRESENT 0x8000000000000000UL /* pte contains a translation */
2e873519
AK		 71/*
72 * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE
73 * Instead of fixing all of them, add an alternate define which
74 * maps CI pte mapping.
75 */
76#define _PAGE_NO_CACHE _PAGE_TOLERANT
77/*
2f18d533
AK		 78 * We support _RPAGE_PA_MAX bit real address in pte. On the linux side
79 * we are limited by _PAGE_PA_MAX. Clear everything above _PAGE_PA_MAX
80 * and every thing below PAGE_SHIFT;
2e873519 81 */
2f18d533 82#define PTE_RPN_MASK (((1UL << _PAGE_PA_MAX) - 1) & (PAGE_MASK))
2e873519
AK		 83/*
84 * set of bits not changed in pmd_modify. Even though we have hash specific bits
85 * in here, on radix we expect them to be zero.
86 */
87#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
88 _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \
89 _PAGE_SOFT_DIRTY)
90/*
91 * user access blocked by key
92 */
93#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY)
94#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ)
95#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \
96 _PAGE_RW | _PAGE_EXEC)
97/*
98 * No page size encoding in the linux PTE
99 */
100#define _PAGE_PSIZE 0
101/*
102 * _PAGE_CHG_MASK masks of bits that are to be preserved across
103 * pgprot changes
104 */
105#define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
106 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
107 _PAGE_SOFT_DIRTY)
108/*
109 * Mask of bits returned by pte_pgprot()
110 */
111#define PAGE_PROT_BITS (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT | \
112 H_PAGE_4K_PFN | _PAGE_PRIVILEGED | _PAGE_ACCESSED | \
113 _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_EXEC | \
114 _PAGE_SOFT_DIRTY)
3dfcb315 115/*
2e873519
AK		 116 * We define 2 sets of base prot bits, one for basic pages (ie,
117 * cacheable kernel and user pages) and one for non cacheable
118 * pages. We always set _PAGE_COHERENT when SMP is enabled or
119 * the processor might need it for DMA coherency.
3dfcb315 120 */
2e873519
AK		 121#define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE)
122#define _PAGE_BASE (_PAGE_BASE_NC)
123
124/* Permission masks used to generate the __P and __S table,
125 *
126 * Note:__pgprot is defined in arch/powerpc/include/asm/page.h
127 *
128 * Write permissions imply read permissions for now (we could make write-only
129 * pages on BookE but we don't bother for now). Execute permission control is
130 * possible on platforms that define _PAGE_EXEC
131 *
132 * Note due to the way vm flags are laid out, the bits are XWR
133 */
134#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_PRIVILEGED)
135#define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_RW)
136#define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_RW | _PAGE_EXEC)
137#define PAGE_COPY __pgprot(_PAGE_BASE | _PAGE_READ)
138#define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
139#define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ)
140#define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC)
141
142#define __P000 PAGE_NONE
143#define __P001 PAGE_READONLY
144#define __P010 PAGE_COPY
145#define __P011 PAGE_COPY
146#define __P100 PAGE_READONLY_X
147#define __P101 PAGE_READONLY_X
148#define __P110 PAGE_COPY_X
149#define __P111 PAGE_COPY_X
150
151#define __S000 PAGE_NONE
152#define __S001 PAGE_READONLY
153#define __S010 PAGE_SHARED
154#define __S011 PAGE_SHARED
155#define __S100 PAGE_READONLY_X
156#define __S101 PAGE_READONLY_X
157#define __S110 PAGE_SHARED_X
158#define __S111 PAGE_SHARED_X
159
160/* Permission masks used for kernel mappings */
161#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW)
162#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
163 _PAGE_TOLERANT)
164#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
165 _PAGE_NON_IDEMPOTENT)
166#define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX)
167#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO)
168#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)
169
170/*
171 * Protection used for kernel text. We want the debuggers to be able to
172 * set breakpoints anywhere, so don't write protect the kernel text
173 * on platforms where such control is possible.
174 */
175#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \
176 defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE)
177#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
178#else
179#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
180#endif
181
182/* Make modules code happy. We don't set RO yet */
183#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
184#define PAGE_AGP (PAGE_KERNEL_NC)
3dfcb315 185
dd1842a2
AK		 186#ifndef __ASSEMBLY__
187/*
188 * page table defines
189 */
190extern unsigned long __pte_index_size;
191extern unsigned long __pmd_index_size;
192extern unsigned long __pud_index_size;
193extern unsigned long __pgd_index_size;
194extern unsigned long __pmd_cache_index;
195#define PTE_INDEX_SIZE __pte_index_size
196#define PMD_INDEX_SIZE __pmd_index_size
197#define PUD_INDEX_SIZE __pud_index_size
198#define PGD_INDEX_SIZE __pgd_index_size
199#define PMD_CACHE_INDEX __pmd_cache_index
200/*
201 * Because of use of pte fragments and THP, size of page table
202 * are not always derived out of index size above.
203 */
204extern unsigned long __pte_table_size;
205extern unsigned long __pmd_table_size;
206extern unsigned long __pud_table_size;
207extern unsigned long __pgd_table_size;
208#define PTE_TABLE_SIZE __pte_table_size
209#define PMD_TABLE_SIZE __pmd_table_size
210#define PUD_TABLE_SIZE __pud_table_size
211#define PGD_TABLE_SIZE __pgd_table_size
a2f41eb9
AK		 212
213extern unsigned long __pmd_val_bits;
214extern unsigned long __pud_val_bits;
215extern unsigned long __pgd_val_bits;
216#define PMD_VAL_BITS __pmd_val_bits
217#define PUD_VAL_BITS __pud_val_bits
218#define PGD_VAL_BITS __pgd_val_bits
5ed7ecd0
AK		 219
220extern unsigned long __pte_frag_nr;
221#define PTE_FRAG_NR __pte_frag_nr
222extern unsigned long __pte_frag_size_shift;
223#define PTE_FRAG_SIZE_SHIFT __pte_frag_size_shift
224#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
dd1842a2
AK		 225/*
226 * Pgtable size used by swapper, init in asm code
dd1842a2 227 */
a2f41eb9 228#define MAX_PGD_TABLE_SIZE (sizeof(pgd_t) << RADIX_PGD_INDEX_SIZE)
dd1842a2
AK		 229
230#define PTRS_PER_PTE (1 << PTE_INDEX_SIZE)
231#define PTRS_PER_PMD (1 << PMD_INDEX_SIZE)
232#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE)
233#define PTRS_PER_PGD (1 << PGD_INDEX_SIZE)
234
235/* PMD_SHIFT determines what a second-level page table entry can map */
236#define PMD_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
237#define PMD_SIZE (1UL << PMD_SHIFT)
238#define PMD_MASK (~(PMD_SIZE-1))
239
240/* PUD_SHIFT determines what a third-level page table entry can map */
241#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE)
242#define PUD_SIZE (1UL << PUD_SHIFT)
243#define PUD_MASK (~(PUD_SIZE-1))
244
245/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
246#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE)
247#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
248#define PGDIR_MASK (~(PGDIR_SIZE-1))
249
250/* Bits to mask out from a PMD to get to the PTE page */
251#define PMD_MASKED_BITS 0xc0000000000000ffUL
252/* Bits to mask out from a PUD to get to the PMD page */
253#define PUD_MASKED_BITS 0xc0000000000000ffUL
254/* Bits to mask out from a PGD to get to the PUD page */
255#define PGD_MASKED_BITS 0xc0000000000000ffUL
d6a9996e
AK		 256
257extern unsigned long __vmalloc_start;
258extern unsigned long __vmalloc_end;
259#define VMALLOC_START __vmalloc_start
260#define VMALLOC_END __vmalloc_end
261
262extern unsigned long __kernel_virt_start;
263extern unsigned long __kernel_virt_size;
264#define KERN_VIRT_START __kernel_virt_start
265#define KERN_VIRT_SIZE __kernel_virt_size
266extern struct page *vmemmap;
267extern unsigned long ioremap_bot;
bfa37087 268extern unsigned long pci_io_base;
dd1842a2 269#endif /* __ASSEMBLY__ */
3dfcb315 270
ab537dca 271#include <asm/book3s/64/hash.h>
b0b5e9b1 272#include <asm/book3s/64/radix.h>
3dfcb315 273
a9252aae
AK		 274#ifdef CONFIG_PPC_64K_PAGES
275#include <asm/book3s/64/pgtable-64k.h>
276#else
277#include <asm/book3s/64/pgtable-4k.h>
278#endif
279
3dfcb315 280#include <asm/barrier.h>
3dfcb315
AK		 281/*
282 * The second half of the kernel virtual space is used for IO mappings,
283 * it's itself carved into the PIO region (ISA and PHB IO space) and
284 * the ioremap space
285 *
286 * ISA_IO_BASE = KERN_IO_START, 64K reserved area
287 * PHB_IO_BASE = ISA_IO_BASE + 64K to ISA_IO_BASE + 2G, PHB IO spaces
288 * IOREMAP_BASE = ISA_IO_BASE + 2G to VMALLOC_START + PGTABLE_RANGE
289 */
290#define KERN_IO_START (KERN_VIRT_START + (KERN_VIRT_SIZE >> 1))
291#define FULL_IO_SIZE 0x80000000ul
292#define ISA_IO_BASE (KERN_IO_START)
293#define ISA_IO_END (KERN_IO_START + 0x10000ul)
294#define PHB_IO_BASE (ISA_IO_END)
295#define PHB_IO_END (KERN_IO_START + FULL_IO_SIZE)
296#define IOREMAP_BASE (PHB_IO_END)
297#define IOREMAP_END (KERN_VIRT_START + KERN_VIRT_SIZE)
298
b0412ea9 299/* Advertise special mapping type for AGP */
b0412ea9
AK		 300#define HAVE_PAGE_AGP
301
302/* Advertise support for _PAGE_SPECIAL */
303#define __HAVE_ARCH_PTE_SPECIAL
304
3dfcb315
AK		 305#ifndef __ASSEMBLY__
306
307/*
308 * This is the default implementation of various PTE accessors, it's
309 * used in all cases except Book3S with 64K pages where we have a
310 * concept of sub-pages
311 */
312#ifndef __real_pte
313
3dfcb315
AK		 314#define __real_pte(e,p) ((real_pte_t){(e)})
315#define __rpte_to_pte(r) ((r).pte)
945537df 316#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
3dfcb315
AK		 317
318#define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \
319 do { \
320 index = 0; \
321 shift = mmu_psize_defs[psize].shift; \
322
323#define pte_iterate_hashed_end() } while(0)
324
325/*
326 * We expect this to be called only for user addresses or kernel virtual
327 * addresses other than the linear mapping.
328 */
329#define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K
330
331#endif /* __real_pte */
332
ac94ac79
AK		 333static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr,
334 pte_t *ptep, unsigned long clr,
335 unsigned long set, int huge)
336{
337 if (radix_enabled())
338 return radix__pte_update(mm, addr, ptep, clr, set, huge);
339 return hash__pte_update(mm, addr, ptep, clr, set, huge);
340}
13f829a5
AK		 341/*
342 * For hash even if we have _PAGE_ACCESSED = 0, we do a pte_update.
343 * We currently remove entries from the hashtable regardless of whether
344 * the entry was young or dirty.
345 *
346 * We should be more intelligent about this but for the moment we override
347 * these functions and force a tlb flush unconditionally
348 * For radix: H_PAGE_HASHPTE should be zero. Hence we can use the same
349 * function for both hash and radix.
350 */
351static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
352 unsigned long addr, pte_t *ptep)
353{
354 unsigned long old;
355
66c570f5 356 if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
13f829a5
AK		 357 return 0;
358 old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
359 return (old & _PAGE_ACCESSED) != 0;
360}
361
362#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
363#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
364({ \
365 int __r; \
366 __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
367 __r; \
368})
369
d19469e8 370static inline int __pte_write(pte_t pte)
52c50ca7
AK		 371{
372 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE));
373}
374
375#ifdef CONFIG_NUMA_BALANCING
376#define pte_savedwrite pte_savedwrite
377static inline bool pte_savedwrite(pte_t pte)
378{
379 /*
380 * Saved write ptes are prot none ptes that doesn't have
381 * privileged bit sit. We mark prot none as one which has
382 * present and pviliged bit set and RWX cleared. To mark
383 * protnone which used to have _PAGE_WRITE set we clear
384 * the privileged bit.
385 */
386 return !(pte_raw(pte) & cpu_to_be64(_PAGE_RWX | _PAGE_PRIVILEGED));
387}
388#else
389#define pte_savedwrite pte_savedwrite
390static inline bool pte_savedwrite(pte_t pte)
391{
392 return false;
393}
394#endif
395
d19469e8
AK		 396static inline int pte_write(pte_t pte)
397{
398 return __pte_write(pte) || pte_savedwrite(pte);
399}
400
13f829a5
AK		 401#define __HAVE_ARCH_PTEP_SET_WRPROTECT
402static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
403 pte_t *ptep)
404{
d19469e8 405 if (__pte_write(*ptep))
52c50ca7
AK		 406 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0);
407 else if (unlikely(pte_savedwrite(*ptep)))
408 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 0);
13f829a5
AK		 409}
410
411static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
412 unsigned long addr, pte_t *ptep)
413{
52c50ca7
AK		 414 /*
415 * We should not find protnone for hugetlb, but this complete the
416 * interface.
417 */
d19469e8 418 if (__pte_write(*ptep))
52c50ca7
AK		 419 pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1);
420 else if (unlikely(pte_savedwrite(*ptep)))
421 pte_update(mm, addr, ptep, 0, _PAGE_PRIVILEGED, 1);
13f829a5
AK		 422}
423
424#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
425static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
426 unsigned long addr, pte_t *ptep)
427{
428 unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
429 return __pte(old);
430}
431
f4894b80
AK		 432#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
433static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
434 unsigned long addr,
435 pte_t *ptep, int full)
436{
437 if (full && radix_enabled()) {
438 /*
439 * Let's skip the DD1 style pte update here. We know that
440 * this is a full mm pte clear and hence can be sure there is
441 * no parallel set_pte.
442 */
443 return radix__ptep_get_and_clear_full(mm, addr, ptep, full);
444 }
445 return ptep_get_and_clear(mm, addr, ptep);
446}
447
448
13f829a5
AK		 449static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
450 pte_t * ptep)
451{
452 pte_update(mm, addr, ptep, ~0UL, 0, 0);
453}
66c570f5 454
66c570f5
AK		 455static inline int pte_dirty(pte_t pte)
456{
457 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY));
458}
459
460static inline int pte_young(pte_t pte)
461{
462 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED));
463}
464
465static inline int pte_special(pte_t pte)
466{
467 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL));
468}
469
13f829a5
AK		 470static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
471
472#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
473static inline bool pte_soft_dirty(pte_t pte)
474{
66c570f5 475 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY));
13f829a5 476}
66c570f5 477
13f829a5
AK		 478static inline pte_t pte_mksoft_dirty(pte_t pte)
479{
480 return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
481}
482
483static inline pte_t pte_clear_soft_dirty(pte_t pte)
484{
485 return __pte(pte_val(pte) & ~_PAGE_SOFT_DIRTY);
486}
487#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
488
489#ifdef CONFIG_NUMA_BALANCING
13f829a5
AK		 490static inline int pte_protnone(pte_t pte)
491{
c137a275
AK		 492 return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE | _PAGE_RWX)) ==
493 cpu_to_be64(_PAGE_PRESENT | _PAGE_PTE);
494}
495
496#define pte_mk_savedwrite pte_mk_savedwrite
497static inline pte_t pte_mk_savedwrite(pte_t pte)
498{
499 /*
500 * Used by Autonuma subsystem to preserve the write bit
501 * while marking the pte PROT_NONE. Only allow this
502 * on PROT_NONE pte
503 */
504 VM_BUG_ON((pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_RWX | _PAGE_PRIVILEGED)) !=
505 cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED));
506 return __pte(pte_val(pte) & ~_PAGE_PRIVILEGED);
507}
508
509#define pte_clear_savedwrite pte_clear_savedwrite
510static inline pte_t pte_clear_savedwrite(pte_t pte)
511{
512 /*
513 * Used by KSM subsystem to make a protnone pte readonly.
514 */
515 VM_BUG_ON(!pte_protnone(pte));
516 return __pte(pte_val(pte) | _PAGE_PRIVILEGED);
517}
d19469e8
AK		 518#else
519#define pte_clear_savedwrite pte_clear_savedwrite
520static inline pte_t pte_clear_savedwrite(pte_t pte)
521{
522 VM_WARN_ON(1);
523 return __pte(pte_val(pte) & ~_PAGE_WRITE);
524}
13f829a5
AK		 525#endif /* CONFIG_NUMA_BALANCING */
526
527static inline int pte_present(pte_t pte)
528{
66c570f5 529 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
13f829a5
AK		 530}
531/*
532 * Conversion functions: convert a page and protection to a page entry,
533 * and a page entry and page directory to the page they refer to.
534 *
535 * Even if PTEs can be unsigned long long, a PFN is always an unsigned
536 * long for now.
537 */
538static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
539{
540 return __pte((((pte_basic_t)(pfn) << PAGE_SHIFT) & PTE_RPN_MASK) |
541 pgprot_val(pgprot));
542}
543
544static inline unsigned long pte_pfn(pte_t pte)
545{
546 return (pte_val(pte) & PTE_RPN_MASK) >> PAGE_SHIFT;
547}
548
549/* Generic modifiers for PTE bits */
550static inline pte_t pte_wrprotect(pte_t pte)
551{
d19469e8
AK		 552 if (unlikely(pte_savedwrite(pte)))
553 return pte_clear_savedwrite(pte);
13f829a5
AK		 554 return __pte(pte_val(pte) & ~_PAGE_WRITE);
555}
556
557static inline pte_t pte_mkclean(pte_t pte)
558{
559 return __pte(pte_val(pte) & ~_PAGE_DIRTY);
560}
561
562static inline pte_t pte_mkold(pte_t pte)
563{
564 return __pte(pte_val(pte) & ~_PAGE_ACCESSED);
565}
566
567static inline pte_t pte_mkwrite(pte_t pte)
568{
569 /*
570 * write implies read, hence set both
571 */
572 return __pte(pte_val(pte) | _PAGE_RW);
573}
574
575static inline pte_t pte_mkdirty(pte_t pte)
576{
577 return __pte(pte_val(pte) | _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
578}
579
580static inline pte_t pte_mkyoung(pte_t pte)
581{
582 return __pte(pte_val(pte) | _PAGE_ACCESSED);
583}
584
585static inline pte_t pte_mkspecial(pte_t pte)
586{
587 return __pte(pte_val(pte) | _PAGE_SPECIAL);
588}
589
590static inline pte_t pte_mkhuge(pte_t pte)
591{
592 return pte;
593}
594
595static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
596{
597 /* FIXME!! check whether this need to be a conditional */
598 return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot));
599}
600
34fbadd8
AK		 601static inline bool pte_user(pte_t pte)
602{
66c570f5 603 return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED));
34fbadd8
AK		 604}
605
606/* Encode and de-code a swap entry */
607#define MAX_SWAPFILES_CHECK() do { \
608 BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS); \
609 /* \
610 * Don't have overlapping bits with _PAGE_HPTEFLAGS \
611 * We filter HPTEFLAGS on set_pte. \
612 */ \
613 BUILD_BUG_ON(_PAGE_HPTEFLAGS & (0x1f << _PAGE_BIT_SWAP_TYPE)); \
614 BUILD_BUG_ON(_PAGE_HPTEFLAGS & _PAGE_SWP_SOFT_DIRTY); \
615 } while (0)
616/*
617 * on pte we don't need handle RADIX_TREE_EXCEPTIONAL_SHIFT;
618 */
619#define SWP_TYPE_BITS 5
620#define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \
621 & ((1UL << SWP_TYPE_BITS) - 1))
622#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PAGE_SHIFT)
623#define __swp_entry(type, offset) ((swp_entry_t) { \
624 ((type) << _PAGE_BIT_SWAP_TYPE) \
625 | (((offset) << PAGE_SHIFT) & PTE_RPN_MASK)})
626/*
627 * swp_entry_t must be independent of pte bits. We build a swp_entry_t from
628 * swap type and offset we get from swap and convert that to pte to find a
629 * matching pte in linux page table.
630 * Clear bits not found in swap entries here.
631 */
632#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) & ~_PAGE_PTE })
633#define __swp_entry_to_pte(x) __pte((x).val | _PAGE_PTE)
634
635#ifdef CONFIG_MEM_SOFT_DIRTY
636#define _PAGE_SWP_SOFT_DIRTY (1UL << (SWP_TYPE_BITS + _PAGE_BIT_SWAP_TYPE))
637#else
638#define _PAGE_SWP_SOFT_DIRTY 0UL
639#endif /* CONFIG_MEM_SOFT_DIRTY */
640
641#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
642static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
643{
644 return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY);
645}
66c570f5 646
34fbadd8
AK		 647static inline bool pte_swp_soft_dirty(pte_t pte)
648{
66c570f5 649 return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY));
34fbadd8 650}
66c570f5 651
34fbadd8
AK		 652static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
653{
654 return __pte(pte_val(pte) & ~_PAGE_SWP_SOFT_DIRTY);
655}
656#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
657
658static inline bool check_pte_access(unsigned long access, unsigned long ptev)
659{
660 /*
661 * This check for _PAGE_RWX and _PAGE_PRESENT bits
662 */
663 if (access & ~ptev)
664 return false;
665 /*
666 * This check for access to privilege space
667 */
668 if ((access & _PAGE_PRIVILEGED) != (ptev & _PAGE_PRIVILEGED))
669 return false;
670
671 return true;
672}
ac94ac79
AK		 673/*
674 * Generic functions with hash/radix callbacks
675 */
676
c6d1a767 677static inline void __ptep_set_access_flags(struct mm_struct *mm,
b3603e17
AK		 678 pte_t *ptep, pte_t entry,
679 unsigned long address)
ac94ac79
AK		 680{
681 if (radix_enabled())
b3603e17 682 return radix__ptep_set_access_flags(mm, ptep, entry, address);
ac94ac79
AK		 683 return hash__ptep_set_access_flags(ptep, entry);
684}
685
686#define __HAVE_ARCH_PTE_SAME
687static inline int pte_same(pte_t pte_a, pte_t pte_b)
688{
689 if (radix_enabled())
690 return radix__pte_same(pte_a, pte_b);
691 return hash__pte_same(pte_a, pte_b);
692}
693
694static inline int pte_none(pte_t pte)
695{
696 if (radix_enabled())
697 return radix__pte_none(pte);
698 return hash__pte_none(pte);
699}
700
701static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
702 pte_t *ptep, pte_t pte, int percpu)
703{
704 if (radix_enabled())
705 return radix__set_pte_at(mm, addr, ptep, pte, percpu);
706 return hash__set_pte_at(mm, addr, ptep, pte, percpu);
707}
34fbadd8 708
13f829a5
AK		 709#define _PAGE_CACHE_CTL (_PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT)
710
711#define pgprot_noncached pgprot_noncached
712static inline pgprot_t pgprot_noncached(pgprot_t prot)
713{
714 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
715 _PAGE_NON_IDEMPOTENT);
716}
717
718#define pgprot_noncached_wc pgprot_noncached_wc
719static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
720{
721 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
722 _PAGE_TOLERANT);
723}
724
725#define pgprot_cached pgprot_cached
726static inline pgprot_t pgprot_cached(pgprot_t prot)
727{
728 return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL));
729}
730
731#define pgprot_writecombine pgprot_writecombine
732static inline pgprot_t pgprot_writecombine(pgprot_t prot)
733{
734 return pgprot_noncached_wc(prot);
735}
736/*
737 * check a pte mapping have cache inhibited property
738 */
739static inline bool pte_ci(pte_t pte)
740{
741 unsigned long pte_v = pte_val(pte);
742
743 if (((pte_v & _PAGE_CACHE_CTL) == _PAGE_TOLERANT) ||
744 ((pte_v & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT))
745 return true;
746 return false;
747}
748
f281b5d5
AK		 749static inline void pmd_set(pmd_t *pmdp, unsigned long val)
750{
751 *pmdp = __pmd(val);
752}
753
754static inline void pmd_clear(pmd_t *pmdp)
755{
756 *pmdp = __pmd(0);
757}
758
66c570f5
AK		 759static inline int pmd_none(pmd_t pmd)
760{
761 return !pmd_raw(pmd);
762}
763
764static inline int pmd_present(pmd_t pmd)
765{
766
767 return !pmd_none(pmd);
768}
3dfcb315 769
ac94ac79
AK		 770static inline int pmd_bad(pmd_t pmd)
771{
772 if (radix_enabled())
773 return radix__pmd_bad(pmd);
774 return hash__pmd_bad(pmd);
775}
776
f281b5d5
AK		 777static inline void pud_set(pud_t *pudp, unsigned long val)
778{
779 *pudp = __pud(val);
780}
781
782static inline void pud_clear(pud_t *pudp)
783{
784 *pudp = __pud(0);
785}
786
66c570f5
AK		 787static inline int pud_none(pud_t pud)
788{
789 return !pud_raw(pud);
790}
791
792static inline int pud_present(pud_t pud)
793{
794 return !pud_none(pud);
795}
3dfcb315
AK		 796
797extern struct page *pud_page(pud_t pud);
371352ca 798extern struct page *pmd_page(pmd_t pmd);
3dfcb315
AK		 799static inline pte_t pud_pte(pud_t pud)
800{
66c570f5 801 return __pte_raw(pud_raw(pud));
3dfcb315
AK		 802}
803
804static inline pud_t pte_pud(pte_t pte)
805{
66c570f5 806 return __pud_raw(pte_raw(pte));
3dfcb315
AK		 807}
808#define pud_write(pud) pte_write(pud_pte(pud))
ac94ac79
AK		 809
810static inline int pud_bad(pud_t pud)
811{
812 if (radix_enabled())
813 return radix__pud_bad(pud);
814 return hash__pud_bad(pud);
815}
816
817
3dfcb315 818#define pgd_write(pgd) pte_write(pgd_pte(pgd))
f281b5d5
AK		 819static inline void pgd_set(pgd_t *pgdp, unsigned long val)
820{
821 *pgdp = __pgd(val);
822}
3dfcb315 823
368ced78
AK		 824static inline void pgd_clear(pgd_t *pgdp)
825{
826 *pgdp = __pgd(0);
827}
828
66c570f5
AK		 829static inline int pgd_none(pgd_t pgd)
830{
831 return !pgd_raw(pgd);
832}
833
834static inline int pgd_present(pgd_t pgd)
835{
836 return !pgd_none(pgd);
837}
368ced78
AK		 838
839static inline pte_t pgd_pte(pgd_t pgd)
840{
66c570f5 841 return __pte_raw(pgd_raw(pgd));
368ced78
AK		 842}
843
844static inline pgd_t pte_pgd(pte_t pte)
845{
66c570f5 846 return __pgd_raw(pte_raw(pte));
368ced78
AK		 847}
848
ac94ac79
AK		 849static inline int pgd_bad(pgd_t pgd)
850{
851 if (radix_enabled())
852 return radix__pgd_bad(pgd);
853 return hash__pgd_bad(pgd);
854}
855
368ced78
AK		 856extern struct page *pgd_page(pgd_t pgd);
857
aba480e1
AK		 858/* Pointers in the page table tree are physical addresses */
859#define __pgtable_ptr_val(ptr) __pa(ptr)
860
861#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS)
862#define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS)
863#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS)
864
865#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
866#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1))
867#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
868#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))
869
3dfcb315
AK		 870/*
871 * Find an entry in a page-table-directory. We combine the address region
872 * (the high order N bits) and the pgd portion of the address.
873 */
3dfcb315
AK		 874
875#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
876
368ced78
AK		 877#define pud_offset(pgdp, addr) \
878 (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr))
3dfcb315 879#define pmd_offset(pudp,addr) \
371352ca 880 (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr))
3dfcb315 881#define pte_offset_kernel(dir,addr) \
371352ca 882 (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr))
3dfcb315
AK		 883
884#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
885#define pte_unmap(pte) do { } while(0)
886
887/* to find an entry in a kernel page-table-directory */
888/* This now only contains the vmalloc pages */
889#define pgd_offset_k(address) pgd_offset(&init_mm, address)
3dfcb315
AK		 890
891#define pte_ERROR(e) \
892 pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e))
893#define pmd_ERROR(e) \
894 pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e))
368ced78
AK		 895#define pud_ERROR(e) \
896 pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e))
3dfcb315
AK		 897#define pgd_ERROR(e) \
898 pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e))
899
31a14fae
AK		 900static inline int map_kernel_page(unsigned long ea, unsigned long pa,
901 unsigned long flags)
7207f436 902{
d9225ad9
AK		 903 if (radix_enabled()) {
904#if defined(CONFIG_PPC_RADIX_MMU) && defined(DEBUG_VM)
905 unsigned long page_size = 1 << mmu_psize_defs[mmu_io_psize].shift;
906 WARN((page_size != PAGE_SIZE), "I/O page size != PAGE_SIZE");
907#endif
908 return radix__map_kernel_page(ea, pa, __pgprot(flags), PAGE_SIZE);
909 }
31a14fae 910 return hash__map_kernel_page(ea, pa, flags);
7207f436 911}
31a14fae
AK		 912
913static inline int __meminit vmemmap_create_mapping(unsigned long start,
914 unsigned long page_size,
915 unsigned long phys)
7207f436 916{
d9225ad9
AK		 917 if (radix_enabled())
918 return radix__vmemmap_create_mapping(start, page_size, phys);
31a14fae 919 return hash__vmemmap_create_mapping(start, page_size, phys);
7207f436 920}
31a14fae
AK		 921
922#ifdef CONFIG_MEMORY_HOTPLUG
923static inline void vmemmap_remove_mapping(unsigned long start,
924 unsigned long page_size)
7207f436 925{
d9225ad9
AK		 926 if (radix_enabled())
927 return radix__vmemmap_remove_mapping(start, page_size);
31a14fae 928 return hash__vmemmap_remove_mapping(start, page_size);
7207f436 929}
31a14fae 930#endif
3dfcb315
AK		 931struct page *realmode_pfn_to_page(unsigned long pfn);
932
3dfcb315
AK		 933static inline pte_t pmd_pte(pmd_t pmd)
934{
66c570f5 935 return __pte_raw(pmd_raw(pmd));
3dfcb315
AK		 936}
937
938static inline pmd_t pte_pmd(pte_t pte)
939{
66c570f5 940 return __pmd_raw(pte_raw(pte));
3dfcb315
AK		 941}
942
943static inline pte_t *pmdp_ptep(pmd_t *pmd)
944{
945 return (pte_t *)pmd;
946}
3dfcb315
AK		 947#define pmd_pfn(pmd) pte_pfn(pmd_pte(pmd))
948#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
949#define pmd_young(pmd) pte_young(pmd_pte(pmd))
950#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
951#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
952#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
d5d6a443 953#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
3dfcb315
AK		 954#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
955#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
c137a275
AK		 956#define pmd_mk_savedwrite(pmd) pte_pmd(pte_mk_savedwrite(pmd_pte(pmd)))
957#define pmd_clear_savedwrite(pmd) pte_pmd(pte_clear_savedwrite(pmd_pte(pmd)))
7207f436
LD		 958
959#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
960#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
961#define pmd_mksoft_dirty(pmd) pte_pmd(pte_mksoft_dirty(pmd_pte(pmd)))
962#define pmd_clear_soft_dirty(pmd) pte_pmd(pte_clear_soft_dirty(pmd_pte(pmd)))
963#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
964
1ca72129
AK		 965#ifdef CONFIG_NUMA_BALANCING
966static inline int pmd_protnone(pmd_t pmd)
967{
968 return pte_protnone(pmd_pte(pmd));
969}
970#endif /* CONFIG_NUMA_BALANCING */
3dfcb315
AK		 971
972#define __HAVE_ARCH_PMD_WRITE
973#define pmd_write(pmd) pte_write(pmd_pte(pmd))
d19469e8 974#define __pmd_write(pmd) __pte_write(pmd_pte(pmd))
c137a275 975#define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd))
3dfcb315 976
6a1ea362
AK		 977#ifdef CONFIG_TRANSPARENT_HUGEPAGE
978extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
979extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
980extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
981extern void set_pmd_at(struct mm_struct *mm, unsigned long addr,
982 pmd_t *pmdp, pmd_t pmd);
983extern void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
984 pmd_t *pmd);
3df33f12
AK		 985extern int hash__has_transparent_hugepage(void);
986static inline int has_transparent_hugepage(void)
987{
bde3eb62
AK		 988 if (radix_enabled())
989 return radix__has_transparent_hugepage();
3df33f12
AK		 990 return hash__has_transparent_hugepage();
991}
c04a5880 992#define has_transparent_hugepage has_transparent_hugepage
6a1ea362 993
3df33f12
AK		 994static inline unsigned long
995pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp,
996 unsigned long clr, unsigned long set)
3dfcb315 997{
bde3eb62
AK		 998 if (radix_enabled())
999 return radix__pmd_hugepage_update(mm, addr, pmdp, clr, set);
3df33f12
AK		 1000 return hash__pmd_hugepage_update(mm, addr, pmdp, clr, set);
1001}
1002
1003static inline int pmd_large(pmd_t pmd)
1004{
66c570f5 1005 return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE));
3df33f12
AK		 1006}
1007
1008static inline pmd_t pmd_mknotpresent(pmd_t pmd)
1009{
1010 return __pmd(pmd_val(pmd) & ~_PAGE_PRESENT);
1011}
1012/*
1013 * For radix we should always find H_PAGE_HASHPTE zero. Hence
1014 * the below will work for radix too
1015 */
1016static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
1017 unsigned long addr, pmd_t *pmdp)
1018{
1019 unsigned long old;
1020
66c570f5 1021 if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0)
3df33f12
AK		 1022 return 0;
1023 old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
1024 return ((old & _PAGE_ACCESSED) != 0);
1025}
1026
1027#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1028static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
1029 pmd_t *pmdp)
1030{
d19469e8 1031 if (__pmd_write((*pmdp)))
52c50ca7
AK		 1032 pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0);
1033 else if (unlikely(pmd_savedwrite(*pmdp)))
1034 pmd_hugepage_update(mm, addr, pmdp, 0, _PAGE_PRIVILEGED);
3dfcb315
AK		 1035}
1036
ab624762
AK		 1037static inline int pmd_trans_huge(pmd_t pmd)
1038{
1039 if (radix_enabled())
1040 return radix__pmd_trans_huge(pmd);
1041 return hash__pmd_trans_huge(pmd);
1042}
1043
1044#define __HAVE_ARCH_PMD_SAME
1045static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b)
1046{
1047 if (radix_enabled())
1048 return radix__pmd_same(pmd_a, pmd_b);
1049 return hash__pmd_same(pmd_a, pmd_b);
1050}
1051
3dfcb315
AK		 1052static inline pmd_t pmd_mkhuge(pmd_t pmd)
1053{
ab624762
AK		 1054 if (radix_enabled())
1055 return radix__pmd_mkhuge(pmd);
1056 return hash__pmd_mkhuge(pmd);
3dfcb315
AK		 1057}
1058
3dfcb315
AK		 1059#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1060extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1061 unsigned long address, pmd_t *pmdp,
1062 pmd_t entry, int dirty);
1063
3dfcb315
AK		 1064#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1065extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1066 unsigned long address, pmd_t *pmdp);
3dfcb315
AK		 1067
1068#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
3df33f12
AK		 1069static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1070 unsigned long addr, pmd_t *pmdp)
1071{
bde3eb62
AK		 1072 if (radix_enabled())
1073 return radix__pmdp_huge_get_and_clear(mm, addr, pmdp);
3df33f12
AK		 1074 return hash__pmdp_huge_get_and_clear(mm, addr, pmdp);
1075}
3dfcb315 1076
3df33f12
AK		 1077static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
1078 unsigned long address, pmd_t *pmdp)
1079{
bde3eb62
AK		 1080 if (radix_enabled())
1081 return radix__pmdp_collapse_flush(vma, address, pmdp);
3df33f12
AK		 1082 return hash__pmdp_collapse_flush(vma, address, pmdp);
1083}
3dfcb315
AK		 1084#define pmdp_collapse_flush pmdp_collapse_flush
1085
1086#define __HAVE_ARCH_PGTABLE_DEPOSIT
3df33f12
AK		 1087static inline void pgtable_trans_huge_deposit(struct mm_struct *mm,
1088 pmd_t *pmdp, pgtable_t pgtable)
1089{
bde3eb62
AK		 1090 if (radix_enabled())
1091 return radix__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
3df33f12
AK		 1092 return hash__pgtable_trans_huge_deposit(mm, pmdp, pgtable);
1093}
1094
3dfcb315 1095#define __HAVE_ARCH_PGTABLE_WITHDRAW
3df33f12
AK		 1096static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
1097 pmd_t *pmdp)
1098{
bde3eb62
AK		 1099 if (radix_enabled())
1100 return radix__pgtable_trans_huge_withdraw(mm, pmdp);
3df33f12
AK		 1101 return hash__pgtable_trans_huge_withdraw(mm, pmdp);
1102}
3dfcb315
AK		 1103
1104#define __HAVE_ARCH_PMDP_INVALIDATE
1105extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
1106 pmd_t *pmdp);
1107
c777e2a8 1108#define __HAVE_ARCH_PMDP_HUGE_SPLIT_PREPARE
3df33f12
AK		 1109static inline void pmdp_huge_split_prepare(struct vm_area_struct *vma,
1110 unsigned long address, pmd_t *pmdp)
1111{
bde3eb62
AK		 1112 if (radix_enabled())
1113 return radix__pmdp_huge_split_prepare(vma, address, pmdp);
3df33f12
AK		 1114 return hash__pmdp_huge_split_prepare(vma, address, pmdp);
1115}
c777e2a8 1116
3dfcb315
AK		 1117#define pmd_move_must_withdraw pmd_move_must_withdraw
1118struct spinlock;
1119static inline int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl,
1dd38b6c
AK		 1120 struct spinlock *old_pmd_ptl,
1121 struct vm_area_struct *vma)
3dfcb315 1122{
bde3eb62
AK		 1123 if (radix_enabled())
1124 return false;
3dfcb315
AK		 1125 /*
1126 * Archs like ppc64 use pgtable to store per pmd
1127 * specific information. So when we switch the pmd,
1128 * we should also withdraw and deposit the pgtable
1129 */
1130 return true;
1131}
953c66c2
AK		 1132
1133
1134#define arch_needs_pgtable_deposit arch_needs_pgtable_deposit
1135static inline bool arch_needs_pgtable_deposit(void)
1136{
1137 if (radix_enabled())
1138 return false;
1139 return true;
1140}
1141
6a1ea362 1142#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
3dfcb315
AK		 1143#endif /* __ASSEMBLY__ */
1144#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
Linux 6.x block layer and io_uring tree(s)