[linux-2.6-block.git] / arch / powerpc / include / asm / book3s / 64 / hash.h

#ifndef _ASM_POWERPC_BOOK3S_64_HASH_H
#define _ASM_POWERPC_BOOK3S_64_HASH_H
#ifdef __KERNEL__

/*
 * Common bits between 4K and 64K pages in a linux-style PTE.
 * These match the bits in the (hardware-defined) PowerPC PTE as closely
 * as possible. Additional bits may be defined in pgtable-hash64-*.h
 *
 * Note: We only support user read/write permissions. Supervisor always
 * have full read/write to pages above PAGE_OFFSET (pages below that
 * always use the user access permissions).
 *
 * We could create separate kernel read-only if we used the 3 PP bits
 * combinations that newer processors provide but we currently don't.
 */
#define _PAGE_PTE		0x00001
#define _PAGE_PRESENT		0x00002 /* software: pte contains a translation */
#define _PAGE_BIT_SWAP_TYPE	2
#define _PAGE_USER		0x00004 /* matches one of the PP bits */
#define _PAGE_EXEC		0x00008 /* No execute on POWER4 and newer (we invert) */
#define _PAGE_GUARDED		0x00010
/* We can derive Memory coherence from _PAGE_NO_CACHE */
#define _PAGE_COHERENT		0x0
#define _PAGE_NO_CACHE		0x00020 /* I: cache inhibit */
#define _PAGE_WRITETHRU		0x00040 /* W: cache write-through */
#define _PAGE_DIRTY		0x00080 /* C: page changed */
#define _PAGE_ACCESSED		0x00100 /* R: page referenced */
#define _PAGE_RW		0x00200 /* software: user write access allowed */
#define _PAGE_HASHPTE		0x00400 /* software: pte has an associated HPTE */
#define _PAGE_BUSY		0x00800 /* software: PTE & hash are busy */
#define _PAGE_F_GIX		0x07000 /* full page: hidx bits */
#define _PAGE_F_GIX_SHIFT	12
#define _PAGE_F_SECOND		0x08000 /* Whether to use secondary hash or not */
#define _PAGE_SPECIAL		0x10000 /* software: special page */

#ifdef CONFIG_MEM_SOFT_DIRTY
#define _PAGE_SOFT_DIRTY	0x20000 /* software: software dirty tracking */
#else
#define _PAGE_SOFT_DIRTY	0x00000
#endif

/*
 * We need to differentiate between explicit huge page and THP huge
 * page, since THP huge page also need to track real subpage details
 */
#define _PAGE_THP_HUGE  _PAGE_4K_PFN

/*
 * set of bits not changed in pmd_modify.
 */
#define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
			 _PAGE_ACCESSED | _PAGE_THP_HUGE | _PAGE_PTE | \
			 _PAGE_SOFT_DIRTY)


#ifdef CONFIG_PPC_64K_PAGES
#include <asm/book3s/64/hash-64k.h>
#else
#include <asm/book3s/64/hash-4k.h>
#endif

/*
 * Size of EA range mapped by our pagetables.
 */
#define PGTABLE_EADDR_SIZE	(PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
				 PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)
#define PGTABLE_RANGE		(ASM_CONST(1) << PGTABLE_EADDR_SIZE)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define PMD_CACHE_INDEX	(PMD_INDEX_SIZE + 1)
#else
#define PMD_CACHE_INDEX	PMD_INDEX_SIZE
#endif
/*
 * Define the address range of the kernel non-linear virtual area
 */
#define KERN_VIRT_START ASM_CONST(0xD000000000000000)
#define KERN_VIRT_SIZE	ASM_CONST(0x0000100000000000)

/*
 * The vmalloc space starts at the beginning of that region, and
 * occupies half of it on hash CPUs and a quarter of it on Book3E
 * (we keep a quarter for the virtual memmap)
 */
#define VMALLOC_START	KERN_VIRT_START
#define VMALLOC_SIZE	(KERN_VIRT_SIZE >> 1)
#define VMALLOC_END	(VMALLOC_START + VMALLOC_SIZE)

/*
 * Region IDs
 */
#define REGION_SHIFT		60UL
#define REGION_MASK		(0xfUL << REGION_SHIFT)
#define REGION_ID(ea)		(((unsigned long)(ea)) >> REGION_SHIFT)

#define VMALLOC_REGION_ID	(REGION_ID(VMALLOC_START))
#define KERNEL_REGION_ID	(REGION_ID(PAGE_OFFSET))
#define VMEMMAP_REGION_ID	(0xfUL)	/* Server only */
#define USER_REGION_ID		(0UL)

/*
 * Defines the address of the vmemap area, in its own region on
 * hash table CPUs.
 */
#define VMEMMAP_BASE		(VMEMMAP_REGION_ID << REGION_SHIFT)

#ifdef CONFIG_PPC_MM_SLICES
#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
#endif /* CONFIG_PPC_MM_SLICES */

/* No separate kernel read-only */
#define _PAGE_KERNEL_RW		(_PAGE_RW | _PAGE_DIRTY) /* user access blocked by key */
#define _PAGE_KERNEL_RO		 _PAGE_KERNEL_RW
#define _PAGE_KERNEL_RWX	(_PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC)

/* Strong Access Ordering */
#define _PAGE_SAO		(_PAGE_WRITETHRU | _PAGE_NO_CACHE | _PAGE_COHERENT)

/* No page size encoding in the linux PTE */
#define _PAGE_PSIZE		0

/* PTEIDX nibble */
#define _PTEIDX_SECONDARY	0x8
#define _PTEIDX_GROUP_IX	0x7

/* Hash table based platforms need atomic updates of the linux PTE */
#define PTE_ATOMIC_UPDATES	1
#define _PTE_NONE_MASK	_PAGE_HPTEFLAGS
/*
 * The mask convered by the RPN must be a ULL on 32-bit platforms with
 * 64-bit PTEs
 */
#define PTE_RPN_MASK	(~((1UL << PTE_RPN_SHIFT) - 1))
/*
 * _PAGE_CHG_MASK masks of bits that are to be preserved across
 * pgprot changes
 */
#define _PAGE_CHG_MASK	(PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
			 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \
			 _PAGE_SOFT_DIRTY)
/*
 * Mask of bits returned by pte_pgprot()
 */
#define PAGE_PROT_BITS	(_PAGE_GUARDED | _PAGE_COHERENT | _PAGE_NO_CACHE | \
			 _PAGE_WRITETHRU | _PAGE_4K_PFN | \
			 _PAGE_USER | _PAGE_ACCESSED |  \
			 _PAGE_RW |  _PAGE_DIRTY | _PAGE_EXEC | \
			 _PAGE_SOFT_DIRTY)
/*
 * We define 2 sets of base prot bits, one for basic pages (ie,
 * cacheable kernel and user pages) and one for non cacheable
 * pages. We always set _PAGE_COHERENT when SMP is enabled or
 * the processor might need it for DMA coherency.
 */
#define _PAGE_BASE_NC	(_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE)
#define _PAGE_BASE	(_PAGE_BASE_NC | _PAGE_COHERENT)

/* Permission masks used to generate the __P and __S table,
 *
 * Note:__pgprot is defined in arch/powerpc/include/asm/page.h
 *
 * Write permissions imply read permissions for now (we could make write-only
 * pages on BookE but we don't bother for now). Execute permission control is
 * possible on platforms that define _PAGE_EXEC
 *
 * Note due to the way vm flags are laid out, the bits are XWR
 */
#define PAGE_NONE	__pgprot(_PAGE_BASE)
#define PAGE_SHARED	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW)
#define PAGE_SHARED_X	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | \
				 _PAGE_EXEC)
#define PAGE_COPY	__pgprot(_PAGE_BASE | _PAGE_USER )
#define PAGE_COPY_X	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC)
#define PAGE_READONLY	__pgprot(_PAGE_BASE | _PAGE_USER )
#define PAGE_READONLY_X	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC)

#define __P000	PAGE_NONE
#define __P001	PAGE_READONLY
#define __P010	PAGE_COPY
#define __P011	PAGE_COPY
#define __P100	PAGE_READONLY_X
#define __P101	PAGE_READONLY_X
#define __P110	PAGE_COPY_X
#define __P111	PAGE_COPY_X

#define __S000	PAGE_NONE
#define __S001	PAGE_READONLY
#define __S010	PAGE_SHARED
#define __S011	PAGE_SHARED
#define __S100	PAGE_READONLY_X
#define __S101	PAGE_READONLY_X
#define __S110	PAGE_SHARED_X
#define __S111	PAGE_SHARED_X

/* Permission masks used for kernel mappings */
#define PAGE_KERNEL	__pgprot(_PAGE_BASE | _PAGE_KERNEL_RW)
#define PAGE_KERNEL_NC	__pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
				 _PAGE_NO_CACHE)
#define PAGE_KERNEL_NCG	__pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \
				 _PAGE_NO_CACHE | _PAGE_GUARDED)
#define PAGE_KERNEL_X	__pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX)
#define PAGE_KERNEL_RO	__pgprot(_PAGE_BASE | _PAGE_KERNEL_RO)
#define PAGE_KERNEL_ROX	__pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX)

/* Protection used for kernel text. We want the debuggers to be able to
 * set breakpoints anywhere, so don't write protect the kernel text
 * on platforms where such control is possible.
 */
#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\
	defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE)
#define PAGE_KERNEL_TEXT	PAGE_KERNEL_X
#else
#define PAGE_KERNEL_TEXT	PAGE_KERNEL_ROX
#endif

/* Make modules code happy. We don't set RO yet */
#define PAGE_KERNEL_EXEC	PAGE_KERNEL_X
#define PAGE_AGP		(PAGE_KERNEL_NC)

#define PMD_BAD_BITS		(PTE_TABLE_SIZE-1)
#define PUD_BAD_BITS		(PMD_TABLE_SIZE-1)

#ifndef __ASSEMBLY__
#define	pmd_bad(pmd)		(!is_kernel_addr(pmd_val(pmd)) \
				 || (pmd_val(pmd) & PMD_BAD_BITS))
#define pmd_page_vaddr(pmd)	(pmd_val(pmd) & ~PMD_MASKED_BITS)

#define	pud_bad(pud)		(!is_kernel_addr(pud_val(pud)) \
				 || (pud_val(pud) & PUD_BAD_BITS))
#define pud_page_vaddr(pud)	(pud_val(pud) & ~PUD_MASKED_BITS)

#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))

extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
			    pte_t *ptep, unsigned long pte, int huge);
extern unsigned long htab_convert_pte_flags(unsigned long pteflags);
/* Atomic PTE updates */
static inline unsigned long pte_update(struct mm_struct *mm,
				       unsigned long addr,
				       pte_t *ptep, unsigned long clr,
				       unsigned long set,
				       int huge)
{
	unsigned long old, tmp;

	__asm__ __volatile__(
	"1:	ldarx	%0,0,%3		# pte_update\n\
	andi.	%1,%0,%6\n\
	bne-	1b \n\
	andc	%1,%0,%4 \n\
	or	%1,%1,%7\n\
	stdcx.	%1,0,%3 \n\
	bne-	1b"
	: "=&r" (old), "=&r" (tmp), "=m" (*ptep)
	: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
	: "cc" );
	/* huge pages use the old page table lock */
	if (!huge)
		assert_pte_locked(mm, addr);

	if (old & _PAGE_HASHPTE)
		hpte_need_flush(mm, addr, ptep, old, huge);

	return old;
}

static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
					      unsigned long addr, pte_t *ptep)
{
	unsigned long old;

	if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
		return 0;
	old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
	return (old & _PAGE_ACCESSED) != 0;
}
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
#define ptep_test_and_clear_young(__vma, __addr, __ptep)		   \
({									   \
	int __r;							   \
	__r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
	__r;								   \
})

#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
				      pte_t *ptep)
{

	if ((pte_val(*ptep) & _PAGE_RW) == 0)
		return;

	pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
}

static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
					   unsigned long addr, pte_t *ptep)
{
	if ((pte_val(*ptep) & _PAGE_RW) == 0)
		return;

	pte_update(mm, addr, ptep, _PAGE_RW, 0, 1);
}

/*
 * We currently remove entries from the hashtable regardless of whether
 * the entry was young or dirty. The generic routines only flush if the
 * entry was young or dirty which is not good enough.
 *
 * We should be more intelligent about this but for the moment we override
 * these functions and force a tlb flush unconditionally
 */
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
#define ptep_clear_flush_young(__vma, __address, __ptep)		\
({									\
	int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \
						  __ptep);		\
	__young;							\
})

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
				       unsigned long addr, pte_t *ptep)
{
	unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
	return __pte(old);
}

static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
			     pte_t * ptep)
{
	pte_update(mm, addr, ptep, ~0UL, 0, 0);
}


/* Set the dirty and/or accessed bits atomically in a linux PTE, this
 * function doesn't need to flush the hash entry
 */
static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)
{
	unsigned long bits = pte_val(entry) &
		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC |
		 _PAGE_SOFT_DIRTY);

	unsigned long old, tmp;

	__asm__ __volatile__(
	"1:	ldarx	%0,0,%4\n\
		andi.	%1,%0,%6\n\
		bne-	1b \n\
		or	%0,%3,%0\n\
		stdcx.	%0,0,%4\n\
		bne-	1b"
	:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
	:"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY)
	:"cc");
}

#define __HAVE_ARCH_PTE_SAME
#define pte_same(A,B)	(((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)

/* Generic accessors to PTE bits */
static inline int pte_write(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_RW);}
static inline int pte_dirty(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_DIRTY); }
static inline int pte_young(pte_t pte)		{ return !!(pte_val(pte) & _PAGE_ACCESSED); }
static inline int pte_special(pte_t pte)	{ return !!(pte_val(pte) & _PAGE_SPECIAL); }
static inline int pte_none(pte_t pte)		{ return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; }
static inline pgprot_t pte_pgprot(pte_t pte)	{ return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }

#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
static inline bool pte_soft_dirty(pte_t pte)
{
	return !!(pte_val(pte) & _PAGE_SOFT_DIRTY);
}
static inline pte_t pte_mksoft_dirty(pte_t pte)
{
	return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY);
}

static inline pte_t pte_clear_soft_dirty(pte_t pte)
{
	return __pte(pte_val(pte) & ~_PAGE_SOFT_DIRTY);
}
#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */

#ifdef CONFIG_NUMA_BALANCING
/*
 * These work without NUMA balancing but the kernel does not care. See the
 * comment in include/asm-generic/pgtable.h . On powerpc, this will only
 * work for user pages and always return true for kernel pages.
 */
static inline int pte_protnone(pte_t pte)
{
	return (pte_val(pte) &
		(_PAGE_PRESENT | _PAGE_USER)) == _PAGE_PRESENT;
}
#endif /* CONFIG_NUMA_BALANCING */

static inline int pte_present(pte_t pte)
{
	return pte_val(pte) & _PAGE_PRESENT;
}

/* Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 *
 * Even if PTEs can be unsigned long long, a PFN is always an unsigned
 * long for now.
 */
static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
{
	return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) |
		     pgprot_val(pgprot));
}

static inline unsigned long pte_pfn(pte_t pte)
{
	return pte_val(pte) >> PTE_RPN_SHIFT;
}

/* Generic modifiers for PTE bits */
static inline pte_t pte_wrprotect(pte_t pte)
{
	return __pte(pte_val(pte) & ~_PAGE_RW);
}

static inline pte_t pte_mkclean(pte_t pte)
{
	return __pte(pte_val(pte) & ~_PAGE_DIRTY);
}

static inline pte_t pte_mkold(pte_t pte)
{
	return __pte(pte_val(pte) & ~_PAGE_ACCESSED);
}

static inline pte_t pte_mkwrite(pte_t pte)
{
	return __pte(pte_val(pte) | _PAGE_RW);
}

static inline pte_t pte_mkdirty(pte_t pte)
{
	return __pte(pte_val(pte) | _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
}

static inline pte_t pte_mkyoung(pte_t pte)
{
	return __pte(pte_val(pte) | _PAGE_ACCESSED);
}

static inline pte_t pte_mkspecial(pte_t pte)
{
	return __pte(pte_val(pte) | _PAGE_SPECIAL);
}

static inline pte_t pte_mkhuge(pte_t pte)
{
	return pte;
}

static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
	return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot));
}

/* This low level function performs the actual PTE insertion
 * Setting the PTE depends on the MMU type and other factors. It's
 * an horrible mess that I'm not going to try to clean up now but
 * I'm keeping it in one place rather than spread around
 */
static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
				pte_t *ptep, pte_t pte, int percpu)
{
	/*
	 * Anything else just stores the PTE normally. That covers all 64-bit
	 * cases, and 32-bit non-hash with 32-bit PTEs.
	 */
	*ptep = pte;
}

/*
 * Macro to mark a page protection value as "uncacheable".
 */

#define _PAGE_CACHE_CTL	(_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \
			 _PAGE_WRITETHRU)

#define pgprot_noncached pgprot_noncached
static inline pgprot_t pgprot_noncached(pgprot_t prot)
{
	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
			_PAGE_NO_CACHE | _PAGE_GUARDED);
}

#define pgprot_noncached_wc pgprot_noncached_wc
static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
{
	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
			_PAGE_NO_CACHE);
}

#define pgprot_cached pgprot_cached
static inline pgprot_t pgprot_cached(pgprot_t prot)
{
	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
			_PAGE_COHERENT);
}

#define pgprot_cached_wthru pgprot_cached_wthru
static inline pgprot_t pgprot_cached_wthru(pgprot_t prot)
{
	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) |
			_PAGE_COHERENT | _PAGE_WRITETHRU);
}

#define pgprot_cached_noncoherent pgprot_cached_noncoherent
static inline pgprot_t pgprot_cached_noncoherent(pgprot_t prot)
{
	return __pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL);
}

#define pgprot_writecombine pgprot_writecombine
static inline pgprot_t pgprot_writecombine(pgprot_t prot)
{
	return pgprot_noncached_wc(prot);
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
				   pmd_t *pmdp, unsigned long old_pmd);
#else
static inline void hpte_do_hugepage_flush(struct mm_struct *mm,
					  unsigned long addr, pmd_t *pmdp,
					  unsigned long old_pmd)
{
	WARN(1, "%s called with THP disabled\n", __func__);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* !__ASSEMBLY__ */
#endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */
Commit	Line	Data
26b6a3d9 AK	1	#ifndef _ASM_POWERPC_BOOK3S_64_HASH_H
26b6a3d9 AK	2	#define _ASM_POWERPC_BOOK3S_64_HASH_H
c605782b BH	3	#ifdef __KERNEL__
c605782b BH	4
e34aa03c AK	5	/*
	6	* Common bits between 4K and 64K pages in a linux-style PTE.
	7	* These match the bits in the (hardware-defined) PowerPC PTE as closely
	8	* as possible. Additional bits may be defined in pgtable-hash64-*.h
	9	*
	10	* Note: We only support user read/write permissions. Supervisor always
	11	* have full read/write to pages above PAGE_OFFSET (pages below that
	12	* always use the user access permissions).
	13	*
	14	* We could create separate kernel read-only if we used the 3 PP bits
	15	* combinations that newer processors provide but we currently don't.
	16	*/
6a119eae AK	17	#define _PAGE_PTE 0x00001
6a119eae AK	18	#define _PAGE_PRESENT 0x00002 /* software: pte contains a translation */
e34aa03c	19	#define _PAGE_BIT_SWAP_TYPE 2
6a119eae AK	20	#define _PAGE_USER 0x00004 /* matches one of the PP bits */
	21	#define _PAGE_EXEC 0x00008 /* No execute on POWER4 and newer (we invert) */
	22	#define _PAGE_GUARDED 0x00010
e34aa03c AK	23	/* We can derive Memory coherence from _PAGE_NO_CACHE */
	24	#define _PAGE_COHERENT 0x0
	25	#define _PAGE_NO_CACHE 0x00020 /* I: cache inhibit */
	26	#define _PAGE_WRITETHRU 0x00040 /* W: cache write-through */
	27	#define _PAGE_DIRTY 0x00080 /* C: page changed */
	28	#define _PAGE_ACCESSED 0x00100 /* R: page referenced */
	29	#define _PAGE_RW 0x00200 /* software: user write access allowed */
	30	#define _PAGE_HASHPTE 0x00400 /* software: pte has an associated HPTE */
	31	#define _PAGE_BUSY 0x00800 /* software: PTE & hash are busy */
	32	#define _PAGE_F_GIX 0x07000 /* full page: hidx bits */
	33	#define _PAGE_F_GIX_SHIFT 12
	34	#define _PAGE_F_SECOND 0x08000 /* Whether to use secondary hash or not */
	35	#define _PAGE_SPECIAL 0x10000 /* software: special page */
2f10f1a7 HD	36
2f10f1a7 HD	37	#ifdef CONFIG_MEM_SOFT_DIRTY
7207f436	38	#define _PAGE_SOFT_DIRTY 0x20000 /* software: software dirty tracking */
2f10f1a7 HD	39	#else
	40	#define _PAGE_SOFT_DIRTY 0x00000
	41	#endif
e34aa03c	42
e34aa03c AK	43	/*
	44	* We need to differentiate between explicit huge page and THP huge
	45	* page, since THP huge page also need to track real subpage details
	46	*/
	47	#define _PAGE_THP_HUGE _PAGE_4K_PFN
	48
	49	/*
	50	* set of bits not changed in pmd_modify.
	51	*/
7aa9a23c	52	#define _HPAGE_CHG_MASK (PTE_RPN_MASK \| _PAGE_HPTEFLAGS \| _PAGE_DIRTY \| \
2d19fc63 AK	53	_PAGE_ACCESSED \| _PAGE_THP_HUGE \| _PAGE_PTE \| \
	54	_PAGE_SOFT_DIRTY)
	55
e34aa03c	56
371352ca AK	57	#ifdef CONFIG_PPC_64K_PAGES
	58	#include <asm/book3s/64/hash-64k.h>
	59	#else
	60	#include <asm/book3s/64/hash-4k.h>
	61	#endif
	62
	63	/*
	64	* Size of EA range mapped by our pagetables.
	65	*/
	66	#define PGTABLE_EADDR_SIZE (PTE_INDEX_SIZE + PMD_INDEX_SIZE + \
	67	PUD_INDEX_SIZE + PGD_INDEX_SIZE + PAGE_SHIFT)
	68	#define PGTABLE_RANGE (ASM_CONST(1) << PGTABLE_EADDR_SIZE)
	69
	70	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	71	#define PMD_CACHE_INDEX (PMD_INDEX_SIZE + 1)
	72	#else
	73	#define PMD_CACHE_INDEX PMD_INDEX_SIZE
	74	#endif
	75	/*
	76	* Define the address range of the kernel non-linear virtual area
	77	*/
	78	#define KERN_VIRT_START ASM_CONST(0xD000000000000000)
	79	#define KERN_VIRT_SIZE ASM_CONST(0x0000100000000000)
	80
	81	/*
	82	* The vmalloc space starts at the beginning of that region, and
	83	* occupies half of it on hash CPUs and a quarter of it on Book3E
	84	* (we keep a quarter for the virtual memmap)
	85	*/
	86	#define VMALLOC_START KERN_VIRT_START
	87	#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
	88	#define VMALLOC_END (VMALLOC_START + VMALLOC_SIZE)
	89
	90	/*
	91	* Region IDs
	92	*/
	93	#define REGION_SHIFT 60UL
	94	#define REGION_MASK (0xfUL << REGION_SHIFT)
	95	#define REGION_ID(ea) (((unsigned long)(ea)) >> REGION_SHIFT)
	96
	97	#define VMALLOC_REGION_ID (REGION_ID(VMALLOC_START))
	98	#define KERNEL_REGION_ID (REGION_ID(PAGE_OFFSET))
	99	#define VMEMMAP_REGION_ID (0xfUL) /* Server only */
	100	#define USER_REGION_ID (0UL)
	101
	102	/*
	103	* Defines the address of the vmemap area, in its own region on
	104	* hash table CPUs.
	105	*/
	106	#define VMEMMAP_BASE (VMEMMAP_REGION_ID << REGION_SHIFT)
	107
	108	#ifdef CONFIG_PPC_MM_SLICES
	109	#define HAVE_ARCH_UNMAPPED_AREA
	110	#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
	111	#endif /* CONFIG_PPC_MM_SLICES */
8d1cf34e BH	112
	113	/* No separate kernel read-only */
	114	#define _PAGE_KERNEL_RW (_PAGE_RW \| _PAGE_DIRTY) /* user access blocked by key */
	115	#define _PAGE_KERNEL_RO _PAGE_KERNEL_RW
b0412ea9	116	#define _PAGE_KERNEL_RWX (_PAGE_DIRTY \| _PAGE_RW \| _PAGE_EXEC)
c605782b BH	117
c605782b BH	118	/* Strong Access Ordering */
8d1cf34e	119	#define _PAGE_SAO (_PAGE_WRITETHRU \| _PAGE_NO_CACHE \| _PAGE_COHERENT)
c605782b	120
8d1cf34e BH	121	/* No page size encoding in the linux PTE */
8d1cf34e BH	122	#define _PAGE_PSIZE 0
c605782b BH	123
	124	/* PTEIDX nibble */
	125	#define _PTEIDX_SECONDARY 0x8
	126	#define _PTEIDX_GROUP_IX 0x7
	127
a033a487 BH	128	/* Hash table based platforms need atomic updates of the linux PTE */
a033a487 BH	129	#define PTE_ATOMIC_UPDATES 1
371352ca AK	130	#define _PTE_NONE_MASK _PAGE_HPTEFLAGS
	131	/*
	132	* The mask convered by the RPN must be a ULL on 32-bit platforms with
	133	* 64-bit PTEs
	134	*/
	135	#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1))
	136	/*
	137	* _PAGE_CHG_MASK masks of bits that are to be preserved across
	138	* pgprot changes
	139	*/
	140	#define _PAGE_CHG_MASK (PTE_RPN_MASK \| _PAGE_HPTEFLAGS \| _PAGE_DIRTY \| \
7207f436 LD	141	_PAGE_ACCESSED \| _PAGE_SPECIAL \| _PAGE_PTE \| \
7207f436 LD	142	_PAGE_SOFT_DIRTY)
371352ca AK	143	/*
	144	* Mask of bits returned by pte_pgprot()
	145	*/
	146	#define PAGE_PROT_BITS (_PAGE_GUARDED \| _PAGE_COHERENT \| _PAGE_NO_CACHE \| \
	147	_PAGE_WRITETHRU \| _PAGE_4K_PFN \| \
	148	_PAGE_USER \| _PAGE_ACCESSED \| \
7207f436 LD	149	_PAGE_RW \| _PAGE_DIRTY \| _PAGE_EXEC \| \
7207f436 LD	150	_PAGE_SOFT_DIRTY)
371352ca AK	151	/*
	152	* We define 2 sets of base prot bits, one for basic pages (ie,
	153	* cacheable kernel and user pages) and one for non cacheable
	154	* pages. We always set _PAGE_COHERENT when SMP is enabled or
	155	* the processor might need it for DMA coherency.
	156	*/
	157	#define _PAGE_BASE_NC (_PAGE_PRESENT \| _PAGE_ACCESSED \| _PAGE_PSIZE)
	158	#define _PAGE_BASE (_PAGE_BASE_NC \| _PAGE_COHERENT)
	159
	160	/* Permission masks used to generate the __P and __S table,
	161	*
	162	* Note:__pgprot is defined in arch/powerpc/include/asm/page.h
	163	*
	164	* Write permissions imply read permissions for now (we could make write-only
	165	* pages on BookE but we don't bother for now). Execute permission control is
	166	* possible on platforms that define _PAGE_EXEC
	167	*
	168	* Note due to the way vm flags are laid out, the bits are XWR
	169	*/
	170	#define PAGE_NONE __pgprot(_PAGE_BASE)
	171	#define PAGE_SHARED __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_RW)
	172	#define PAGE_SHARED_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_RW \| \
	173	_PAGE_EXEC)
	174	#define PAGE_COPY __pgprot(_PAGE_BASE \| _PAGE_USER )
	175	#define PAGE_COPY_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_EXEC)
	176	#define PAGE_READONLY __pgprot(_PAGE_BASE \| _PAGE_USER )
	177	#define PAGE_READONLY_X __pgprot(_PAGE_BASE \| _PAGE_USER \| _PAGE_EXEC)
	178
	179	#define __P000 PAGE_NONE
	180	#define __P001 PAGE_READONLY
	181	#define __P010 PAGE_COPY
	182	#define __P011 PAGE_COPY
	183	#define __P100 PAGE_READONLY_X
	184	#define __P101 PAGE_READONLY_X
	185	#define __P110 PAGE_COPY_X
	186	#define __P111 PAGE_COPY_X
	187
	188	#define __S000 PAGE_NONE
	189	#define __S001 PAGE_READONLY
	190	#define __S010 PAGE_SHARED
	191	#define __S011 PAGE_SHARED
	192	#define __S100 PAGE_READONLY_X
	193	#define __S101 PAGE_READONLY_X
	194	#define __S110 PAGE_SHARED_X
	195	#define __S111 PAGE_SHARED_X
	196
	197	/* Permission masks used for kernel mappings */
	198	#define PAGE_KERNEL __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RW)
	199	#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC \| _PAGE_KERNEL_RW \| \
	200	_PAGE_NO_CACHE)
	201	#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC \| _PAGE_KERNEL_RW \| \
	202	_PAGE_NO_CACHE \| _PAGE_GUARDED)
	203	#define PAGE_KERNEL_X __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RWX)
	204	#define PAGE_KERNEL_RO __pgprot(_PAGE_BASE \| _PAGE_KERNEL_RO)
	205	#define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE \| _PAGE_KERNEL_ROX)
	206
	207	/* Protection used for kernel text. We want the debuggers to be able to
	208	* set breakpoints anywhere, so don't write protect the kernel text
	209	* on platforms where such control is possible.
	210	*/
	211	#if defined(CONFIG_KGDB) \|\| defined(CONFIG_XMON) \|\| defined(CONFIG_BDI_SWITCH) \|\|\
	212	defined(CONFIG_KPROBES) \|\| defined(CONFIG_DYNAMIC_FTRACE)
	213	#define PAGE_KERNEL_TEXT PAGE_KERNEL_X
c605782b	214	#else
371352ca	215	#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX
c605782b BH	216	#endif
c605782b BH	217
371352ca AK	218	/* Make modules code happy. We don't set RO yet */
	219	#define PAGE_KERNEL_EXEC PAGE_KERNEL_X
	220	#define PAGE_AGP (PAGE_KERNEL_NC)
	221
	222	#define PMD_BAD_BITS (PTE_TABLE_SIZE-1)
	223	#define PUD_BAD_BITS (PMD_TABLE_SIZE-1)
371352ca AK	224
	225	#ifndef __ASSEMBLY__
	226	#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \
	227	\|\| (pmd_val(pmd) & PMD_BAD_BITS))
	228	#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS)
	229
	230	#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \
	231	\|\| (pud_val(pud) & PUD_BAD_BITS))
	232	#define pud_page_vaddr(pud) (pud_val(pud) & ~PUD_MASKED_BITS)
	233
	234	#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1))
	235	#define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1))
	236	#define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1))
	237
	238	extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
	239	pte_t *ptep, unsigned long pte, int huge);
c6a3c495	240	extern unsigned long htab_convert_pte_flags(unsigned long pteflags);
371352ca AK	241	/* Atomic PTE updates */
	242	static inline unsigned long pte_update(struct mm_struct *mm,
	243	unsigned long addr,
	244	pte_t *ptep, unsigned long clr,
	245	unsigned long set,
	246	int huge)
	247	{
	248	unsigned long old, tmp;
	249
	250	__asm__ __volatile__(
	251	"1: ldarx %0,0,%3 # pte_update\n\
	252	andi. %1,%0,%6\n\
	253	bne- 1b \n\
	254	andc %1,%0,%4 \n\
	255	or %1,%1,%7\n\
	256	stdcx. %1,0,%3 \n\
	257	bne- 1b"
	258	: "=&r" (old), "=&r" (tmp), "=m" (*ptep)
	259	: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
	260	: "cc" );
	261	/* huge pages use the old page table lock */
	262	if (!huge)
	263	assert_pte_locked(mm, addr);
	264
	265	if (old & _PAGE_HASHPTE)
	266	hpte_need_flush(mm, addr, ptep, old, huge);
	267
	268	return old;
	269	}
	270
	271	static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
	272	unsigned long addr, pte_t *ptep)
	273	{
	274	unsigned long old;
	275
	276	if ((pte_val(*ptep) & (_PAGE_ACCESSED \| _PAGE_HASHPTE)) == 0)
	277	return 0;
	278	old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
	279	return (old & _PAGE_ACCESSED) != 0;
	280	}
	281	#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
	282	#define ptep_test_and_clear_young(__vma, __addr, __ptep) \
	283	({ \
	284	int __r; \
	285	__r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \
	286	__r; \
	287	})
	288
	289	#define __HAVE_ARCH_PTEP_SET_WRPROTECT
	290	static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
	291	pte_t *ptep)
	292	{
	293
	294	if ((pte_val(*ptep) & _PAGE_RW) == 0)
	295	return;
	296
	297	pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
	298	}
	299
	300	static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
	301	unsigned long addr, pte_t *ptep)
	302	{
	303	if ((pte_val(*ptep) & _PAGE_RW) == 0)
	304	return;
305
306	pte_update(mm, addr, ptep, _PAGE_RW, 0, 1);
307	}
308
309	/*
310	* We currently remove entries from the hashtable regardless of whether
311	* the entry was young or dirty. The generic routines only flush if the
312	* entry was young or dirty which is not good enough.
313	*
314	* We should be more intelligent about this but for the moment we override
315	* these functions and force a tlb flush unconditionally
316	*/
317	#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
318	#define ptep_clear_flush_young(__vma, __address, __ptep) \
319	({ \
320	int __young = __ptep_test_and_clear_young((__vma)->vm_mm, __address, \
321	__ptep); \
322	__young; \
323	})
324
325	#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
326	static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
327	unsigned long addr, pte_t *ptep)
328	{
329	unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
330	return __pte(old);
331	}
332
333	static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
334	pte_t * ptep)
335	{
336	pte_update(mm, addr, ptep, ~0UL, 0, 0);
337	}
338
339
340	/* Set the dirty and/or accessed bits atomically in a linux PTE, this
341	* function doesn't need to flush the hash entry
342	*/
343	static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)
344	{
345	unsigned long bits = pte_val(entry) &
7207f436 LD	346	(_PAGE_DIRTY \| _PAGE_ACCESSED \| _PAGE_RW \| _PAGE_EXEC \|
7207f436 LD	347	_PAGE_SOFT_DIRTY);
371352ca AK	348
	349	unsigned long old, tmp;
	350
	351	__asm__ __volatile__(
	352	"1: ldarx %0,0,%4\n\
	353	andi. %1,%0,%6\n\
	354	bne- 1b \n\
	355	or %0,%3,%0\n\
	356	stdcx. %0,0,%4\n\
	357	bne- 1b"
	358	:"=&r" (old), "=&r" (tmp), "=m" (*ptep)
	359	:"r" (bits), "r" (ptep), "m" (*ptep), "i" (_PAGE_BUSY)
	360	:"cc");
	361	}
	362
	363	#define __HAVE_ARCH_PTE_SAME
	364	#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)
	365
1ca72129 AK	366	/* Generic accessors to PTE bits */
	367	static inline int pte_write(pte_t pte) { return !!(pte_val(pte) & _PAGE_RW);}
	368	static inline int pte_dirty(pte_t pte) { return !!(pte_val(pte) & _PAGE_DIRTY); }
	369	static inline int pte_young(pte_t pte) { return !!(pte_val(pte) & _PAGE_ACCESSED); }
	370	static inline int pte_special(pte_t pte) { return !!(pte_val(pte) & _PAGE_SPECIAL); }
	371	static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; }
	372	static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
	373
7207f436 LD	374	#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
	375	static inline bool pte_soft_dirty(pte_t pte)
	376	{
	377	return !!(pte_val(pte) & _PAGE_SOFT_DIRTY);
	378	}
	379	static inline pte_t pte_mksoft_dirty(pte_t pte)
	380	{
	381	return __pte(pte_val(pte) \| _PAGE_SOFT_DIRTY);
	382	}
	383
	384	static inline pte_t pte_clear_soft_dirty(pte_t pte)
	385	{
	386	return __pte(pte_val(pte) & ~_PAGE_SOFT_DIRTY);
	387	}
	388	#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
	389
1ca72129 AK	390	#ifdef CONFIG_NUMA_BALANCING
	391	/*
	392	* These work without NUMA balancing but the kernel does not care. See the
	393	* comment in include/asm-generic/pgtable.h . On powerpc, this will only
	394	* work for user pages and always return true for kernel pages.
	395	*/
	396	static inline int pte_protnone(pte_t pte)
	397	{
	398	return (pte_val(pte) &
	399	(_PAGE_PRESENT \| _PAGE_USER)) == _PAGE_PRESENT;
	400	}
	401	#endif /* CONFIG_NUMA_BALANCING */
	402
	403	static inline int pte_present(pte_t pte)
	404	{
	405	return pte_val(pte) & _PAGE_PRESENT;
	406	}
	407
	408	/* Conversion functions: convert a page and protection to a page entry,
	409	* and a page entry and page directory to the page they refer to.
	410	*
	411	* Even if PTEs can be unsigned long long, a PFN is always an unsigned
	412	* long for now.
	413	*/
	414	static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
	415	{
	416	return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) \|
	417	pgprot_val(pgprot));
	418	}
	419
	420	static inline unsigned long pte_pfn(pte_t pte)
	421	{
	422	return pte_val(pte) >> PTE_RPN_SHIFT;
	423	}
	424
	425	/* Generic modifiers for PTE bits */
	426	static inline pte_t pte_wrprotect(pte_t pte)
	427	{
	428	return __pte(pte_val(pte) & ~_PAGE_RW);
	429	}
	430
	431	static inline pte_t pte_mkclean(pte_t pte)
	432	{
	433	return __pte(pte_val(pte) & ~_PAGE_DIRTY);
	434	}
	435
	436	static inline pte_t pte_mkold(pte_t pte)
	437	{
	438	return __pte(pte_val(pte) & ~_PAGE_ACCESSED);
	439	}
	440
	441	static inline pte_t pte_mkwrite(pte_t pte)
	442	{
	443	return __pte(pte_val(pte) \| _PAGE_RW);
	444	}
	445
	446	static inline pte_t pte_mkdirty(pte_t pte)
	447	{
7207f436	448	return __pte(pte_val(pte) \| _PAGE_DIRTY \| _PAGE_SOFT_DIRTY);
1ca72129 AK	449	}
	450
	451	static inline pte_t pte_mkyoung(pte_t pte)
	452	{
	453	return __pte(pte_val(pte) \| _PAGE_ACCESSED);
	454	}
	455
	456	static inline pte_t pte_mkspecial(pte_t pte)
	457	{
	458	return __pte(pte_val(pte) \| _PAGE_SPECIAL);
	459	}
	460
	461	static inline pte_t pte_mkhuge(pte_t pte)
	462	{
	463	return pte;
	464	}
	465
	466	static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
	467	{
	468	return __pte((pte_val(pte) & _PAGE_CHG_MASK) \| pgprot_val(newprot));
	469	}
	470
	471	/* This low level function performs the actual PTE insertion
	472	* Setting the PTE depends on the MMU type and other factors. It's
	473	* an horrible mess that I'm not going to try to clean up now but
	474	* I'm keeping it in one place rather than spread around
	475	*/
	476	static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
	477	pte_t *ptep, pte_t pte, int percpu)
	478	{
	479	/*
	480	* Anything else just stores the PTE normally. That covers all 64-bit
	481	* cases, and 32-bit non-hash with 32-bit PTEs.
	482	*/
	483	*ptep = pte;
	484	}
	485
	486	/*
	487	* Macro to mark a page protection value as "uncacheable".
	488	*/
	489
	490	#define _PAGE_CACHE_CTL (_PAGE_COHERENT \| _PAGE_GUARDED \| _PAGE_NO_CACHE \| \
	491	_PAGE_WRITETHRU)
	492
	493	#define pgprot_noncached pgprot_noncached
	494	static inline pgprot_t pgprot_noncached(pgprot_t prot)
	495	{
	496	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) \|
	497	_PAGE_NO_CACHE \| _PAGE_GUARDED);
	498	}
	499
	500	#define pgprot_noncached_wc pgprot_noncached_wc
	501	static inline pgprot_t pgprot_noncached_wc(pgprot_t prot)
	502	{
	503	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) \|
	504	_PAGE_NO_CACHE);
	505	}
	506
	507	#define pgprot_cached pgprot_cached
	508	static inline pgprot_t pgprot_cached(pgprot_t prot)
	509	{
	510	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) \|
	511	_PAGE_COHERENT);
	512	}
513
514	#define pgprot_cached_wthru pgprot_cached_wthru
515	static inline pgprot_t pgprot_cached_wthru(pgprot_t prot)
516	{
517	return __pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) \|
518	_PAGE_COHERENT \| _PAGE_WRITETHRU);
519	}
520
521	#define pgprot_cached_noncoherent pgprot_cached_noncoherent
522	static inline pgprot_t pgprot_cached_noncoherent(pgprot_t prot)
523	{
524	return __pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL);
525	}
526
527	#define pgprot_writecombine pgprot_writecombine
528	static inline pgprot_t pgprot_writecombine(pgprot_t prot)
529	{
530	return pgprot_noncached_wc(prot);
531	}
532
371352ca AK	533	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	534	extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
	535	pmd_t *pmdp, unsigned long old_pmd);
	536	#else
	537	static inline void hpte_do_hugepage_flush(struct mm_struct *mm,
	538	unsigned long addr, pmd_t *pmdp,
	539	unsigned long old_pmd)
	540	{
	541	WARN(1, "%s called with THP disabled\n", __func__);
	542	}
	543	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	544
	545	#endif /* !__ASSEMBLY__ */
c605782b	546	#endif /* __KERNEL__ */
26b6a3d9	547	#endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */