[linux-2.6-block.git] / include / linux / huge_mm.h

#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
				      struct vm_area_struct *vma,
				      unsigned long address, pmd_t *pmd,
				      unsigned int flags);
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
extern void huge_pmd_set_accessed(struct mm_struct *mm,
				  struct vm_area_struct *vma,
				  unsigned long address, pmd_t *pmd,
				  pmd_t orig_pmd, int dirty);
extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
			       unsigned long address, pmd_t *pmd,
			       pmd_t orig_pmd);
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr);
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);
extern int move_huge_pmd(struct vm_area_struct *vma,
			 struct vm_area_struct *new_vma,
			 unsigned long old_addr,
			 unsigned long new_addr, unsigned long old_end,
			 pmd_t *old_pmd, pmd_t *new_pmd);
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, pgprot_t newprot,
			int prot_numa);

enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

enum page_check_address_pmd_flag {
	PAGE_CHECK_ADDRESS_PMD_FLAG,
	PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
	PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
};
extern pmd_t *page_check_address_pmd(struct page *page,
				     struct mm_struct *mm,
				     unsigned long address,
				     enum page_check_address_pmd_flag flag);

#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))

extern bool is_vma_temporary_stack(struct vm_area_struct *vma);

#define transparent_hugepage_enabled(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_FLAG) ||				\
	  (transparent_hugepage_flags &					\
	   (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
	 !is_vma_temporary_stack(__vma))
#define transparent_hugepage_defrag(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||			\
	 (transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&		\
	  (__vma)->vm_flags & VM_HUGEPAGE))
#define transparent_hugepage_use_zero_page()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long transparent_hugepage_flags;
extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			  pmd_t *dst_pmd, pmd_t *src_pmd,
			  struct vm_area_struct *vma,
			  unsigned long addr, unsigned long end);
extern int split_huge_page_to_list(struct page *page, struct list_head *list);
static inline int split_huge_page(struct page *page)
{
	return split_huge_page_to_list(page, NULL);
}
extern void __split_huge_page_pmd(struct vm_area_struct *vma,
		unsigned long address, pmd_t *pmd);
#define split_huge_page_pmd(__vma, __address, __pmd)			\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		if (unlikely(pmd_trans_huge(*____pmd)))			\
			__split_huge_page_pmd(__vma, __address,		\
					____pmd);			\
	}  while (0)
#define wait_split_huge_page(__anon_vma, __pmd)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		anon_vma_lock_write(__anon_vma);			\
		anon_vma_unlock_write(__anon_vma);			\
		BUG_ON(pmd_trans_splitting(*____pmd) ||			\
		       pmd_trans_huge(*____pmd));			\
	} while (0)
extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
		pmd_t *pmd);
#if HPAGE_PMD_ORDER >= MAX_ORDER
#error "hugepages can't be allocated by the buddy allocator"
#endif
extern int hugepage_madvise(struct vm_area_struct *vma,
			    unsigned long *vm_flags, int advice);
extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
				    unsigned long start,
				    unsigned long end,
				    long adjust_next);
extern int __pmd_trans_huge_lock(pmd_t *pmd,
				 struct vm_area_struct *vma);
/* mmap_sem must be held on entry */
static inline int pmd_trans_huge_lock(pmd_t *pmd,
				      struct vm_area_struct *vma)
{
	VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
	if (pmd_trans_huge(*pmd))
		return __pmd_trans_huge_lock(pmd, vma);
	else
		return 0;
}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
	if (!vma->anon_vma || vma->vm_ops)
		return;
	__vma_adjust_trans_huge(vma, start, end, adjust_next);
}
static inline int hpage_nr_pages(struct page *page)
{
	if (unlikely(PageTransHuge(page)))
		return HPAGE_PMD_NR;
	return 1;
}
static inline struct page *compound_trans_head(struct page *page)
{
	if (PageTail(page)) {
		struct page *head;
		head = page->first_page;
		smp_rmb();
		/*
		 * head may be a dangling pointer.
		 * __split_huge_page_refcount clears PageTail before
		 * overwriting first_page, so if PageTail is still
		 * there it means the head pointer isn't dangling.
		 */
		if (PageTail(page))
			return head;
	}
	return page;
}

extern int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
				unsigned long addr, pmd_t pmd, pmd_t *pmdp);

#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })

#define hpage_nr_pages(x) 1

#define transparent_hugepage_enabled(__vma) 0

#define transparent_hugepage_flags 0UL
static inline int
split_huge_page_to_list(struct page *page, struct list_head *list)
{
	return 0;
}
static inline int split_huge_page(struct page *page)
{
	return 0;
}
#define split_huge_page_pmd(__vma, __address, __pmd)	\
	do { } while (0)
#define wait_split_huge_page(__anon_vma, __pmd)	\
	do { } while (0)
#define split_huge_page_pmd_mm(__mm, __address, __pmd)	\
	do { } while (0)
#define compound_trans_head(page) compound_head(page)
static inline int hugepage_madvise(struct vm_area_struct *vma,
				   unsigned long *vm_flags, int advice)
{
	BUG();
	return 0;
}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
}
static inline int pmd_trans_huge_lock(pmd_t *pmd,
				      struct vm_area_struct *vma)
{
	return 0;
}

static inline int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
					unsigned long addr, pmd_t pmd, pmd_t *pmdp)
{
	return 0;
}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */
Commit	Line	Data
71e3aac0 AA	1	#ifndef _LINUX_HUGE_MM_H
	2	#define _LINUX_HUGE_MM_H
	3
	4	extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
	5	struct vm_area_struct *vma,
	6	unsigned long address, pmd_t *pmd,
	7	unsigned int flags);
	8	extern int copy_huge_pmd(struct mm_struct dst_mm, struct mm_struct src_mm,
	9	pmd_t dst_pmd, pmd_t src_pmd, unsigned long addr,
	10	struct vm_area_struct *vma);
a1dd450b WD	11	extern void huge_pmd_set_accessed(struct mm_struct *mm,
	12	struct vm_area_struct *vma,
	13	unsigned long address, pmd_t *pmd,
	14	pmd_t orig_pmd, int dirty);
71e3aac0 AA	15	extern int do_huge_pmd_wp_page(struct mm_struct mm, struct vm_area_struct vma,
	16	unsigned long address, pmd_t *pmd,
	17	pmd_t orig_pmd);
b676b293	18	extern struct page follow_trans_huge_pmd(struct vm_area_struct vma,
71e3aac0 AA	19	unsigned long addr,
	20	pmd_t *pmd,
	21	unsigned int flags);
	22	extern int zap_huge_pmd(struct mmu_gather *tlb,
	23	struct vm_area_struct *vma,
f21760b1	24	pmd_t *pmd, unsigned long addr);
0ca1634d JW	25	extern int mincore_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
	26	unsigned long addr, unsigned long end,
	27	unsigned char *vec);
37a1c49a AA	28	extern int move_huge_pmd(struct vm_area_struct *vma,
	29	struct vm_area_struct *new_vma,
	30	unsigned long old_addr,
	31	unsigned long new_addr, unsigned long old_end,
	32	pmd_t old_pmd, pmd_t new_pmd);
cd7548ab	33	extern int change_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
4b10e7d5 MG	34	unsigned long addr, pgprot_t newprot,
4b10e7d5 MG	35	int prot_numa);
71e3aac0 AA	36
	37	enum transparent_hugepage_flag {
	38	TRANSPARENT_HUGEPAGE_FLAG,
	39	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	40	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	41	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f	42	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
79da5407	43	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
71e3aac0 AA	44	#ifdef CONFIG_DEBUG_VM
	45	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
	46	#endif
	47	};
	48
	49	enum page_check_address_pmd_flag {
	50	PAGE_CHECK_ADDRESS_PMD_FLAG,
	51	PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
	52	PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
	53	};
	54	extern pmd_t page_check_address_pmd(struct page page,
	55	struct mm_struct *mm,
	56	unsigned long address,
	57	enum page_check_address_pmd_flag flag);
	58
d8c37c48 NH	59	#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
	60	#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
	61
71e3aac0	62	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
fde52796 AK	63	#define HPAGE_PMD_SHIFT PMD_SHIFT
	64	#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
	65	#define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1))
71e3aac0	66
20995974 AS	67	extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
20995974 AS	68
71e3aac0	69	#define transparent_hugepage_enabled(__vma) \
a664b2d8 AA	70	((transparent_hugepage_flags & \
	71	(1<<TRANSPARENT_HUGEPAGE_FLAG) \|\| \
	72	(transparent_hugepage_flags & \
	73	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \
	74	((__vma)->vm_flags & VM_HUGEPAGE))) && \
a7d6e4ec AA	75	!((__vma)->vm_flags & VM_NOHUGEPAGE) && \
a7d6e4ec AA	76	!is_vma_temporary_stack(__vma))
71e3aac0 AA	77	#define transparent_hugepage_defrag(__vma) \
	78	((transparent_hugepage_flags & \
	79	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) \|\| \
	80	(transparent_hugepage_flags & \
	81	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \
	82	(__vma)->vm_flags & VM_HUGEPAGE))
79da5407 KS	83	#define transparent_hugepage_use_zero_page() \
	84	(transparent_hugepage_flags & \
	85	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
71e3aac0 AA	86	#ifdef CONFIG_DEBUG_VM
	87	#define transparent_hugepage_debug_cow() \
	88	(transparent_hugepage_flags & \
	89	(1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
	90	#else /* CONFIG_DEBUG_VM */
	91	#define transparent_hugepage_debug_cow() 0
	92	#endif /* CONFIG_DEBUG_VM */
	93
	94	extern unsigned long transparent_hugepage_flags;
	95	extern int copy_pte_range(struct mm_struct dst_mm, struct mm_struct src_mm,
	96	pmd_t dst_pmd, pmd_t src_pmd,
	97	struct vm_area_struct *vma,
	98	unsigned long addr, unsigned long end);
5bc7b8ac SL	99	extern int split_huge_page_to_list(struct page page, struct list_head list);
	100	static inline int split_huge_page(struct page *page)
	101	{
	102	return split_huge_page_to_list(page, NULL);
	103	}
e180377f KS	104	extern void __split_huge_page_pmd(struct vm_area_struct *vma,
	105	unsigned long address, pmd_t *pmd);
	106	#define split_huge_page_pmd(__vma, __address, __pmd) \
71e3aac0 AA	107	do { \
	108	pmd_t *____pmd = (__pmd); \
	109	if (unlikely(pmd_trans_huge(*____pmd))) \
e180377f KS	110	__split_huge_page_pmd(__vma, __address, \
e180377f KS	111	____pmd); \
71e3aac0 AA	112	} while (0)
	113	#define wait_split_huge_page(__anon_vma, __pmd) \
	114	do { \
	115	pmd_t *____pmd = (__pmd); \
4fc3f1d6	116	anon_vma_lock_write(__anon_vma); \
08b52706	117	anon_vma_unlock_write(__anon_vma); \
71e3aac0 AA	118	BUG_ON(pmd_trans_splitting(*____pmd) \|\| \
	119	pmd_trans_huge(*____pmd)); \
	120	} while (0)
e180377f KS	121	extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
e180377f KS	122	pmd_t *pmd);
dfa5e237	123	#if HPAGE_PMD_ORDER >= MAX_ORDER
71e3aac0 AA	124	#error "hugepages can't be allocated by the buddy allocator"
71e3aac0 AA	125	#endif
60ab3244 AA	126	extern int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	127	unsigned long *vm_flags, int advice);
94fcc585 AA	128	extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
	129	unsigned long start,
	130	unsigned long end,
	131	long adjust_next);
025c5b24 NH	132	extern int __pmd_trans_huge_lock(pmd_t *pmd,
	133	struct vm_area_struct *vma);
	134	/* mmap_sem must be held on entry */
	135	static inline int pmd_trans_huge_lock(pmd_t *pmd,
	136	struct vm_area_struct *vma)
	137	{
	138	VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
	139	if (pmd_trans_huge(*pmd))
	140	return __pmd_trans_huge_lock(pmd, vma);
	141	else
	142	return 0;
	143	}
94fcc585 AA	144	static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
	145	unsigned long start,
	146	unsigned long end,
	147	long adjust_next)
	148	{
78f11a25	149	if (!vma->anon_vma \|\| vma->vm_ops)
94fcc585 AA	150	return;
	151	__vma_adjust_trans_huge(vma, start, end, adjust_next);
	152	}
2c888cfb RR	153	static inline int hpage_nr_pages(struct page *page)
	154	{
	155	if (unlikely(PageTransHuge(page)))
	156	return HPAGE_PMD_NR;
	157	return 1;
	158	}
22e5c47e AA	159	static inline struct page compound_trans_head(struct page page)
	160	{
	161	if (PageTail(page)) {
	162	struct page *head;
	163	head = page->first_page;
	164	smp_rmb();
	165	/*
	166	* head may be a dangling pointer.
	167	* __split_huge_page_refcount clears PageTail before
	168	* overwriting first_page, so if PageTail is still
	169	* there it means the head pointer isn't dangling.
	170	*/
	171	if (PageTail(page))
	172	return head;
	173	}
	174	return page;
	175	}
d10e63f2	176
4daae3b4 MG	177	extern int do_huge_pmd_numa_page(struct mm_struct mm, struct vm_area_struct vma,
4daae3b4 MG	178	unsigned long addr, pmd_t pmd, pmd_t *pmdp);
d10e63f2	179
71e3aac0	180	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
d8c37c48 NH	181	#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
	182	#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
	183	#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
71e3aac0	184
2c888cfb RR	185	#define hpage_nr_pages(x) 1
2c888cfb RR	186
71e3aac0 AA	187	#define transparent_hugepage_enabled(__vma) 0
	188
	189	#define transparent_hugepage_flags 0UL
5bc7b8ac SL	190	static inline int
	191	split_huge_page_to_list(struct page page, struct list_head list)
	192	{
	193	return 0;
	194	}
71e3aac0 AA	195	static inline int split_huge_page(struct page *page)
	196	{
	197	return 0;
	198	}
e180377f	199	#define split_huge_page_pmd(__vma, __address, __pmd) \
71e3aac0 AA	200	do { } while (0)
	201	#define wait_split_huge_page(__anon_vma, __pmd) \
	202	do { } while (0)
e180377f KS	203	#define split_huge_page_pmd_mm(__mm, __address, __pmd) \
e180377f KS	204	do { } while (0)
22e5c47e	205	#define compound_trans_head(page) compound_head(page)
60ab3244 AA	206	static inline int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	207	unsigned long *vm_flags, int advice)
0af4e98b AA	208	{
	209	BUG();
	210	return 0;
	211	}
94fcc585 AA	212	static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
	213	unsigned long start,
	214	unsigned long end,
	215	long adjust_next)
	216	{
	217	}
025c5b24 NH	218	static inline int pmd_trans_huge_lock(pmd_t *pmd,
	219	struct vm_area_struct *vma)
	220	{
	221	return 0;
	222	}
d10e63f2	223
4daae3b4 MG	224	static inline int do_huge_pmd_numa_page(struct mm_struct mm, struct vm_area_struct vma,
4daae3b4 MG	225	unsigned long addr, pmd_t pmd, pmd_t *pmdp)
d10e63f2	226	{
4daae3b4	227	return 0;
d10e63f2 MG	228	}
d10e63f2 MG	229
71e3aac0 AA	230	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	231
	232	#endif /* _LINUX_HUGE_MM_H */