[linux-2.6-block.git] / mm / pgtable-generic.c

/*
 *  mm/pgtable-generic.c
 *
 *  Generic pgtable methods declared in asm-generic/pgtable.h
 *
 *  Copyright (C) 2010  Linus Torvalds
 */

#include <linux/pagemap.h>
#include <asm/tlb.h>
#include <asm-generic/pgtable.h>

/*
 * If a p?d_bad entry is found while walking page tables, report
 * the error, before resetting entry to p?d_none.  Usually (but
 * very seldom) called out from the p?d_none_or_clear_bad macros.
 */

void pgd_clear_bad(pgd_t *pgd)
{
	pgd_ERROR(*pgd);
	pgd_clear(pgd);
}

void pud_clear_bad(pud_t *pud)
{
	pud_ERROR(*pud);
	pud_clear(pud);
}

void pmd_clear_bad(pmd_t *pmd)
{
	pmd_ERROR(*pmd);
	pmd_clear(pmd);
}

#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
 * Only sets the access flags (dirty, accessed), as well as write 
 * permission. Furthermore, we know it always gets set to a "more
 * permissive" setting, which allows most architectures to optimize
 * this. We return whether the PTE actually changed, which in turn
 * instructs the caller to do things like update__mmu_cache.  This
 * used to be done in the caller, but sparc needs minor faults to
 * force that call on sun4c so we changed this macro slightly
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);
	if (changed) {
		set_pte_at(vma->vm_mm, address, ptep, entry);
		flush_tlb_fix_spurious_fault(vma, address);
	}
	return changed;
}
#endif

#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	int changed = !pmd_same(*pmdp, entry);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	if (changed) {
		set_pmd_at(vma->vm_mm, address, pmdp, entry);
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
	return changed;
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	BUG();
	return 0;
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	int young;
	young = ptep_test_and_clear_young(vma, address, ptep);
	if (young)
		flush_tlb_page(vma, address);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
#else
	BUG();
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
		       pte_t *ptep)
{
	struct mm_struct *mm = (vma)->vm_mm;
	pte_t pte;
	pte = ptep_get_and_clear(mm, address, ptep);
	if (pte_accessible(mm, pte))
		flush_tlb_page(vma, address);
	return pte;
}
#endif

#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
			    pmd_t *pmdp)
{
	pmd_t pmd;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	VM_BUG_ON(!pmd_trans_huge(*pmdp));
	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return pmd;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
			  pmd_t *pmdp)
{
	pmd_t pmd = pmd_mksplitting(*pmdp);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	/* tlb flush only to serialize against gup-fast */
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
				pgtable_t pgtable)
{
	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	if (!pmd_huge_pte(mm, pmdp))
		INIT_LIST_HEAD(&pgtable->lru);
	else
		list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
	pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
	pgtable_t pgtable;

	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	pgtable = pmd_huge_pte(mm, pmdp);
	if (list_empty(&pgtable->lru))
		pmd_huge_pte(mm, pmdp) = NULL;
	else {
		pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
					      struct page, lru);
		list_del(&pgtable->lru);
	}
	return pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
		     pmd_t *pmdp)
{
	pmd_t entry = *pmdp;
	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry));
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef pmdp_collapse_flush
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
			  pmd_t *pmdp)
{
	/*
	 * pmd and hugepage pte format are same. So we could
	 * use the same function.
	 */
	pmd_t pmd;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	VM_BUG_ON(pmd_trans_huge(*pmdp));
	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return pmd;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
Commit	Line	Data
e2cda322 AA	1	/*
	2	* mm/pgtable-generic.c
	3	*
	4	* Generic pgtable methods declared in asm-generic/pgtable.h
	5	*
	6	* Copyright (C) 2010 Linus Torvalds
	7	*/
	8
f95ba941	9	#include <linux/pagemap.h>
e2cda322 AA	10	#include <asm/tlb.h>
	11	#include <asm-generic/pgtable.h>
	12
bc4b4448 JK	13	/*
	14	* If a p?d_bad entry is found while walking page tables, report
	15	* the error, before resetting entry to p?d_none. Usually (but
	16	* very seldom) called out from the p?d_none_or_clear_bad macros.
	17	*/
	18
	19	void pgd_clear_bad(pgd_t *pgd)
	20	{
	21	pgd_ERROR(*pgd);
	22	pgd_clear(pgd);
	23	}
	24
	25	void pud_clear_bad(pud_t *pud)
	26	{
	27	pud_ERROR(*pud);
	28	pud_clear(pud);
	29	}
	30
	31	void pmd_clear_bad(pmd_t *pmd)
	32	{
	33	pmd_ERROR(*pmd);
	34	pmd_clear(pmd);
	35	}
	36
e2cda322 AA	37	#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
e2cda322 AA	38	/*
cef23d9d RR	39	* Only sets the access flags (dirty, accessed), as well as write
cef23d9d RR	40	* permission. Furthermore, we know it always gets set to a "more
e2cda322 AA	41	* permissive" setting, which allows most architectures to optimize
	42	* this. We return whether the PTE actually changed, which in turn
	43	* instructs the caller to do things like update__mmu_cache. This
	44	* used to be done in the caller, but sparc needs minor faults to
	45	* force that call on sun4c so we changed this macro slightly
	46	*/
	47	int ptep_set_access_flags(struct vm_area_struct *vma,
	48	unsigned long address, pte_t *ptep,
	49	pte_t entry, int dirty)
	50	{
	51	int changed = !pte_same(*ptep, entry);
	52	if (changed) {
	53	set_pte_at(vma->vm_mm, address, ptep, entry);
cef23d9d	54	flush_tlb_fix_spurious_fault(vma, address);
e2cda322 AA	55	}
	56	return changed;
	57	}
	58	#endif
	59
	60	#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
	61	int pmdp_set_access_flags(struct vm_area_struct *vma,
	62	unsigned long address, pmd_t *pmdp,
	63	pmd_t entry, int dirty)
	64	{
	65	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	66	int changed = !pmd_same(*pmdp, entry);
	67	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	68	if (changed) {
	69	set_pmd_at(vma->vm_mm, address, pmdp, entry);
	70	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	71	}
	72	return changed;
	73	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	74	BUG();
	75	return 0;
	76	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	77	}
	78	#endif
	79
	80	#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
	81	int ptep_clear_flush_young(struct vm_area_struct *vma,
	82	unsigned long address, pte_t *ptep)
	83	{
	84	int young;
	85	young = ptep_test_and_clear_young(vma, address, ptep);
	86	if (young)
	87	flush_tlb_page(vma, address);
	88	return young;
	89	}
	90	#endif
	91
	92	#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
	93	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	94	unsigned long address, pmd_t *pmdp)
	95	{
	96	int young;
d8c37c48 NH	97	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	98	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	99	#else
e2cda322 AA	100	BUG();
e2cda322 AA	101	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	102	young = pmdp_test_and_clear_young(vma, address, pmdp);
	103	if (young)
	104	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	105	return young;
	106	}
	107	#endif
	108
	109	#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
	110	pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
	111	pte_t *ptep)
	112	{
20841405	113	struct mm_struct *mm = (vma)->vm_mm;
e2cda322	114	pte_t pte;
20841405 RR	115	pte = ptep_get_and_clear(mm, address, ptep);
20841405 RR	116	if (pte_accessible(mm, pte))
8d1acce4	117	flush_tlb_page(vma, address);
e2cda322 AA	118	return pte;
	119	}
	120	#endif
	121
8809aa2d	122	#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
b3697c02	123	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
8809aa2d AK	124	pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
8809aa2d AK	125	pmd_t *pmdp)
e2cda322 AA	126	{
e2cda322 AA	127	pmd_t pmd;
e2cda322	128	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
f28b6ff8	129	VM_BUG_ON(!pmd_trans_huge(*pmdp));
8809aa2d	130	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
e2cda322 AA	131	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	132	return pmd;
	133	}
b3697c02	134	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	135	#endif
	136
	137	#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
b3697c02	138	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
73636b1a CM	139	void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
73636b1a CM	140	pmd_t *pmdp)
e2cda322	141	{
e2cda322 AA	142	pmd_t pmd = pmd_mksplitting(*pmdp);
	143	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	144	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	145	/* tlb flush only to serialize against gup-fast */
	146	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
e2cda322	147	}
b3697c02	148	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322	149	#endif
e3ebcf64 GS	150
	151	#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
	152	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6b0b50b0 AK	153	void pgtable_trans_huge_deposit(struct mm_struct mm, pmd_t pmdp,
6b0b50b0 AK	154	pgtable_t pgtable)
e3ebcf64	155	{
c4088ebd	156	assert_spin_locked(pmd_lockptr(mm, pmdp));
e3ebcf64 GS	157
e3ebcf64 GS	158	/* FIFO */
c389a250	159	if (!pmd_huge_pte(mm, pmdp))
e3ebcf64 GS	160	INIT_LIST_HEAD(&pgtable->lru);
e3ebcf64 GS	161	else
c389a250 KS	162	list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
c389a250 KS	163	pmd_huge_pte(mm, pmdp) = pgtable;
e3ebcf64 GS	164	}
	165	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	166	#endif
	167
	168	#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
	169	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	170	/* no "address" argument so destroys page coloring of some arch */
6b0b50b0	171	pgtable_t pgtable_trans_huge_withdraw(struct mm_struct mm, pmd_t pmdp)
e3ebcf64 GS	172	{
	173	pgtable_t pgtable;
	174
c4088ebd	175	assert_spin_locked(pmd_lockptr(mm, pmdp));
e3ebcf64 GS	176
e3ebcf64 GS	177	/* FIFO */
c389a250	178	pgtable = pmd_huge_pte(mm, pmdp);
e3ebcf64	179	if (list_empty(&pgtable->lru))
c389a250	180	pmd_huge_pte(mm, pmdp) = NULL;
e3ebcf64	181	else {
c389a250	182	pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
e3ebcf64 GS	183	struct page, lru);
	184	list_del(&pgtable->lru);
	185	}
	186	return pgtable;
	187	}
	188	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	189	#endif
46dcde73 GS	190
	191	#ifndef __HAVE_ARCH_PMDP_INVALIDATE
	192	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	193	void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
	194	pmd_t *pmdp)
	195	{
67f87463	196	pmd_t entry = *pmdp;
ce8369bc	197	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry));
46dcde73 GS	198	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	199	}
	200	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	201	#endif
f28b6ff8 AK	202
	203	#ifndef pmdp_collapse_flush
	204	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	205	pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
	206	pmd_t *pmdp)
	207	{
8809aa2d AK	208	/*
	209	* pmd and hugepage pte format are same. So we could
	210	* use the same function.
	211	*/
f28b6ff8 AK	212	pmd_t pmd;
	213
	214	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	215	VM_BUG_ON(pmd_trans_huge(*pmdp));
8809aa2d	216	pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
f28b6ff8 AK	217	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	218	return pmd;
	219	}
	220	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	221	#endif