[linux-2.6-block.git] / mm / pgtable-generic.c

/*
 *  mm/pgtable-generic.c
 *
 *  Generic pgtable methods declared in asm-generic/pgtable.h
 *
 *  Copyright (C) 2010  Linus Torvalds
 */

#include <linux/pagemap.h>
#include <asm/tlb.h>
#include <asm-generic/pgtable.h>

/*
 * If a p?d_bad entry is found while walking page tables, report
 * the error, before resetting entry to p?d_none.  Usually (but
 * very seldom) called out from the p?d_none_or_clear_bad macros.
 */

void pgd_clear_bad(pgd_t *pgd)
{
	pgd_ERROR(*pgd);
	pgd_clear(pgd);
}

void pud_clear_bad(pud_t *pud)
{
	pud_ERROR(*pud);
	pud_clear(pud);
}

void pmd_clear_bad(pmd_t *pmd)
{
	pmd_ERROR(*pmd);
	pmd_clear(pmd);
}

#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
 * Only sets the access flags (dirty, accessed), as well as write 
 * permission. Furthermore, we know it always gets set to a "more
 * permissive" setting, which allows most architectures to optimize
 * this. We return whether the PTE actually changed, which in turn
 * instructs the caller to do things like update__mmu_cache.  This
 * used to be done in the caller, but sparc needs minor faults to
 * force that call on sun4c so we changed this macro slightly
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);
	if (changed) {
		set_pte_at(vma->vm_mm, address, ptep, entry);
		flush_tlb_fix_spurious_fault(vma, address);
	}
	return changed;
}
#endif

#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	int changed = !pmd_same(*pmdp, entry);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	if (changed) {
		set_pmd_at(vma->vm_mm, address, pmdp, entry);
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
	return changed;
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	BUG();
	return 0;
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	int young;
	young = ptep_test_and_clear_young(vma, address, ptep);
	if (young)
		flush_tlb_page(vma, address);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
#else
	BUG();
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
		       pte_t *ptep)
{
	struct mm_struct *mm = (vma)->vm_mm;
	pte_t pte;
	pte = ptep_get_and_clear(mm, address, ptep);
	if (pte_accessible(mm, pte))
		flush_tlb_page(vma, address);
	return pte;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
		       pmd_t *pmdp)
{
	pmd_t pmd;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp);
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return pmd;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
			  pmd_t *pmdp)
{
	pmd_t pmd = pmd_mksplitting(*pmdp);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	/* tlb flush only to serialize against gup-fast */
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
				pgtable_t pgtable)
{
	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	if (!pmd_huge_pte(mm, pmdp))
		INIT_LIST_HEAD(&pgtable->lru);
	else
		list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
	pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
	pgtable_t pgtable;

	assert_spin_locked(pmd_lockptr(mm, pmdp));

	/* FIFO */
	pgtable = pmd_huge_pte(mm, pmdp);
	if (list_empty(&pgtable->lru))
		pmd_huge_pte(mm, pmdp) = NULL;
	else {
		pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
					      struct page, lru);
		list_del(&pgtable->lru);
	}
	return pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
		     pmd_t *pmdp)
{
	pmd_t entry = *pmdp;
	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry));
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
Commit	Line	Data
e2cda322 AA	1	/*
	2	* mm/pgtable-generic.c
	3	*
	4	* Generic pgtable methods declared in asm-generic/pgtable.h
	5	*
	6	* Copyright (C) 2010 Linus Torvalds
	7	*/
	8
f95ba941	9	#include <linux/pagemap.h>
e2cda322 AA	10	#include <asm/tlb.h>
	11	#include <asm-generic/pgtable.h>
	12
bc4b4448 JK	13	/*
	14	* If a p?d_bad entry is found while walking page tables, report
	15	* the error, before resetting entry to p?d_none. Usually (but
	16	* very seldom) called out from the p?d_none_or_clear_bad macros.
	17	*/
	18
	19	void pgd_clear_bad(pgd_t *pgd)
	20	{
	21	pgd_ERROR(*pgd);
	22	pgd_clear(pgd);
	23	}
	24
	25	void pud_clear_bad(pud_t *pud)
	26	{
	27	pud_ERROR(*pud);
	28	pud_clear(pud);
	29	}
	30
	31	void pmd_clear_bad(pmd_t *pmd)
	32	{
	33	pmd_ERROR(*pmd);
	34	pmd_clear(pmd);
	35	}
	36
e2cda322 AA	37	#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
e2cda322 AA	38	/*
cef23d9d RR	39	* Only sets the access flags (dirty, accessed), as well as write
cef23d9d RR	40	* permission. Furthermore, we know it always gets set to a "more
e2cda322 AA	41	* permissive" setting, which allows most architectures to optimize
	42	* this. We return whether the PTE actually changed, which in turn
	43	* instructs the caller to do things like update__mmu_cache. This
	44	* used to be done in the caller, but sparc needs minor faults to
	45	* force that call on sun4c so we changed this macro slightly
	46	*/
	47	int ptep_set_access_flags(struct vm_area_struct *vma,
	48	unsigned long address, pte_t *ptep,
	49	pte_t entry, int dirty)
	50	{
	51	int changed = !pte_same(*ptep, entry);
	52	if (changed) {
	53	set_pte_at(vma->vm_mm, address, ptep, entry);
cef23d9d	54	flush_tlb_fix_spurious_fault(vma, address);
e2cda322 AA	55	}
	56	return changed;
	57	}
	58	#endif
	59
	60	#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
	61	int pmdp_set_access_flags(struct vm_area_struct *vma,
	62	unsigned long address, pmd_t *pmdp,
	63	pmd_t entry, int dirty)
	64	{
	65	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	66	int changed = !pmd_same(*pmdp, entry);
	67	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	68	if (changed) {
	69	set_pmd_at(vma->vm_mm, address, pmdp, entry);
	70	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	71	}
	72	return changed;
	73	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	74	BUG();
	75	return 0;
	76	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	77	}
	78	#endif
	79
	80	#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
	81	int ptep_clear_flush_young(struct vm_area_struct *vma,
	82	unsigned long address, pte_t *ptep)
	83	{
	84	int young;
	85	young = ptep_test_and_clear_young(vma, address, ptep);
	86	if (young)
	87	flush_tlb_page(vma, address);
	88	return young;
	89	}
	90	#endif
	91
	92	#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
	93	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	94	unsigned long address, pmd_t *pmdp)
	95	{
	96	int young;
d8c37c48 NH	97	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	98	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	99	#else
e2cda322 AA	100	BUG();
e2cda322 AA	101	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	102	young = pmdp_test_and_clear_young(vma, address, pmdp);
	103	if (young)
	104	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	105	return young;
	106	}
	107	#endif
	108
	109	#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
	110	pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
	111	pte_t *ptep)
	112	{
20841405	113	struct mm_struct *mm = (vma)->vm_mm;
e2cda322	114	pte_t pte;
20841405 RR	115	pte = ptep_get_and_clear(mm, address, ptep);
20841405 RR	116	if (pte_accessible(mm, pte))
8d1acce4	117	flush_tlb_page(vma, address);
e2cda322 AA	118	return pte;
	119	}
	120	#endif
	121
	122	#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
b3697c02	123	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
e2cda322 AA	124	pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
	125	pmd_t *pmdp)
	126	{
	127	pmd_t pmd;
e2cda322 AA	128	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	129	pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp);
	130	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	131	return pmd;
	132	}
b3697c02	133	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	134	#endif
	135
	136	#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
b3697c02	137	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
73636b1a CM	138	void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
73636b1a CM	139	pmd_t *pmdp)
e2cda322	140	{
e2cda322 AA	141	pmd_t pmd = pmd_mksplitting(*pmdp);
	142	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	143	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	144	/* tlb flush only to serialize against gup-fast */
	145	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
e2cda322	146	}
b3697c02	147	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322	148	#endif
e3ebcf64 GS	149
	150	#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
	151	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6b0b50b0 AK	152	void pgtable_trans_huge_deposit(struct mm_struct mm, pmd_t pmdp,
6b0b50b0 AK	153	pgtable_t pgtable)
e3ebcf64	154	{
c4088ebd	155	assert_spin_locked(pmd_lockptr(mm, pmdp));
e3ebcf64 GS	156
e3ebcf64 GS	157	/* FIFO */
c389a250	158	if (!pmd_huge_pte(mm, pmdp))
e3ebcf64 GS	159	INIT_LIST_HEAD(&pgtable->lru);
e3ebcf64 GS	160	else
c389a250 KS	161	list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
c389a250 KS	162	pmd_huge_pte(mm, pmdp) = pgtable;
e3ebcf64 GS	163	}
	164	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	165	#endif
	166
	167	#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
	168	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	169	/* no "address" argument so destroys page coloring of some arch */
6b0b50b0	170	pgtable_t pgtable_trans_huge_withdraw(struct mm_struct mm, pmd_t pmdp)
e3ebcf64 GS	171	{
	172	pgtable_t pgtable;
	173
c4088ebd	174	assert_spin_locked(pmd_lockptr(mm, pmdp));
e3ebcf64 GS	175
e3ebcf64 GS	176	/* FIFO */
c389a250	177	pgtable = pmd_huge_pte(mm, pmdp);
e3ebcf64	178	if (list_empty(&pgtable->lru))
c389a250	179	pmd_huge_pte(mm, pmdp) = NULL;
e3ebcf64	180	else {
c389a250	181	pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
e3ebcf64 GS	182	struct page, lru);
	183	list_del(&pgtable->lru);
	184	}
	185	return pgtable;
	186	}
	187	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	188	#endif
46dcde73 GS	189
	190	#ifndef __HAVE_ARCH_PMDP_INVALIDATE
	191	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	192	void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
	193	pmd_t *pmdp)
	194	{
67f87463	195	pmd_t entry = *pmdp;
ce8369bc	196	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(entry));
46dcde73 GS	197	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	198	}
	199	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	200	#endif