[linux-2.6-block.git] / mm / pgtable-generic.c

/*
 *  mm/pgtable-generic.c
 *
 *  Generic pgtable methods declared in asm-generic/pgtable.h
 *
 *  Copyright (C) 2010  Linus Torvalds
 */

#include <linux/pagemap.h>
#include <asm/tlb.h>
#include <asm-generic/pgtable.h>

/*
 * If a p?d_bad entry is found while walking page tables, report
 * the error, before resetting entry to p?d_none.  Usually (but
 * very seldom) called out from the p?d_none_or_clear_bad macros.
 */

void pgd_clear_bad(pgd_t *pgd)
{
	pgd_ERROR(*pgd);
	pgd_clear(pgd);
}

void pud_clear_bad(pud_t *pud)
{
	pud_ERROR(*pud);
	pud_clear(pud);
}

void pmd_clear_bad(pmd_t *pmd)
{
	pmd_ERROR(*pmd);
	pmd_clear(pmd);
}

#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
 * Only sets the access flags (dirty, accessed), as well as write 
 * permission. Furthermore, we know it always gets set to a "more
 * permissive" setting, which allows most architectures to optimize
 * this. We return whether the PTE actually changed, which in turn
 * instructs the caller to do things like update__mmu_cache.  This
 * used to be done in the caller, but sparc needs minor faults to
 * force that call on sun4c so we changed this macro slightly
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);
	if (changed) {
		set_pte_at(vma->vm_mm, address, ptep, entry);
		flush_tlb_fix_spurious_fault(vma, address);
	}
	return changed;
}
#endif

#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	int changed = !pmd_same(*pmdp, entry);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	if (changed) {
		set_pmd_at(vma->vm_mm, address, pmdp, entry);
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
	return changed;
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	BUG();
	return 0;
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	int young;
	young = ptep_test_and_clear_young(vma, address, ptep);
	if (young)
		flush_tlb_page(vma, address);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
#else
	BUG();
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return young;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
		       pte_t *ptep)
{
	pte_t pte;
	pte = ptep_get_and_clear((vma)->vm_mm, address, ptep);
	if (pte_accessible(pte))
		flush_tlb_page(vma, address);
	return pte;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
		       pmd_t *pmdp)
{
	pmd_t pmd;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp);
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	return pmd;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
			  pmd_t *pmdp)
{
	pmd_t pmd = pmd_mksplitting(*pmdp);
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	/* tlb flush only to serialize against gup-fast */
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
				pgtable_t pgtable)
{
	assert_spin_locked(&mm->page_table_lock);

	/* FIFO */
	if (!pmd_huge_pte(mm, pmdp))
		INIT_LIST_HEAD(&pgtable->lru);
	else
		list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
	pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
	pgtable_t pgtable;

	assert_spin_locked(&mm->page_table_lock);

	/* FIFO */
	pgtable = pmd_huge_pte(mm, pmdp);
	if (list_empty(&pgtable->lru))
		pmd_huge_pte(mm, pmdp) = NULL;
	else {
		pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
					      struct page, lru);
		list_del(&pgtable->lru);
	}
	return pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
		     pmd_t *pmdp)
{
	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(*pmdp));
	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif
Commit	Line	Data
e2cda322 AA	1	/*
	2	* mm/pgtable-generic.c
	3	*
	4	* Generic pgtable methods declared in asm-generic/pgtable.h
	5	*
	6	* Copyright (C) 2010 Linus Torvalds
	7	*/
	8
f95ba941	9	#include <linux/pagemap.h>
e2cda322 AA	10	#include <asm/tlb.h>
	11	#include <asm-generic/pgtable.h>
	12
bc4b4448 JK	13	/*
	14	* If a p?d_bad entry is found while walking page tables, report
	15	* the error, before resetting entry to p?d_none. Usually (but
	16	* very seldom) called out from the p?d_none_or_clear_bad macros.
	17	*/
	18
	19	void pgd_clear_bad(pgd_t *pgd)
	20	{
	21	pgd_ERROR(*pgd);
	22	pgd_clear(pgd);
	23	}
	24
	25	void pud_clear_bad(pud_t *pud)
	26	{
	27	pud_ERROR(*pud);
	28	pud_clear(pud);
	29	}
	30
	31	void pmd_clear_bad(pmd_t *pmd)
	32	{
	33	pmd_ERROR(*pmd);
	34	pmd_clear(pmd);
	35	}
	36
e2cda322 AA	37	#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
e2cda322 AA	38	/*
cef23d9d RR	39	* Only sets the access flags (dirty, accessed), as well as write
cef23d9d RR	40	* permission. Furthermore, we know it always gets set to a "more
e2cda322 AA	41	* permissive" setting, which allows most architectures to optimize
	42	* this. We return whether the PTE actually changed, which in turn
	43	* instructs the caller to do things like update__mmu_cache. This
	44	* used to be done in the caller, but sparc needs minor faults to
	45	* force that call on sun4c so we changed this macro slightly
	46	*/
	47	int ptep_set_access_flags(struct vm_area_struct *vma,
	48	unsigned long address, pte_t *ptep,
	49	pte_t entry, int dirty)
	50	{
	51	int changed = !pte_same(*ptep, entry);
	52	if (changed) {
	53	set_pte_at(vma->vm_mm, address, ptep, entry);
cef23d9d	54	flush_tlb_fix_spurious_fault(vma, address);
e2cda322 AA	55	}
	56	return changed;
	57	}
	58	#endif
	59
	60	#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
	61	int pmdp_set_access_flags(struct vm_area_struct *vma,
	62	unsigned long address, pmd_t *pmdp,
	63	pmd_t entry, int dirty)
	64	{
	65	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	66	int changed = !pmd_same(*pmdp, entry);
	67	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	68	if (changed) {
	69	set_pmd_at(vma->vm_mm, address, pmdp, entry);
	70	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	71	}
	72	return changed;
	73	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	74	BUG();
	75	return 0;
	76	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	77	}
	78	#endif
	79
	80	#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
	81	int ptep_clear_flush_young(struct vm_area_struct *vma,
	82	unsigned long address, pte_t *ptep)
	83	{
	84	int young;
	85	young = ptep_test_and_clear_young(vma, address, ptep);
	86	if (young)
	87	flush_tlb_page(vma, address);
	88	return young;
	89	}
	90	#endif
	91
	92	#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
	93	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	94	unsigned long address, pmd_t *pmdp)
	95	{
	96	int young;
d8c37c48 NH	97	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	98	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	99	#else
e2cda322 AA	100	BUG();
e2cda322 AA	101	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	102	young = pmdp_test_and_clear_young(vma, address, pmdp);
	103	if (young)
	104	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	105	return young;
	106	}
	107	#endif
	108
	109	#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
	110	pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
	111	pte_t *ptep)
	112	{
	113	pte_t pte;
	114	pte = ptep_get_and_clear((vma)->vm_mm, address, ptep);
8d1acce4 RR	115	if (pte_accessible(pte))
8d1acce4 RR	116	flush_tlb_page(vma, address);
e2cda322 AA	117	return pte;
	118	}
	119	#endif
	120
	121	#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH
b3697c02	122	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
e2cda322 AA	123	pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address,
	124	pmd_t *pmdp)
	125	{
	126	pmd_t pmd;
e2cda322 AA	127	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	128	pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp);
	129	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	130	return pmd;
	131	}
b3697c02	132	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322 AA	133	#endif
	134
	135	#ifndef __HAVE_ARCH_PMDP_SPLITTING_FLUSH
b3697c02	136	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
73636b1a CM	137	void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
73636b1a CM	138	pmd_t *pmdp)
e2cda322	139	{
e2cda322 AA	140	pmd_t pmd = pmd_mksplitting(*pmdp);
	141	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	142	set_pmd_at(vma->vm_mm, address, pmdp, pmd);
	143	/* tlb flush only to serialize against gup-fast */
	144	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
e2cda322	145	}
b3697c02	146	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
e2cda322	147	#endif
e3ebcf64 GS	148
	149	#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
	150	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6b0b50b0 AK	151	void pgtable_trans_huge_deposit(struct mm_struct mm, pmd_t pmdp,
6b0b50b0 AK	152	pgtable_t pgtable)
e3ebcf64 GS	153	{
	154	assert_spin_locked(&mm->page_table_lock);
	155
	156	/* FIFO */
c389a250	157	if (!pmd_huge_pte(mm, pmdp))
e3ebcf64 GS	158	INIT_LIST_HEAD(&pgtable->lru);
e3ebcf64 GS	159	else
c389a250 KS	160	list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
c389a250 KS	161	pmd_huge_pte(mm, pmdp) = pgtable;
e3ebcf64 GS	162	}
	163	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	164	#endif
	165
	166	#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
	167	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	168	/* no "address" argument so destroys page coloring of some arch */
6b0b50b0	169	pgtable_t pgtable_trans_huge_withdraw(struct mm_struct mm, pmd_t pmdp)
e3ebcf64 GS	170	{
	171	pgtable_t pgtable;
	172
	173	assert_spin_locked(&mm->page_table_lock);
	174
	175	/* FIFO */
c389a250	176	pgtable = pmd_huge_pte(mm, pmdp);
e3ebcf64	177	if (list_empty(&pgtable->lru))
c389a250	178	pmd_huge_pte(mm, pmdp) = NULL;
e3ebcf64	179	else {
c389a250	180	pmd_huge_pte(mm, pmdp) = list_entry(pgtable->lru.next,
e3ebcf64 GS	181	struct page, lru);
	182	list_del(&pgtable->lru);
	183	}
	184	return pgtable;
	185	}
	186	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	187	#endif
46dcde73 GS	188
	189	#ifndef __HAVE_ARCH_PMDP_INVALIDATE
	190	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	191	void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
	192	pmd_t *pmdp)
	193	{
	194	set_pmd_at(vma->vm_mm, address, pmdp, pmd_mknotpresent(*pmdp));
	195	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	196	}
	197	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	198	#endif