[linux-2.6-block.git] / arch / x86 / mm / pgtable.c

#include <linux/mm.h>
#include <linux/gfp.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/tlb.h>
#include <asm/fixmap.h>

#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO

#ifdef CONFIG_HIGHPTE
#define PGALLOC_USER_GFP __GFP_HIGHMEM
#else
#define PGALLOC_USER_GFP 0
#endif

gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;

pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
	return (pte_t *)__get_free_page(PGALLOC_GFP);
}

pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
	struct page *pte;

	pte = alloc_pages(__userpte_alloc_gfp, 0);
	if (pte)
		pgtable_page_ctor(pte);
	return pte;
}

static int __init setup_userpte(char *arg)
{
	if (!arg)
		return -EINVAL;

	/*
	 * "userpte=nohigh" disables allocation of user pagetables in
	 * high memory.
	 */
	if (strcmp(arg, "nohigh") == 0)
		__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	else
		return -EINVAL;
	return 0;
}
early_param("userpte", setup_userpte);

void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
	pgtable_page_dtor(pte);
	paravirt_release_pte(page_to_pfn(pte));
	tlb_remove_page(tlb, pte);
}

#if PAGETABLE_LEVELS > 2
void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
{
	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
	tlb_remove_page(tlb, virt_to_page(pmd));
}

#if PAGETABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
	tlb_remove_page(tlb, virt_to_page(pud));
}
#endif	/* PAGETABLE_LEVELS > 3 */
#endif	/* PAGETABLE_LEVELS > 2 */

static inline void pgd_list_add(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_add(&page->lru, &pgd_list);
}

static inline void pgd_list_del(pgd_t *pgd)
{
	struct page *page = virt_to_page(pgd);

	list_del(&page->lru);
}

#define UNSHARED_PTRS_PER_PGD				\
	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)


static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
{
	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	virt_to_page(pgd)->index = (pgoff_t)mm;
}

struct mm_struct *pgd_page_get_mm(struct page *page)
{
	return (struct mm_struct *)page->index;
}

static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
{
	/* If the pgd points to a shared pagetable level (either the
	   ptes in non-PAE, or shared PMD in PAE), then just copy the
	   references from swapper_pg_dir. */
	if (PAGETABLE_LEVELS == 2 ||
	    (PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
	    PAGETABLE_LEVELS == 4) {
		clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
				swapper_pg_dir + KERNEL_PGD_BOUNDARY,
				KERNEL_PGD_PTRS);
	}

	/* list required to sync kernel mapping updates */
	if (!SHARED_KERNEL_PMD) {
		pgd_set_mm(pgd, mm);
		pgd_list_add(pgd);
	}
}

static void pgd_dtor(pgd_t *pgd)
{
	unsigned long flags; /* can be called from interrupt context */

	if (SHARED_KERNEL_PMD)
		return;

	spin_lock_irqsave(&pgd_lock, flags);
	pgd_list_del(pgd);
	spin_unlock_irqrestore(&pgd_lock, flags);
}

/*
 * List of all pgd's needed for non-PAE so it can invalidate entries
 * in both cached and uncached pgd's; not needed for PAE since the
 * kernel pmd is shared. If PAE were not to share the pmd a similar
 * tactic would be needed. This is essentially codepath-based locking
 * against pageattr.c; it is the unique case in which a valid change
 * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 * vmalloc faults work because attached pagetables are never freed.
 * -- wli
 */

#ifdef CONFIG_X86_PAE
/*
 * In PAE mode, we need to do a cr3 reload (=tlb flush) when
 * updating the top-level pagetable entries to guarantee the
 * processor notices the update.  Since this is expensive, and
 * all 4 top-level entries are used almost immediately in a
 * new process's life, we just pre-populate them here.
 *
 * Also, if we're in a paravirt environment where the kernel pmd is
 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
 * and initialize the kernel pmds here.
 */
#define PREALLOCATED_PMDS	UNSHARED_PTRS_PER_PGD

void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
{
	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);

	/* Note: almost everything apart from _PAGE_PRESENT is
	   reserved at the pmd (PDPT) level. */
	set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));

	/*
	 * According to Intel App note "TLBs, Paging-Structure Caches,
	 * and Their Invalidation", April 2007, document 317080-001,
	 * section 8.1: in PAE mode we explicitly have to flush the
	 * TLB via cr3 if the top-level pgd is changed...
	 */
	if (mm == current->active_mm)
		write_cr3(read_cr3());
}
#else  /* !CONFIG_X86_PAE */

/* No need to prepopulate any pagetable entries in non-PAE modes. */
#define PREALLOCATED_PMDS	0

#endif	/* CONFIG_X86_PAE */

static void free_pmds(pmd_t *pmds[])
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++)
		if (pmds[i])
			free_page((unsigned long)pmds[i]);
}

static int preallocate_pmds(pmd_t *pmds[])
{
	int i;
	bool failed = false;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
		if (pmd == NULL)
			failed = true;
		pmds[i] = pmd;
	}

	if (failed) {
		free_pmds(pmds);
		return -ENOMEM;
	}

	return 0;
}

/*
 * Mop up any pmd pages which may still be attached to the pgd.
 * Normally they will be freed by munmap/exit_mmap, but any pmd we
 * preallocate which never got a corresponding vma will need to be
 * freed manually.
 */
static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
{
	int i;

	for(i = 0; i < PREALLOCATED_PMDS; i++) {
		pgd_t pgd = pgdp[i];

		if (pgd_val(pgd) != 0) {
			pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);

			pgdp[i] = native_make_pgd(0);

			paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
			pmd_free(mm, pmd);
		}
	}
}

static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
{
	pud_t *pud;
	unsigned long addr;
	int i;

	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
		return;

	pud = pud_offset(pgd, 0);

 	for (addr = i = 0; i < PREALLOCATED_PMDS;
	     i++, pud++, addr += PUD_SIZE) {
		pmd_t *pmd = pmds[i];

		if (i >= KERNEL_PGD_BOUNDARY)
			memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
			       sizeof(pmd_t) * PTRS_PER_PMD);

		pud_populate(mm, pud, pmd);
	}
}

pgd_t *pgd_alloc(struct mm_struct *mm)
{
	pgd_t *pgd;
	pmd_t *pmds[PREALLOCATED_PMDS];
	unsigned long flags;

	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);

	if (pgd == NULL)
		goto out;

	mm->pgd = pgd;

	if (preallocate_pmds(pmds) != 0)
		goto out_free_pgd;

	if (paravirt_pgd_alloc(mm) != 0)
		goto out_free_pmds;

	/*
	 * Make sure that pre-populating the pmds is atomic with
	 * respect to anything walking the pgd_list, so that they
	 * never see a partially populated pgd.
	 */
	spin_lock_irqsave(&pgd_lock, flags);

	pgd_ctor(mm, pgd);
	pgd_prepopulate_pmd(mm, pgd, pmds);

	spin_unlock_irqrestore(&pgd_lock, flags);

	return pgd;

out_free_pmds:
	free_pmds(pmds);
out_free_pgd:
	free_page((unsigned long)pgd);
out:
	return NULL;
}

void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
	pgd_mop_up_pmds(mm, pgd);
	pgd_dtor(pgd);
	paravirt_pgd_free(mm, pgd);
	free_page((unsigned long)pgd);
}

int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
	int changed = !pte_same(*ptep, entry);

	if (changed && dirty) {
		*ptep = entry;
		pte_update_defer(vma->vm_mm, address, ptep);
		flush_tlb_page(vma, address);
	}

	return changed;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp,
			  pmd_t entry, int dirty)
{
	int changed = !pmd_same(*pmdp, entry);

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	if (changed && dirty) {
		*pmdp = entry;
		pmd_update_defer(vma->vm_mm, address, pmdp);
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}

	return changed;
}
#endif

int ptep_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pte_t *ptep)
{
	int ret = 0;

	if (pte_young(*ptep))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *) &ptep->pte);

	if (ret)
		pte_update(vma->vm_mm, addr, ptep);

	return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_test_and_clear_young(struct vm_area_struct *vma,
			      unsigned long addr, pmd_t *pmdp)
{
	int ret = 0;

	if (pmd_young(*pmdp))
		ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
					 (unsigned long *)pmdp);

	if (ret)
		pmd_update(vma->vm_mm, addr, pmdp);

	return ret;
}
#endif

int ptep_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pte_t *ptep)
{
	int young;

	young = ptep_test_and_clear_young(vma, address, ptep);
	if (young)
		flush_tlb_page(vma, address);

	return young;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
int pmdp_clear_flush_young(struct vm_area_struct *vma,
			   unsigned long address, pmd_t *pmdp)
{
	int young;

	VM_BUG_ON(address & ~HPAGE_PMD_MASK);

	young = pmdp_test_and_clear_young(vma, address, pmdp);
	if (young)
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);

	return young;
}

void pmdp_splitting_flush(struct vm_area_struct *vma,
			  unsigned long address, pmd_t *pmdp)
{
	int set;
	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
				(unsigned long *)pmdp);
	if (set) {
		pmd_update(vma->vm_mm, address, pmdp);
		/* need tlb flush only to serialize against gup-fast */
		flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	}
}
#endif

/**
 * reserve_top_address - reserves a hole in the top of kernel address space
 * @reserve - size of hole to reserve
 *
 * Can be used to relocate the fixmap area and poke a hole in the top
 * of kernel address space to make room for a hypervisor.
 */
void __init reserve_top_address(unsigned long reserve)
{
#ifdef CONFIG_X86_32
	BUG_ON(fixmaps_set > 0);
	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
	       (int)-reserve);
	__FIXADDR_TOP = -reserve - PAGE_SIZE;
#endif
}

int fixmaps_set;

void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		BUG();
		return;
	}
	set_pte_vaddr(address, pte);
	fixmaps_set++;
}

void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
		       pgprot_t flags)
{
	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
}
Commit	Line	Data
4f76cd38	1	#include <linux/mm.h>
5a0e3ad6	2	#include <linux/gfp.h>
4f76cd38	3	#include <asm/pgalloc.h>
ee5aa8d3	4	#include <asm/pgtable.h>
4f76cd38	5	#include <asm/tlb.h>
a1d5a869	6	#include <asm/fixmap.h>
4f76cd38	7
9e730237 VN	8	#define PGALLOC_GFP GFP_KERNEL \| __GFP_NOTRACK \| __GFP_REPEAT \| __GFP_ZERO
9e730237 VN	9
14315592 IC	10	#ifdef CONFIG_HIGHPTE
	11	#define PGALLOC_USER_GFP __GFP_HIGHMEM
	12	#else
	13	#define PGALLOC_USER_GFP 0
	14	#endif
	15
	16	gfp_t __userpte_alloc_gfp = PGALLOC_GFP \| PGALLOC_USER_GFP;
	17
4f76cd38 JF	18	pte_t pte_alloc_one_kernel(struct mm_struct mm, unsigned long address)
4f76cd38 JF	19	{
9e730237	20	return (pte_t *)__get_free_page(PGALLOC_GFP);
4f76cd38 JF	21	}
	22
	23	pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
	24	{
	25	struct page *pte;
	26
14315592	27	pte = alloc_pages(__userpte_alloc_gfp, 0);
4f76cd38 JF	28	if (pte)
	29	pgtable_page_ctor(pte);
	30	return pte;
	31	}
	32
14315592 IC	33	static int __init setup_userpte(char *arg)
	34	{
	35	if (!arg)
	36	return -EINVAL;
	37
	38	/*
	39	* "userpte=nohigh" disables allocation of user pagetables in
	40	* high memory.
	41	*/
	42	if (strcmp(arg, "nohigh") == 0)
	43	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
	44	else
	45	return -EINVAL;
	46	return 0;
	47	}
	48	early_param("userpte", setup_userpte);
	49
9e1b32ca	50	void ___pte_free_tlb(struct mmu_gather tlb, struct page pte)
397f687a JF	51	{
397f687a JF	52	pgtable_page_dtor(pte);
6944a9c8	53	paravirt_release_pte(page_to_pfn(pte));
397f687a JF	54	tlb_remove_page(tlb, pte);
	55	}
	56
170fdff7	57	#if PAGETABLE_LEVELS > 2
9e1b32ca	58	void ___pmd_free_tlb(struct mmu_gather tlb, pmd_t pmd)
170fdff7	59	{
6944a9c8	60	paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
170fdff7 JF	61	tlb_remove_page(tlb, virt_to_page(pmd));
170fdff7 JF	62	}
5a5f8f42 JF	63
5a5f8f42 JF	64	#if PAGETABLE_LEVELS > 3
9e1b32ca	65	void ___pud_free_tlb(struct mmu_gather tlb, pud_t pud)
5a5f8f42	66	{
2761fa09	67	paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
5a5f8f42 JF	68	tlb_remove_page(tlb, virt_to_page(pud));
	69	}
	70	#endif /* PAGETABLE_LEVELS > 3 */
170fdff7 JF	71	#endif /* PAGETABLE_LEVELS > 2 */
170fdff7 JF	72
4f76cd38 JF	73	static inline void pgd_list_add(pgd_t *pgd)
	74	{
	75	struct page *page = virt_to_page(pgd);
4f76cd38	76
4f76cd38	77	list_add(&page->lru, &pgd_list);
4f76cd38 JF	78	}
	79
	80	static inline void pgd_list_del(pgd_t *pgd)
	81	{
	82	struct page *page = virt_to_page(pgd);
4f76cd38	83
4f76cd38	84	list_del(&page->lru);
4f76cd38 JF	85	}
4f76cd38 JF	86
4f76cd38	87	#define UNSHARED_PTRS_PER_PGD \
68db065c	88	(SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
4f76cd38	89
617d34d9 JF	90
	91	static void pgd_set_mm(pgd_t pgd, struct mm_struct mm)
	92	{
	93	BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
	94	virt_to_page(pgd)->index = (pgoff_t)mm;
	95	}
	96
	97	struct mm_struct pgd_page_get_mm(struct page page)
	98	{
	99	return (struct mm_struct *)page->index;
	100	}
	101
	102	static void pgd_ctor(struct mm_struct mm, pgd_t pgd)
4f76cd38	103	{
4f76cd38 JF	104	/* If the pgd points to a shared pagetable level (either the
	105	ptes in non-PAE, or shared PMD in PAE), then just copy the
	106	references from swapper_pg_dir. */
	107	if (PAGETABLE_LEVELS == 2 \|\|
85958b46 JF	108	(PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) \|\|
85958b46 JF	109	PAGETABLE_LEVELS == 4) {
68db065c JF	110	clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
68db065c JF	111	swapper_pg_dir + KERNEL_PGD_BOUNDARY,
4f76cd38	112	KERNEL_PGD_PTRS);
4f76cd38 JF	113	}
	114
	115	/* list required to sync kernel mapping updates */
617d34d9 JF	116	if (!SHARED_KERNEL_PMD) {
617d34d9 JF	117	pgd_set_mm(pgd, mm);
4f76cd38	118	pgd_list_add(pgd);
617d34d9	119	}
4f76cd38 JF	120	}
4f76cd38 JF	121
17b74627	122	static void pgd_dtor(pgd_t *pgd)
4f76cd38 JF	123	{
	124	unsigned long flags; /* can be called from interrupt context */
	125
	126	if (SHARED_KERNEL_PMD)
	127	return;
	128
	129	spin_lock_irqsave(&pgd_lock, flags);
	130	pgd_list_del(pgd);
	131	spin_unlock_irqrestore(&pgd_lock, flags);
	132	}
	133
85958b46 JF	134	/*
	135	* List of all pgd's needed for non-PAE so it can invalidate entries
	136	* in both cached and uncached pgd's; not needed for PAE since the
	137	* kernel pmd is shared. If PAE were not to share the pmd a similar
	138	* tactic would be needed. This is essentially codepath-based locking
	139	* against pageattr.c; it is the unique case in which a valid change
	140	* of kernel pagetables can't be lazily synchronized by vmalloc faults.
	141	* vmalloc faults work because attached pagetables are never freed.
	142	* -- wli
	143	*/
	144
4f76cd38	145	#ifdef CONFIG_X86_PAE
d8d5900e JF	146	/*
	147	* In PAE mode, we need to do a cr3 reload (=tlb flush) when
	148	* updating the top-level pagetable entries to guarantee the
	149	* processor notices the update. Since this is expensive, and
	150	* all 4 top-level entries are used almost immediately in a
	151	* new process's life, we just pre-populate them here.
	152	*
	153	* Also, if we're in a paravirt environment where the kernel pmd is
	154	* not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
	155	* and initialize the kernel pmds here.
	156	*/
	157	#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD
	158
	159	void pud_populate(struct mm_struct mm, pud_t pudp, pmd_t *pmd)
	160	{
	161	paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
	162
	163	/* Note: almost everything apart from _PAGE_PRESENT is
	164	reserved at the pmd (PDPT) level. */
	165	set_pud(pudp, __pud(__pa(pmd) \| _PAGE_PRESENT));
	166
	167	/*
	168	* According to Intel App note "TLBs, Paging-Structure Caches,
	169	* and Their Invalidation", April 2007, document 317080-001,
	170	* section 8.1: in PAE mode we explicitly have to flush the
	171	* TLB via cr3 if the top-level pgd is changed...
	172	*/
	173	if (mm == current->active_mm)
	174	write_cr3(read_cr3());
	175	}
	176	#else /* !CONFIG_X86_PAE */
	177
	178	/* No need to prepopulate any pagetable entries in non-PAE modes. */
	179	#define PREALLOCATED_PMDS 0
	180
	181	#endif /* CONFIG_X86_PAE */
	182
	183	static void free_pmds(pmd_t *pmds[])
	184	{
	185	int i;
	186
	187	for(i = 0; i < PREALLOCATED_PMDS; i++)
	188	if (pmds[i])
	189	free_page((unsigned long)pmds[i]);
	190	}
	191
	192	static int preallocate_pmds(pmd_t *pmds[])
	193	{
	194	int i;
	195	bool failed = false;
	196
	197	for(i = 0; i < PREALLOCATED_PMDS; i++) {
9e730237	198	pmd_t pmd = (pmd_t )__get_free_page(PGALLOC_GFP);
d8d5900e JF	199	if (pmd == NULL)
	200	failed = true;
	201	pmds[i] = pmd;
	202	}
	203
	204	if (failed) {
	205	free_pmds(pmds);
	206	return -ENOMEM;
	207	}
	208
	209	return 0;
	210	}
	211
4f76cd38 JF	212	/*
	213	* Mop up any pmd pages which may still be attached to the pgd.
	214	* Normally they will be freed by munmap/exit_mmap, but any pmd we
	215	* preallocate which never got a corresponding vma will need to be
	216	* freed manually.
	217	*/
	218	static void pgd_mop_up_pmds(struct mm_struct mm, pgd_t pgdp)
	219	{
	220	int i;
	221
d8d5900e	222	for(i = 0; i < PREALLOCATED_PMDS; i++) {
4f76cd38 JF	223	pgd_t pgd = pgdp[i];
	224
	225	if (pgd_val(pgd) != 0) {
	226	pmd_t pmd = (pmd_t )pgd_page_vaddr(pgd);
	227
	228	pgdp[i] = native_make_pgd(0);
	229
6944a9c8	230	paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
4f76cd38 JF	231	pmd_free(mm, pmd);
	232	}
	233	}
	234	}
	235
d8d5900e	236	static void pgd_prepopulate_pmd(struct mm_struct mm, pgd_t pgd, pmd_t *pmds[])
4f76cd38 JF	237	{
	238	pud_t *pud;
	239	unsigned long addr;
	240	int i;
	241
cf3e5050 JF	242	if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
	243	return;
	244
4f76cd38	245	pud = pud_offset(pgd, 0);
4f76cd38	246
d8d5900e JF	247	for (addr = i = 0; i < PREALLOCATED_PMDS;
	248	i++, pud++, addr += PUD_SIZE) {
	249	pmd_t *pmd = pmds[i];
4f76cd38	250
68db065c	251	if (i >= KERNEL_PGD_BOUNDARY)
4f76cd38 JF	252	memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
	253	sizeof(pmd_t) * PTRS_PER_PMD);
	254
	255	pud_populate(mm, pud, pmd);
	256	}
4f76cd38	257	}
1ec1fe73	258
d8d5900e	259	pgd_t pgd_alloc(struct mm_struct mm)
1ec1fe73	260	{
d8d5900e JF	261	pgd_t *pgd;
	262	pmd_t *pmds[PREALLOCATED_PMDS];
	263	unsigned long flags;
1ec1fe73	264
9e730237	265	pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
d8d5900e JF	266
	267	if (pgd == NULL)
	268	goto out;
	269
	270	mm->pgd = pgd;
	271
	272	if (preallocate_pmds(pmds) != 0)
	273	goto out_free_pgd;
	274
	275	if (paravirt_pgd_alloc(mm) != 0)
	276	goto out_free_pmds;
1ec1fe73 IM	277
1ec1fe73 IM	278	/*
d8d5900e JF	279	* Make sure that pre-populating the pmds is atomic with
	280	* respect to anything walking the pgd_list, so that they
	281	* never see a partially populated pgd.
1ec1fe73	282	*/
d8d5900e	283	spin_lock_irqsave(&pgd_lock, flags);
4f76cd38	284
617d34d9	285	pgd_ctor(mm, pgd);
d8d5900e	286	pgd_prepopulate_pmd(mm, pgd, pmds);
4f76cd38	287
d8d5900e	288	spin_unlock_irqrestore(&pgd_lock, flags);
4f76cd38 JF	289
4f76cd38 JF	290	return pgd;
d8d5900e JF	291
	292	out_free_pmds:
	293	free_pmds(pmds);
	294	out_free_pgd:
	295	free_page((unsigned long)pgd);
	296	out:
	297	return NULL;
4f76cd38 JF	298	}
	299
	300	void pgd_free(struct mm_struct mm, pgd_t pgd)
	301	{
	302	pgd_mop_up_pmds(mm, pgd);
	303	pgd_dtor(pgd);
eba0045f	304	paravirt_pgd_free(mm, pgd);
4f76cd38 JF	305	free_page((unsigned long)pgd);
4f76cd38 JF	306	}
ee5aa8d3 JF	307
	308	int ptep_set_access_flags(struct vm_area_struct *vma,
	309	unsigned long address, pte_t *ptep,
	310	pte_t entry, int dirty)
	311	{
	312	int changed = !pte_same(*ptep, entry);
	313
	314	if (changed && dirty) {
	315	*ptep = entry;
	316	pte_update_defer(vma->vm_mm, address, ptep);
	317	flush_tlb_page(vma, address);
	318	}
	319
	320	return changed;
	321	}
f9fbf1a3	322
db3eb96f AA	323	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	324	int pmdp_set_access_flags(struct vm_area_struct *vma,
	325	unsigned long address, pmd_t *pmdp,
	326	pmd_t entry, int dirty)
	327	{
	328	int changed = !pmd_same(*pmdp, entry);
	329
	330	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	331
	332	if (changed && dirty) {
	333	*pmdp = entry;
	334	pmd_update_defer(vma->vm_mm, address, pmdp);
	335	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	336	}
	337
	338	return changed;
	339	}
	340	#endif
	341
f9fbf1a3 JF	342	int ptep_test_and_clear_young(struct vm_area_struct *vma,
	343	unsigned long addr, pte_t *ptep)
	344	{
	345	int ret = 0;
	346
	347	if (pte_young(*ptep))
	348	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
48e23957	349	(unsigned long *) &ptep->pte);
f9fbf1a3 JF	350
	351	if (ret)
	352	pte_update(vma->vm_mm, addr, ptep);
	353
	354	return ret;
	355	}
c20311e1	356
db3eb96f AA	357	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	358	int pmdp_test_and_clear_young(struct vm_area_struct *vma,
	359	unsigned long addr, pmd_t *pmdp)
	360	{
	361	int ret = 0;
	362
	363	if (pmd_young(*pmdp))
	364	ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
f2d6bfe9	365	(unsigned long *)pmdp);
db3eb96f AA	366
	367	if (ret)
	368	pmd_update(vma->vm_mm, addr, pmdp);
	369
	370	return ret;
	371	}
	372	#endif
	373
c20311e1 JF	374	int ptep_clear_flush_young(struct vm_area_struct *vma,
	375	unsigned long address, pte_t *ptep)
	376	{
	377	int young;
	378
	379	young = ptep_test_and_clear_young(vma, address, ptep);
	380	if (young)
	381	flush_tlb_page(vma, address);
	382
	383	return young;
	384	}
7c7e6e07	385
db3eb96f AA	386	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	387	int pmdp_clear_flush_young(struct vm_area_struct *vma,
	388	unsigned long address, pmd_t *pmdp)
	389	{
	390	int young;
	391
	392	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	393
	394	young = pmdp_test_and_clear_young(vma, address, pmdp);
	395	if (young)
	396	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	397
	398	return young;
	399	}
	400
	401	void pmdp_splitting_flush(struct vm_area_struct *vma,
	402	unsigned long address, pmd_t *pmdp)
	403	{
	404	int set;
	405	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
	406	set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
f2d6bfe9	407	(unsigned long *)pmdp);
db3eb96f AA	408	if (set) {
	409	pmd_update(vma->vm_mm, address, pmdp);
	410	/* need tlb flush only to serialize against gup-fast */
	411	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
	412	}
	413	}
	414	#endif
	415
fd862dde GP	416	/**
	417	* reserve_top_address - reserves a hole in the top of kernel address space
	418	* @reserve - size of hole to reserve
	419	*
	420	* Can be used to relocate the fixmap area and poke a hole in the top
	421	* of kernel address space to make room for a hypervisor.
	422	*/
	423	void __init reserve_top_address(unsigned long reserve)
	424	{
	425	#ifdef CONFIG_X86_32
	426	BUG_ON(fixmaps_set > 0);
	427	printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
	428	(int)-reserve);
	429	__FIXADDR_TOP = -reserve - PAGE_SIZE;
fd862dde GP	430	#endif
	431	}
	432
7c7e6e07 JF	433	int fixmaps_set;
7c7e6e07 JF	434
aeaaa59c	435	void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
7c7e6e07 JF	436	{
	437	unsigned long address = __fix_to_virt(idx);
	438
	439	if (idx >= __end_of_fixed_addresses) {
	440	BUG();
	441	return;
	442	}
aeaaa59c	443	set_pte_vaddr(address, pte);
7c7e6e07 JF	444	fixmaps_set++;
7c7e6e07 JF	445	}
aeaaa59c	446
3b3809ac MH	447	void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
3b3809ac MH	448	pgprot_t flags)
aeaaa59c JF	449	{
	450	__native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
	451	}