[linux-2.6-block.git] / arch / x86_64 / mm / init.c

/*
 *  linux/arch/x86_64/mm/init.c
 *
 *  Copyright (C) 1995  Linus Torvalds
 *  Copyright (C) 2000  Pavel Machek <pavel@suse.cz>
 *  Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
 */

#include <linux/config.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/bootmem.h>
#include <linux/proc_fs.h>

#include <asm/processor.h>
#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/dma.h>
#include <asm/fixmap.h>
#include <asm/e820.h>
#include <asm/apic.h>
#include <asm/tlb.h>
#include <asm/mmu_context.h>
#include <asm/proto.h>
#include <asm/smp.h>

#ifndef Dprintk
#define Dprintk(x...)
#endif

#ifdef CONFIG_GART_IOMMU
extern int swiotlb;
#endif

extern char _stext[];

DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);

/*
 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
 * physical space so we can cache the place of the first one and move
 * around without checking the pgd every time.
 */

void show_mem(void)
{
	long i, total = 0, reserved = 0;
	long shared = 0, cached = 0;
	pg_data_t *pgdat;
	struct page *page;

	printk(KERN_INFO "Mem-info:\n");
	show_free_areas();
	printk(KERN_INFO "Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));

	for_each_pgdat(pgdat) {
               for (i = 0; i < pgdat->node_spanned_pages; ++i) {
			page = pfn_to_page(pgdat->node_start_pfn + i);
			total++;
			if (PageReserved(page))
				reserved++;
			else if (PageSwapCache(page))
				cached++;
			else if (page_count(page))
				shared += page_count(page) - 1;
               }
	}
	printk(KERN_INFO "%lu pages of RAM\n", total);
	printk(KERN_INFO "%lu reserved pages\n",reserved);
	printk(KERN_INFO "%lu pages shared\n",shared);
	printk(KERN_INFO "%lu pages swap cached\n",cached);
}

/* References to section boundaries */

extern char _text, _etext, _edata, __bss_start, _end[];
extern char __init_begin, __init_end;

int after_bootmem;

static void *spp_getpage(void)
{ 
	void *ptr;
	if (after_bootmem)
		ptr = (void *) get_zeroed_page(GFP_ATOMIC); 
	else
		ptr = alloc_bootmem_pages(PAGE_SIZE);
	if (!ptr || ((unsigned long)ptr & ~PAGE_MASK))
		panic("set_pte_phys: cannot allocate page data %s\n", after_bootmem?"after bootmem":"");

	Dprintk("spp_getpage %p\n", ptr);
	return ptr;
} 

static void set_pte_phys(unsigned long vaddr,
			 unsigned long phys, pgprot_t prot)
{
	pgd_t *pgd;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte, new_pte;

	Dprintk("set_pte_phys %lx to %lx\n", vaddr, phys);

	pgd = pgd_offset_k(vaddr);
	if (pgd_none(*pgd)) {
		printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
		return;
	}
	pud = pud_offset(pgd, vaddr);
	if (pud_none(*pud)) {
		pmd = (pmd_t *) spp_getpage(); 
		set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | _PAGE_USER));
		if (pmd != pmd_offset(pud, 0)) {
			printk("PAGETABLE BUG #01! %p <-> %p\n", pmd, pmd_offset(pud,0));
			return;
		}
	}
	pmd = pmd_offset(pud, vaddr);
	if (pmd_none(*pmd)) {
		pte = (pte_t *) spp_getpage();
		set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE | _PAGE_USER));
		if (pte != pte_offset_kernel(pmd, 0)) {
			printk("PAGETABLE BUG #02!\n");
			return;
		}
	}
	new_pte = pfn_pte(phys >> PAGE_SHIFT, prot);

	pte = pte_offset_kernel(pmd, vaddr);
	if (!pte_none(*pte) &&
	    pte_val(*pte) != (pte_val(new_pte) & __supported_pte_mask))
		pte_ERROR(*pte);
	set_pte(pte, new_pte);

	/*
	 * It's enough to flush this one mapping.
	 * (PGE mappings get flushed as well)
	 */
	__flush_tlb_one(vaddr);
}

/* NOTE: this is meant to be run only at boot */
void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
{
	unsigned long address = __fix_to_virt(idx);

	if (idx >= __end_of_fixed_addresses) {
		printk("Invalid __set_fixmap\n");
		return;
	}
	set_pte_phys(address, phys, prot);
}

unsigned long __initdata table_start, table_end; 

extern pmd_t temp_boot_pmds[]; 

static  struct temp_map { 
	pmd_t *pmd;
	void  *address; 
	int    allocated; 
} temp_mappings[] __initdata = { 
	{ &temp_boot_pmds[0], (void *)(40UL * 1024 * 1024) },
	{ &temp_boot_pmds[1], (void *)(42UL * 1024 * 1024) }, 
	{}
}; 

static __init void *alloc_low_page(int *index, unsigned long *phys) 
{ 
	struct temp_map *ti;
	int i; 
	unsigned long pfn = table_end++, paddr; 
	void *adr;

	if (pfn >= end_pfn) 
		panic("alloc_low_page: ran out of memory"); 
	for (i = 0; temp_mappings[i].allocated; i++) {
		if (!temp_mappings[i].pmd) 
			panic("alloc_low_page: ran out of temp mappings"); 
	} 
	ti = &temp_mappings[i];
	paddr = (pfn << PAGE_SHIFT) & PMD_MASK; 
	set_pmd(ti->pmd, __pmd(paddr | _KERNPG_TABLE | _PAGE_PSE)); 
	ti->allocated = 1; 
	__flush_tlb(); 	       
	adr = ti->address + ((pfn << PAGE_SHIFT) & ~PMD_MASK); 
	*index = i; 
	*phys  = pfn * PAGE_SIZE;  
	return adr; 
} 

static __init void unmap_low_page(int i)
{ 
	struct temp_map *ti = &temp_mappings[i];
	set_pmd(ti->pmd, __pmd(0));
	ti->allocated = 0; 
} 

static void __init phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
{ 
	long i, j; 

	i = pud_index(address);
	pud = pud + i;
	for (; i < PTRS_PER_PUD; pud++, i++) {
		int map; 
		unsigned long paddr, pmd_phys;
		pmd_t *pmd;

		paddr = address + i*PUD_SIZE;
		if (paddr >= end) { 
			for (; i < PTRS_PER_PUD; i++, pud++) 
				set_pud(pud, __pud(0)); 
			break;
		} 

		if (!e820_mapped(paddr, paddr+PUD_SIZE, 0)) { 
			set_pud(pud, __pud(0)); 
			continue;
		} 

		pmd = alloc_low_page(&map, &pmd_phys);
		set_pud(pud, __pud(pmd_phys | _KERNPG_TABLE));
		for (j = 0; j < PTRS_PER_PMD; pmd++, j++, paddr += PMD_SIZE) {
			unsigned long pe;

			if (paddr >= end) { 
				for (; j < PTRS_PER_PMD; j++, pmd++)
					set_pmd(pmd,  __pmd(0)); 
				break;
		}
			pe = _PAGE_NX|_PAGE_PSE | _KERNPG_TABLE | _PAGE_GLOBAL | paddr;
			pe &= __supported_pte_mask;
			set_pmd(pmd, __pmd(pe));
		}
		unmap_low_page(map);
	}
	__flush_tlb();
} 

static void __init find_early_table_space(unsigned long end)
{
	unsigned long puds, pmds, tables;

	puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
	pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
	tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
		 round_up(pmds * sizeof(pmd_t), PAGE_SIZE);

	table_start = find_e820_area(0x8000, __pa_symbol(&_text), tables);
	if (table_start == -1UL)
		panic("Cannot find space for the kernel page tables");

	table_start >>= PAGE_SHIFT;
	table_end = table_start;
}

/* Setup the direct mapping of the physical memory at PAGE_OFFSET.
   This runs before bootmem is initialized and gets pages directly from the 
   physical memory. To access them they are temporarily mapped. */
void __init init_memory_mapping(unsigned long start, unsigned long end)
{ 
	unsigned long next; 

	Dprintk("init_memory_mapping\n");

	/* 
	 * Find space for the kernel direct mapping tables.
	 * Later we should allocate these tables in the local node of the memory
	 * mapped.  Unfortunately this is done currently before the nodes are 
	 * discovered.
	 */
	find_early_table_space(end);

	start = (unsigned long)__va(start);
	end = (unsigned long)__va(end);

	for (; start < end; start = next) {
		int map;
		unsigned long pud_phys; 
		pud_t *pud = alloc_low_page(&map, &pud_phys);
		next = start + PGDIR_SIZE;
		if (next > end) 
			next = end; 
		phys_pud_init(pud, __pa(start), __pa(next));
		set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
		unmap_low_page(map);   
	} 

	asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
	__flush_tlb_all();
	early_printk("kernel direct mapping tables upto %lx @ %lx-%lx\n", end, 
	       table_start<<PAGE_SHIFT, 
	       table_end<<PAGE_SHIFT);
}

extern struct x8664_pda cpu_pda[NR_CPUS];

/* Assumes all CPUs still execute in init_mm */
void zap_low_mappings(void)
{
	pgd_t *pgd = pgd_offset_k(0UL);
	pgd_clear(pgd);
	flush_tlb_all();
}

/* Compute zone sizes for the DMA and DMA32 zones in a node. */
__init void
size_zones(unsigned long *z, unsigned long *h,
	   unsigned long start_pfn, unsigned long end_pfn)
{
 	int i;
 	unsigned long w;

 	for (i = 0; i < MAX_NR_ZONES; i++)
 		z[i] = 0;

 	if (start_pfn < MAX_DMA_PFN)
 		z[ZONE_DMA] = MAX_DMA_PFN - start_pfn;
 	if (start_pfn < MAX_DMA32_PFN) {
 		unsigned long dma32_pfn = MAX_DMA32_PFN;
 		if (dma32_pfn > end_pfn)
 			dma32_pfn = end_pfn;
 		z[ZONE_DMA32] = dma32_pfn - start_pfn;
 	}
 	z[ZONE_NORMAL] = end_pfn - start_pfn;

 	/* Remove lower zones from higher ones. */
 	w = 0;
 	for (i = 0; i < MAX_NR_ZONES; i++) {
 		if (z[i])
 			z[i] -= w;
 	        w += z[i];
	}

	/* Compute holes */
	w = 0;
	for (i = 0; i < MAX_NR_ZONES; i++) {
		unsigned long s = w;
		w += z[i];
		h[i] = e820_hole_size(s, w);
	}
}

#ifndef CONFIG_NUMA
void __init paging_init(void)
{
	unsigned long zones[MAX_NR_ZONES], holes[MAX_NR_ZONES];
	size_zones(zones, holes, 0, end_pfn);
	free_area_init_node(0, NODE_DATA(0), zones,
			    __pa(PAGE_OFFSET) >> PAGE_SHIFT, holes);
}
#endif

/* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
   from the CPU leading to inconsistent cache lines. address and size
   must be aligned to 2MB boundaries. 
   Does nothing when the mapping doesn't exist. */
void __init clear_kernel_mapping(unsigned long address, unsigned long size) 
{
	unsigned long end = address + size;

	BUG_ON(address & ~LARGE_PAGE_MASK);
	BUG_ON(size & ~LARGE_PAGE_MASK); 
	
	for (; address < end; address += LARGE_PAGE_SIZE) { 
		pgd_t *pgd = pgd_offset_k(address);
		pud_t *pud;
		pmd_t *pmd;
		if (pgd_none(*pgd))
			continue;
		pud = pud_offset(pgd, address);
		if (pud_none(*pud))
			continue; 
		pmd = pmd_offset(pud, address);
		if (!pmd || pmd_none(*pmd))
			continue; 
		if (0 == (pmd_val(*pmd) & _PAGE_PSE)) { 
			/* Could handle this, but it should not happen currently. */
			printk(KERN_ERR 
	       "clear_kernel_mapping: mapping has been split. will leak memory\n"); 
			pmd_ERROR(*pmd); 
		}
		set_pmd(pmd, __pmd(0)); 		
	}
	__flush_tlb_all();
} 

static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
			 kcore_vsyscall;

void __init mem_init(void)
{
	long codesize, reservedpages, datasize, initsize;

#ifdef CONFIG_SWIOTLB
	if (!iommu_aperture &&
	    (end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
	       swiotlb = 1;
	if (swiotlb)
		swiotlb_init();	
#endif

	/* How many end-of-memory variables you have, grandma! */
	max_low_pfn = end_pfn;
	max_pfn = end_pfn;
	num_physpages = end_pfn;
	high_memory = (void *) __va(end_pfn * PAGE_SIZE);

	/* clear the zero-page */
	memset(empty_zero_page, 0, PAGE_SIZE);

	reservedpages = 0;

	/* this will put all low memory onto the freelists */
#ifdef CONFIG_NUMA
	totalram_pages = numa_free_all_bootmem();
#else
	totalram_pages = free_all_bootmem();
#endif
	reservedpages = end_pfn - totalram_pages - e820_hole_size(0, end_pfn);

	after_bootmem = 1;

	codesize =  (unsigned long) &_etext - (unsigned long) &_text;
	datasize =  (unsigned long) &_edata - (unsigned long) &_etext;
	initsize =  (unsigned long) &__init_end - (unsigned long) &__init_begin;

	/* Register memory areas for /proc/kcore */
	kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT); 
	kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, 
		   VMALLOC_END-VMALLOC_START);
	kclist_add(&kcore_kernel, &_stext, _end - _stext);
	kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
	kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START, 
				 VSYSCALL_END - VSYSCALL_START);

	printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, %ldk data, %ldk init)\n",
		(unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
		end_pfn << (PAGE_SHIFT-10),
		codesize >> 10,
		reservedpages << (PAGE_SHIFT-10),
		datasize >> 10,
		initsize >> 10);

	/*
	 * Subtle. SMP is doing its boot stuff late (because it has to
	 * fork idle threads) - but it also needs low mappings for the
	 * protected-mode entry to work. We zap these entries only after
	 * the WP-bit has been tested.
	 */
#ifndef CONFIG_SMP
	zap_low_mappings();
#endif
}

extern char __initdata_begin[], __initdata_end[];

void free_initmem(void)
{
	unsigned long addr;

	addr = (unsigned long)(&__init_begin);
	for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
		ClearPageReserved(virt_to_page(addr));
		set_page_count(virt_to_page(addr), 1);
		memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE); 
		free_page(addr);
		totalram_pages++;
	}
	memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
	printk ("Freeing unused kernel memory: %luk freed\n", (&__init_end - &__init_begin) >> 10);
}

#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
	if (start < (unsigned long)&_end)
		return;
	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
	for (; start < end; start += PAGE_SIZE) {
		ClearPageReserved(virt_to_page(start));
		set_page_count(virt_to_page(start), 1);
		free_page(start);
		totalram_pages++;
	}
}
#endif

void __init reserve_bootmem_generic(unsigned long phys, unsigned len) 
{ 
	/* Should check here against the e820 map to avoid double free */ 
#ifdef CONFIG_NUMA
	int nid = phys_to_nid(phys);
  	reserve_bootmem_node(NODE_DATA(nid), phys, len);
#else       		
	reserve_bootmem(phys, len);    
#endif
}

int kern_addr_valid(unsigned long addr) 
{ 
	unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
       pgd_t *pgd;
       pud_t *pud;
       pmd_t *pmd;
       pte_t *pte;

	if (above != 0 && above != -1UL)
		return 0; 
	
	pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd))
		return 0;

	pud = pud_offset(pgd, addr);
	if (pud_none(*pud))
		return 0; 

	pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd))
		return 0;
	if (pmd_large(*pmd))
		return pfn_valid(pmd_pfn(*pmd));

	pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte))
		return 0;
	return pfn_valid(pte_pfn(*pte));
}

#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>

extern int exception_trace, page_fault_trace;

static ctl_table debug_table2[] = {
	{ 99, "exception-trace", &exception_trace, sizeof(int), 0644, NULL,
	  proc_dointvec },
#ifdef CONFIG_CHECKING
	{ 100, "page-fault-trace", &page_fault_trace, sizeof(int), 0644, NULL,
	  proc_dointvec },
#endif
	{ 0, }
}; 

static ctl_table debug_root_table2[] = { 
	{ .ctl_name = CTL_DEBUG, .procname = "debug", .mode = 0555, 
	   .child = debug_table2 }, 
	{ 0 }, 
}; 

static __init int x8664_sysctl_init(void)
{ 
	register_sysctl_table(debug_root_table2, 1);
	return 0;
}
__initcall(x8664_sysctl_init);
#endif

/* A pseudo VMAs to allow ptrace access for the vsyscall page.   This only
   covers the 64bit vsyscall page now. 32bit has a real VMA now and does
   not need special handling anymore. */

static struct vm_area_struct gate_vma = {
	.vm_start = VSYSCALL_START,
	.vm_end = VSYSCALL_END,
	.vm_page_prot = PAGE_READONLY
};

struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
{
#ifdef CONFIG_IA32_EMULATION
	if (test_tsk_thread_flag(tsk, TIF_IA32))
		return NULL;
#endif
	return &gate_vma;
}

int in_gate_area(struct task_struct *task, unsigned long addr)
{
	struct vm_area_struct *vma = get_gate_vma(task);
	if (!vma)
		return 0;
	return (addr >= vma->vm_start) && (addr < vma->vm_end);
}

/* Use this when you have no reliable task/vma, typically from interrupt
 * context.  It is less reliable than using the task's vma and may give
 * false positives.
 */
int in_gate_area_no_task(unsigned long addr)
{
	return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/x86_64/mm/init.c
	3	*
	4	* Copyright (C) 1995 Linus Torvalds
	5	* Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
	6	* Copyright (C) 2002,2003 Andi Kleen <ak@suse.de>
	7	*/
	8
	9	#include <linux/config.h>
	10	#include <linux/signal.h>
	11	#include <linux/sched.h>
	12	#include <linux/kernel.h>
	13	#include <linux/errno.h>
	14	#include <linux/string.h>
	15	#include <linux/types.h>
	16	#include <linux/ptrace.h>
	17	#include <linux/mman.h>
	18	#include <linux/mm.h>
	19	#include <linux/swap.h>
	20	#include <linux/smp.h>
	21	#include <linux/init.h>
	22	#include <linux/pagemap.h>
	23	#include <linux/bootmem.h>
	24	#include <linux/proc_fs.h>
	25
	26	#include <asm/processor.h>
	27	#include <asm/system.h>
	28	#include <asm/uaccess.h>
	29	#include <asm/pgtable.h>
	30	#include <asm/pgalloc.h>
	31	#include <asm/dma.h>
	32	#include <asm/fixmap.h>
	33	#include <asm/e820.h>
	34	#include <asm/apic.h>
	35	#include <asm/tlb.h>
	36	#include <asm/mmu_context.h>
	37	#include <asm/proto.h>
	38	#include <asm/smp.h>
	39
	40	#ifndef Dprintk
	41	#define Dprintk(x...)
	42	#endif
	43
	44	#ifdef CONFIG_GART_IOMMU
	45	extern int swiotlb;
	46	#endif
	47
	48	extern char _stext[];
	49
	50	DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
	51
	52	/*
	53	* NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the
	54	* physical space so we can cache the place of the first one and move
	55	* around without checking the pgd every time.
	56	*/
	57
	58	void show_mem(void)
	59	{
e92343cc AK	60	long i, total = 0, reserved = 0;
e92343cc AK	61	long shared = 0, cached = 0;
1da177e4 LT	62	pg_data_t *pgdat;
	63	struct page *page;
	64
e92343cc	65	printk(KERN_INFO "Mem-info:\n");
1da177e4	66	show_free_areas();
e92343cc	67	printk(KERN_INFO "Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
1da177e4 LT	68
	69	for_each_pgdat(pgdat) {
	70	for (i = 0; i < pgdat->node_spanned_pages; ++i) {
	71	page = pfn_to_page(pgdat->node_start_pfn + i);
	72	total++;
e92343cc AK	73	if (PageReserved(page))
	74	reserved++;
	75	else if (PageSwapCache(page))
	76	cached++;
	77	else if (page_count(page))
	78	shared += page_count(page) - 1;
1da177e4 LT	79	}
1da177e4 LT	80	}
e92343cc AK	81	printk(KERN_INFO "%lu pages of RAM\n", total);
	82	printk(KERN_INFO "%lu reserved pages\n",reserved);
	83	printk(KERN_INFO "%lu pages shared\n",shared);
	84	printk(KERN_INFO "%lu pages swap cached\n",cached);
1da177e4 LT	85	}
	86
	87	/* References to section boundaries */
	88
	89	extern char _text, _etext, _edata, __bss_start, _end[];
	90	extern char __init_begin, __init_end;
	91
	92	int after_bootmem;
	93
	94	static void *spp_getpage(void)
	95	{
	96	void *ptr;
	97	if (after_bootmem)
	98	ptr = (void *) get_zeroed_page(GFP_ATOMIC);
	99	else
	100	ptr = alloc_bootmem_pages(PAGE_SIZE);
	101	if (!ptr \|\| ((unsigned long)ptr & ~PAGE_MASK))
	102	panic("set_pte_phys: cannot allocate page data %s\n", after_bootmem?"after bootmem":"");
	103
	104	Dprintk("spp_getpage %p\n", ptr);
	105	return ptr;
	106	}
	107
	108	static void set_pte_phys(unsigned long vaddr,
	109	unsigned long phys, pgprot_t prot)
	110	{
	111	pgd_t *pgd;
	112	pud_t *pud;
	113	pmd_t *pmd;
	114	pte_t *pte, new_pte;
	115
	116	Dprintk("set_pte_phys %lx to %lx\n", vaddr, phys);
	117
	118	pgd = pgd_offset_k(vaddr);
	119	if (pgd_none(*pgd)) {
	120	printk("PGD FIXMAP MISSING, it should be setup in head.S!\n");
	121	return;
	122	}
	123	pud = pud_offset(pgd, vaddr);
	124	if (pud_none(*pud)) {
	125	pmd = (pmd_t *) spp_getpage();
	126	set_pud(pud, __pud(__pa(pmd) \| _KERNPG_TABLE \| _PAGE_USER));
	127	if (pmd != pmd_offset(pud, 0)) {
	128	printk("PAGETABLE BUG #01! %p <-> %p\n", pmd, pmd_offset(pud,0));
	129	return;
	130	}
	131	}
	132	pmd = pmd_offset(pud, vaddr);
	133	if (pmd_none(*pmd)) {
	134	pte = (pte_t *) spp_getpage();
	135	set_pmd(pmd, __pmd(__pa(pte) \| _KERNPG_TABLE \| _PAGE_USER));
	136	if (pte != pte_offset_kernel(pmd, 0)) {
	137	printk("PAGETABLE BUG #02!\n");
	138	return;
	139	}
	140	}
	141	new_pte = pfn_pte(phys >> PAGE_SHIFT, prot);
	142
	143	pte = pte_offset_kernel(pmd, vaddr);
	144	if (!pte_none(*pte) &&
	145	pte_val(*pte) != (pte_val(new_pte) & __supported_pte_mask))
	146	pte_ERROR(*pte);
	147	set_pte(pte, new_pte);
	148
149	/*
150	* It's enough to flush this one mapping.
151	* (PGE mappings get flushed as well)
152	*/
153	__flush_tlb_one(vaddr);
154	}
155
156	/* NOTE: this is meant to be run only at boot */
157	void __set_fixmap (enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
158	{
159	unsigned long address = __fix_to_virt(idx);
160
161	if (idx >= __end_of_fixed_addresses) {
162	printk("Invalid __set_fixmap\n");
163	return;
164	}
165	set_pte_phys(address, phys, prot);
166	}
167
168	unsigned long __initdata table_start, table_end;
169
170	extern pmd_t temp_boot_pmds[];
171
172	static struct temp_map {
173	pmd_t *pmd;
174	void *address;
175	int allocated;
176	} temp_mappings[] __initdata = {
177	{ &temp_boot_pmds[0], (void )(40UL 1024 * 1024) },
178	{ &temp_boot_pmds[1], (void )(42UL 1024 * 1024) },
179	{}
180	};
181
182	static __init void alloc_low_page(int index, unsigned long *phys)
183	{
184	struct temp_map *ti;
185	int i;
186	unsigned long pfn = table_end++, paddr;
187	void *adr;
188
189	if (pfn >= end_pfn)
190	panic("alloc_low_page: ran out of memory");
191	for (i = 0; temp_mappings[i].allocated; i++) {
192	if (!temp_mappings[i].pmd)
193	panic("alloc_low_page: ran out of temp mappings");
194	}
195	ti = &temp_mappings[i];
196	paddr = (pfn << PAGE_SHIFT) & PMD_MASK;
197	set_pmd(ti->pmd, __pmd(paddr \| _KERNPG_TABLE \| _PAGE_PSE));
198	ti->allocated = 1;
199	__flush_tlb();
200	adr = ti->address + ((pfn << PAGE_SHIFT) & ~PMD_MASK);
201	*index = i;
202	phys = pfn PAGE_SIZE;
203	return adr;
204	}
205
206	static __init void unmap_low_page(int i)
207	{
208	struct temp_map *ti = &temp_mappings[i];
209	set_pmd(ti->pmd, __pmd(0));
210	ti->allocated = 0;
211	}
212
213	static void __init phys_pud_init(pud_t *pud, unsigned long address, unsigned long end)
214	{
215	long i, j;
216
217	i = pud_index(address);
218	pud = pud + i;
219	for (; i < PTRS_PER_PUD; pud++, i++) {
220	int map;
221	unsigned long paddr, pmd_phys;
222	pmd_t *pmd;
223
224	paddr = address + i*PUD_SIZE;
225	if (paddr >= end) {
226	for (; i < PTRS_PER_PUD; i++, pud++)
227	set_pud(pud, __pud(0));
228	break;
229	}
230
231	if (!e820_mapped(paddr, paddr+PUD_SIZE, 0)) {
232	set_pud(pud, __pud(0));
233	continue;
234	}
235
236	pmd = alloc_low_page(&map, &pmd_phys);
237	set_pud(pud, __pud(pmd_phys \| _KERNPG_TABLE));
238	for (j = 0; j < PTRS_PER_PMD; pmd++, j++, paddr += PMD_SIZE) {
239	unsigned long pe;
240
241	if (paddr >= end) {
242	for (; j < PTRS_PER_PMD; j++, pmd++)
243	set_pmd(pmd, __pmd(0));
244	break;
245	}
246	pe = _PAGE_NX\|_PAGE_PSE \| _KERNPG_TABLE \| _PAGE_GLOBAL \| paddr;
247	pe &= __supported_pte_mask;
248	set_pmd(pmd, __pmd(pe));
249	}
250	unmap_low_page(map);
251	}
252	__flush_tlb();
253	}
254
255	static void __init find_early_table_space(unsigned long end)
256	{
257	unsigned long puds, pmds, tables;
258
259	puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
260	pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
261	tables = round_up(puds * sizeof(pud_t), PAGE_SIZE) +
262	round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
263
264	table_start = find_e820_area(0x8000, __pa_symbol(&_text), tables);
265	if (table_start == -1UL)
266	panic("Cannot find space for the kernel page tables");
267
268	table_start >>= PAGE_SHIFT;
269	table_end = table_start;
270	}
271
272	/* Setup the direct mapping of the physical memory at PAGE_OFFSET.
273	This runs before bootmem is initialized and gets pages directly from the
274	physical memory. To access them they are temporarily mapped. */
275	void __init init_memory_mapping(unsigned long start, unsigned long end)
276	{
277	unsigned long next;
278
279	Dprintk("init_memory_mapping\n");
280
281	/*
282	* Find space for the kernel direct mapping tables.
283	* Later we should allocate these tables in the local node of the memory
284	* mapped. Unfortunately this is done currently before the nodes are
285	* discovered.
286	*/
287	find_early_table_space(end);
288
289	start = (unsigned long)__va(start);
290	end = (unsigned long)__va(end);
291
292	for (; start < end; start = next) {
293	int map;
294	unsigned long pud_phys;
295	pud_t *pud = alloc_low_page(&map, &pud_phys);
296	next = start + PGDIR_SIZE;
297	if (next > end)
298	next = end;
299	phys_pud_init(pud, __pa(start), __pa(next));
300	set_pgd(pgd_offset_k(start), mk_kernel_pgd(pud_phys));
301	unmap_low_page(map);
302	}
303
304	asm volatile("movq %%cr4,%0" : "=r" (mmu_cr4_features));
305	__flush_tlb_all();
306	early_printk("kernel direct mapping tables upto %lx @ %lx-%lx\n", end,
307	table_start<<PAGE_SHIFT,
308	table_end<<PAGE_SHIFT);
309	}
310
311	extern struct x8664_pda cpu_pda[NR_CPUS];
312
313	/* Assumes all CPUs still execute in init_mm */
314	void zap_low_mappings(void)
315	{
316	pgd_t *pgd = pgd_offset_k(0UL);
317	pgd_clear(pgd);
318	flush_tlb_all();
319	}
320
a2f1b424 AK	321	/* Compute zone sizes for the DMA and DMA32 zones in a node. */
	322	__init void
	323	size_zones(unsigned long z, unsigned long h,
	324	unsigned long start_pfn, unsigned long end_pfn)
	325	{
	326	int i;
	327	unsigned long w;
	328
	329	for (i = 0; i < MAX_NR_ZONES; i++)
	330	z[i] = 0;
	331
	332	if (start_pfn < MAX_DMA_PFN)
	333	z[ZONE_DMA] = MAX_DMA_PFN - start_pfn;
	334	if (start_pfn < MAX_DMA32_PFN) {
	335	unsigned long dma32_pfn = MAX_DMA32_PFN;
	336	if (dma32_pfn > end_pfn)
	337	dma32_pfn = end_pfn;
	338	z[ZONE_DMA32] = dma32_pfn - start_pfn;
	339	}
	340	z[ZONE_NORMAL] = end_pfn - start_pfn;
	341
	342	/* Remove lower zones from higher ones. */
	343	w = 0;
	344	for (i = 0; i < MAX_NR_ZONES; i++) {
	345	if (z[i])
	346	z[i] -= w;
	347	w += z[i];
	348	}
	349
	350	/* Compute holes */
	351	w = 0;
	352	for (i = 0; i < MAX_NR_ZONES; i++) {
	353	unsigned long s = w;
	354	w += z[i];
	355	h[i] = e820_hole_size(s, w);
	356	}
	357	}
	358
2b97690f	359	#ifndef CONFIG_NUMA
1da177e4 LT	360	void __init paging_init(void)
1da177e4 LT	361	{
a2f1b424 AK	362	unsigned long zones[MAX_NR_ZONES], holes[MAX_NR_ZONES];
	363	size_zones(zones, holes, 0, end_pfn);
	364	free_area_init_node(0, NODE_DATA(0), zones,
	365	__pa(PAGE_OFFSET) >> PAGE_SHIFT, holes);
1da177e4 LT	366	}
	367	#endif
	368
	369	/* Unmap a kernel mapping if it exists. This is useful to avoid prefetches
	370	from the CPU leading to inconsistent cache lines. address and size
	371	must be aligned to 2MB boundaries.
	372	Does nothing when the mapping doesn't exist. */
	373	void __init clear_kernel_mapping(unsigned long address, unsigned long size)
	374	{
	375	unsigned long end = address + size;
	376
	377	BUG_ON(address & ~LARGE_PAGE_MASK);
	378	BUG_ON(size & ~LARGE_PAGE_MASK);
	379
	380	for (; address < end; address += LARGE_PAGE_SIZE) {
	381	pgd_t *pgd = pgd_offset_k(address);
	382	pud_t *pud;
	383	pmd_t *pmd;
	384	if (pgd_none(*pgd))
	385	continue;
	386	pud = pud_offset(pgd, address);
	387	if (pud_none(*pud))
	388	continue;
	389	pmd = pmd_offset(pud, address);
	390	if (!pmd \|\| pmd_none(*pmd))
	391	continue;
	392	if (0 == (pmd_val(*pmd) & _PAGE_PSE)) {
	393	/* Could handle this, but it should not happen currently. */
	394	printk(KERN_ERR
	395	"clear_kernel_mapping: mapping has been split. will leak memory\n");
	396	pmd_ERROR(*pmd);
	397	}
	398	set_pmd(pmd, __pmd(0));
	399	}
	400	__flush_tlb_all();
	401	}
	402
1da177e4 LT	403	static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
	404	kcore_vsyscall;
	405
	406	void __init mem_init(void)
	407	{
0a43e4bf	408	long codesize, reservedpages, datasize, initsize;
1da177e4 LT	409
1da177e4 LT	410	#ifdef CONFIG_SWIOTLB
1da177e4 LT	411	if (!iommu_aperture &&
	412	(end_pfn >= 0xffffffff>>PAGE_SHIFT \|\| force_iommu))
	413	swiotlb = 1;
	414	if (swiotlb)
	415	swiotlb_init();
	416	#endif
	417
	418	/* How many end-of-memory variables you have, grandma! */
	419	max_low_pfn = end_pfn;
	420	max_pfn = end_pfn;
	421	num_physpages = end_pfn;
	422	high_memory = (void ) __va(end_pfn PAGE_SIZE);
	423
	424	/* clear the zero-page */
	425	memset(empty_zero_page, 0, PAGE_SIZE);
	426
	427	reservedpages = 0;
	428
	429	/* this will put all low memory onto the freelists */
2b97690f	430	#ifdef CONFIG_NUMA
0a43e4bf	431	totalram_pages = numa_free_all_bootmem();
1da177e4	432	#else
0a43e4bf	433	totalram_pages = free_all_bootmem();
1da177e4	434	#endif
0a43e4bf	435	reservedpages = end_pfn - totalram_pages - e820_hole_size(0, end_pfn);
1da177e4 LT	436
	437	after_bootmem = 1;
	438
	439	codesize = (unsigned long) &_etext - (unsigned long) &_text;
	440	datasize = (unsigned long) &_edata - (unsigned long) &_etext;
	441	initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
	442
	443	/* Register memory areas for /proc/kcore */
	444	kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
	445	kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
	446	VMALLOC_END-VMALLOC_START);
	447	kclist_add(&kcore_kernel, &_stext, _end - _stext);
	448	kclist_add(&kcore_modules, (void *)MODULES_VADDR, MODULES_LEN);
	449	kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START,
	450	VSYSCALL_END - VSYSCALL_START);
	451
0a43e4bf	452	printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, %ldk data, %ldk init)\n",
1da177e4 LT	453	(unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
	454	end_pfn << (PAGE_SHIFT-10),
	455	codesize >> 10,
	456	reservedpages << (PAGE_SHIFT-10),
	457	datasize >> 10,
	458	initsize >> 10);
	459
	460	/*
	461	* Subtle. SMP is doing its boot stuff late (because it has to
	462	* fork idle threads) - but it also needs low mappings for the
	463	* protected-mode entry to work. We zap these entries only after
	464	* the WP-bit has been tested.
	465	*/
	466	#ifndef CONFIG_SMP
	467	zap_low_mappings();
	468	#endif
	469	}
	470
	471	extern char __initdata_begin[], __initdata_end[];
	472
	473	void free_initmem(void)
	474	{
	475	unsigned long addr;
	476
	477	addr = (unsigned long)(&__init_begin);
	478	for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
	479	ClearPageReserved(virt_to_page(addr));
	480	set_page_count(virt_to_page(addr), 1);
	481	memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE);
	482	free_page(addr);
	483	totalram_pages++;
	484	}
	485	memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
	486	printk ("Freeing unused kernel memory: %luk freed\n", (&__init_end - &__init_begin) >> 10);
	487	}
	488
	489	#ifdef CONFIG_BLK_DEV_INITRD
	490	void free_initrd_mem(unsigned long start, unsigned long end)
	491	{
	492	if (start < (unsigned long)&_end)
	493	return;
	494	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
	495	for (; start < end; start += PAGE_SIZE) {
	496	ClearPageReserved(virt_to_page(start));
	497	set_page_count(virt_to_page(start), 1);
	498	free_page(start);
	499	totalram_pages++;
	500	}
	501	}
	502	#endif
	503
	504	void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
	505	{
	506	/* Should check here against the e820 map to avoid double free */
2b97690f	507	#ifdef CONFIG_NUMA
1da177e4 LT	508	int nid = phys_to_nid(phys);
	509	reserve_bootmem_node(NODE_DATA(nid), phys, len);
	510	#else
	511	reserve_bootmem(phys, len);
	512	#endif
	513	}
	514
	515	int kern_addr_valid(unsigned long addr)
	516	{
	517	unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT;
	518	pgd_t *pgd;
	519	pud_t *pud;
	520	pmd_t *pmd;
	521	pte_t *pte;
	522
	523	if (above != 0 && above != -1UL)
	524	return 0;
	525
	526	pgd = pgd_offset_k(addr);
	527	if (pgd_none(*pgd))
	528	return 0;
	529
	530	pud = pud_offset(pgd, addr);
	531	if (pud_none(*pud))
	532	return 0;
	533
	534	pmd = pmd_offset(pud, addr);
	535	if (pmd_none(*pmd))
	536	return 0;
	537	if (pmd_large(*pmd))
	538	return pfn_valid(pmd_pfn(*pmd));
	539
	540	pte = pte_offset_kernel(pmd, addr);
	541	if (pte_none(*pte))
	542	return 0;
	543	return pfn_valid(pte_pfn(*pte));
	544	}
	545
	546	#ifdef CONFIG_SYSCTL
	547	#include <linux/sysctl.h>
	548
	549	extern int exception_trace, page_fault_trace;
	550
	551	static ctl_table debug_table2[] = {
	552	{ 99, "exception-trace", &exception_trace, sizeof(int), 0644, NULL,
	553	proc_dointvec },
	554	#ifdef CONFIG_CHECKING
	555	{ 100, "page-fault-trace", &page_fault_trace, sizeof(int), 0644, NULL,
	556	proc_dointvec },
	557	#endif
	558	{ 0, }
	559	};
	560
	561	static ctl_table debug_root_table2[] = {
	562	{ .ctl_name = CTL_DEBUG, .procname = "debug", .mode = 0555,
	563	.child = debug_table2 },
	564	{ 0 },
	565	};
	566
	567	static __init int x8664_sysctl_init(void)
	568	{
	569	register_sysctl_table(debug_root_table2, 1);
	570	return 0;
	571	}
572	__initcall(x8664_sysctl_init);
573	#endif
574
1e014410 AK	575	/* A pseudo VMAs to allow ptrace access for the vsyscall page. This only
	576	covers the 64bit vsyscall page now. 32bit has a real VMA now and does
	577	not need special handling anymore. */
1da177e4 LT	578
	579	static struct vm_area_struct gate_vma = {
	580	.vm_start = VSYSCALL_START,
	581	.vm_end = VSYSCALL_END,
	582	.vm_page_prot = PAGE_READONLY
	583	};
	584
1da177e4 LT	585	struct vm_area_struct get_gate_vma(struct task_struct tsk)
	586	{
	587	#ifdef CONFIG_IA32_EMULATION
1e014410 AK	588	if (test_tsk_thread_flag(tsk, TIF_IA32))
1e014410 AK	589	return NULL;
1da177e4 LT	590	#endif
	591	return &gate_vma;
	592	}
	593
	594	int in_gate_area(struct task_struct *task, unsigned long addr)
	595	{
	596	struct vm_area_struct *vma = get_gate_vma(task);
1e014410 AK	597	if (!vma)
1e014410 AK	598	return 0;
1da177e4 LT	599	return (addr >= vma->vm_start) && (addr < vma->vm_end);
	600	}
	601
	602	/* Use this when you have no reliable task/vma, typically from interrupt
	603	* context. It is less reliable than using the task's vma and may give
	604	* false positives.
	605	*/
	606	int in_gate_area_no_task(unsigned long addr)
	607	{
1e014410	608	return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END);
1da177e4	609	}