[linux-2.6-block.git] / arch / powerpc / mm / mem.c

/*
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
 *    Copyright (C) 1996 Paul Mackerras
 *  PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
 *
 *  Derived from "arch/i386/mm/init.c"
 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 *
 */

#include <linux/export.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/stddef.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <linux/initrd.h>
#include <linux/pagemap.h>
#include <linux/suspend.h>
#include <linux/memblock.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>

#include <asm/pgalloc.h>
#include <asm/prom.h>
#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/smp.h>
#include <asm/machdep.h>
#include <asm/btext.h>
#include <asm/tlb.h>
#include <asm/sections.h>
#include <asm/sparsemem.h>
#include <asm/vdso.h>
#include <asm/fixmap.h>
#include <asm/swiotlb.h>
#include <asm/rtas.h>

#include "mmu_decl.h"

#ifndef CPU_FTR_COHERENT_ICACHE
#define CPU_FTR_COHERENT_ICACHE	0	/* XXX for now */
#define CPU_FTR_NOEXECUTE	0
#endif

int init_bootmem_done;
int mem_init_done;
unsigned long long memory_limit;

#ifdef CONFIG_HIGHMEM
pte_t *kmap_pte;
EXPORT_SYMBOL(kmap_pte);
pgprot_t kmap_prot;
EXPORT_SYMBOL(kmap_prot);

static inline pte_t *virt_to_kpte(unsigned long vaddr)
{
	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
			vaddr), vaddr), vaddr);
}
#endif

int page_is_ram(unsigned long pfn)
{
#ifndef CONFIG_PPC64	/* XXX for now */
	return pfn < max_pfn;
#else
	unsigned long paddr = (pfn << PAGE_SHIFT);
	struct memblock_region *reg;

	for_each_memblock(memory, reg)
		if (paddr >= reg->base && paddr < (reg->base + reg->size))
			return 1;
	return 0;
#endif
}

pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
			      unsigned long size, pgprot_t vma_prot)
{
	if (ppc_md.phys_mem_access_prot)
		return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);

	if (!page_is_ram(pfn))
		vma_prot = pgprot_noncached(vma_prot);

	return vma_prot;
}
EXPORT_SYMBOL(phys_mem_access_prot);

#ifdef CONFIG_MEMORY_HOTPLUG

#ifdef CONFIG_NUMA
int memory_add_physaddr_to_nid(u64 start)
{
	return hot_add_scn_to_nid(start);
}
#endif

int arch_add_memory(int nid, u64 start, u64 size)
{
	struct pglist_data *pgdata;
	struct zone *zone;
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;

	pgdata = NODE_DATA(nid);

	start = (unsigned long)__va(start);
	if (create_section_mapping(start, start + size))
		return -EINVAL;

	/* this should work for most non-highmem platforms */
	zone = pgdata->node_zones;

	return __add_pages(nid, zone, start_pfn, nr_pages);
}

#ifdef CONFIG_MEMORY_HOTREMOVE
int arch_remove_memory(u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	unsigned long nr_pages = size >> PAGE_SHIFT;
	struct zone *zone;

	zone = page_zone(pfn_to_page(start_pfn));
	return __remove_pages(zone, start_pfn, nr_pages);
}
#endif
#endif /* CONFIG_MEMORY_HOTPLUG */

/*
 * walk_memory_resource() needs to make sure there is no holes in a given
 * memory range.  PPC64 does not maintain the memory layout in /proc/iomem.
 * Instead it maintains it in memblock.memory structures.  Walk through the
 * memory regions, find holes and callback for contiguous regions.
 */
int
walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
		void *arg, int (*func)(unsigned long, unsigned long, void *))
{
	struct memblock_region *reg;
	unsigned long end_pfn = start_pfn + nr_pages;
	unsigned long tstart, tend;
	int ret = -1;

	for_each_memblock(memory, reg) {
		tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
		tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
		if (tstart >= tend)
			continue;
		ret = (*func)(tstart, tend - tstart, arg);
		if (ret)
			break;
	}
	return ret;
}
EXPORT_SYMBOL_GPL(walk_system_ram_range);

/*
 * Initialize the bootmem system and give it all the memory we
 * have available.  If we are using highmem, we only put the
 * lowmem into the bootmem system.
 */
#ifndef CONFIG_NEED_MULTIPLE_NODES
void __init do_init_bootmem(void)
{
	unsigned long start, bootmap_pages;
	unsigned long total_pages;
	struct memblock_region *reg;
	int boot_mapsize;

	max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
	total_pages = (memblock_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
#ifdef CONFIG_HIGHMEM
	total_pages = total_lowmem >> PAGE_SHIFT;
	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
#endif

	/*
	 * Find an area to use for the bootmem bitmap.  Calculate the size of
	 * bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
	 * Add 1 additional page in case the address isn't page-aligned.
	 */
	bootmap_pages = bootmem_bootmap_pages(total_pages);

	start = memblock_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);

	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
	boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);

	/* Place all memblock_regions in the same node and merge contiguous
	 * memblock_regions
	 */
	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);

	/* Add all physical memory to the bootmem map, mark each area
	 * present.
	 */
#ifdef CONFIG_HIGHMEM
	free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);

	/* reserve the sections we're already using */
	for_each_memblock(reserved, reg) {
		unsigned long top = reg->base + reg->size - 1;
		if (top < lowmem_end_addr)
			reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
		else if (reg->base < lowmem_end_addr) {
			unsigned long trunc_size = lowmem_end_addr - reg->base;
			reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
		}
	}
#else
	free_bootmem_with_active_regions(0, max_pfn);

	/* reserve the sections we're already using */
	for_each_memblock(reserved, reg)
		reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
#endif
	/* XXX need to clip this if using highmem? */
	sparse_memory_present_with_active_regions(0);

	init_bootmem_done = 1;
}

/* mark pages that don't exist as nosave */
static int __init mark_nonram_nosave(void)
{
	struct memblock_region *reg, *prev = NULL;

	for_each_memblock(memory, reg) {
		if (prev &&
		    memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
			register_nosave_region(memblock_region_memory_end_pfn(prev),
					       memblock_region_memory_base_pfn(reg));
		prev = reg;
	}
	return 0;
}

/*
 * paging_init() sets up the page tables - in fact we've already done this.
 */
void __init paging_init(void)
{
	unsigned long long total_ram = memblock_phys_mem_size();
	phys_addr_t top_of_ram = memblock_end_of_DRAM();
	unsigned long max_zone_pfns[MAX_NR_ZONES];

#ifdef CONFIG_PPC32
	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	unsigned long end = __fix_to_virt(FIX_HOLE);

	for (; v < end; v += PAGE_SIZE)
		map_page(v, 0, 0); /* XXX gross */
#endif

#ifdef CONFIG_HIGHMEM
	map_page(PKMAP_BASE, 0, 0);	/* XXX gross */
	pkmap_page_table = virt_to_kpte(PKMAP_BASE);

	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
	kmap_prot = PAGE_KERNEL;
#endif /* CONFIG_HIGHMEM */

	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
	       (unsigned long long)top_of_ram, total_ram);
	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
	       (long int)((top_of_ram - total_ram) >> 20));
	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
#ifdef CONFIG_HIGHMEM
	max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
	max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
#else
	max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
#endif
	free_area_init_nodes(max_zone_pfns);

	mark_nonram_nosave();
}
#endif /* ! CONFIG_NEED_MULTIPLE_NODES */

void __init mem_init(void)
{
#ifdef CONFIG_SWIOTLB
	swiotlb_init(0);
#endif

	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
	set_max_mapnr(max_pfn);
	free_all_bootmem();

#ifdef CONFIG_HIGHMEM
	{
		unsigned long pfn, highmem_mapnr;

		highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
		for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
			phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
			struct page *page = pfn_to_page(pfn);
			if (!memblock_is_reserved(paddr))
				free_highmem_page(page);
		}
	}
#endif /* CONFIG_HIGHMEM */

#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
	/*
	 * If smp is enabled, next_tlbcam_idx is initialized in the cpu up
	 * functions.... do it here for the non-smp case.
	 */
	per_cpu(next_tlbcam_idx, smp_processor_id()) =
		(mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
#endif

	mem_init_print_info(NULL);
#ifdef CONFIG_PPC32
	pr_info("Kernel virtual memory layout:\n");
	pr_info("  * 0x%08lx..0x%08lx  : fixmap\n", FIXADDR_START, FIXADDR_TOP);
#ifdef CONFIG_HIGHMEM
	pr_info("  * 0x%08lx..0x%08lx  : highmem PTEs\n",
		PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
#endif /* CONFIG_HIGHMEM */
#ifdef CONFIG_NOT_COHERENT_CACHE
	pr_info("  * 0x%08lx..0x%08lx  : consistent mem\n",
		IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
#endif /* CONFIG_NOT_COHERENT_CACHE */
	pr_info("  * 0x%08lx..0x%08lx  : early ioremap\n",
		ioremap_bot, IOREMAP_TOP);
	pr_info("  * 0x%08lx..0x%08lx  : vmalloc & ioremap\n",
		VMALLOC_START, VMALLOC_END);
#endif /* CONFIG_PPC32 */

	mem_init_done = 1;
}

void free_initmem(void)
{
	ppc_md.progress = ppc_printk_progress;
	free_initmem_default(POISON_FREE_INITMEM);
}

#ifdef CONFIG_BLK_DEV_INITRD
void __init free_initrd_mem(unsigned long start, unsigned long end)
{
	free_reserved_area((void *)start, (void *)end, -1, "initrd");
}
#endif

/*
 * This is called when a page has been modified by the kernel.
 * It just marks the page as not i-cache clean.  We do the i-cache
 * flush later when the page is given to a user process, if necessary.
 */
void flush_dcache_page(struct page *page)
{
	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
		return;
	/* avoid an atomic op if possible */
	if (test_bit(PG_arch_1, &page->flags))
		clear_bit(PG_arch_1, &page->flags);
}
EXPORT_SYMBOL(flush_dcache_page);

void flush_dcache_icache_page(struct page *page)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (PageCompound(page)) {
		flush_dcache_icache_hugepage(page);
		return;
	}
#endif
#ifdef CONFIG_BOOKE
	{
		void *start = kmap_atomic(page);
		__flush_dcache_icache(start);
		kunmap_atomic(start);
	}
#elif defined(CONFIG_8xx) || defined(CONFIG_PPC64)
	/* On 8xx there is no need to kmap since highmem is not supported */
	__flush_dcache_icache(page_address(page)); 
#else
	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
#endif
}
EXPORT_SYMBOL(flush_dcache_icache_page);

void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
{
	clear_page(page);

	/*
	 * We shouldn't have to do this, but some versions of glibc
	 * require it (ld.so assumes zero filled pages are icache clean)
	 * - Anton
	 */
	flush_dcache_page(pg);
}
EXPORT_SYMBOL(clear_user_page);

void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
		    struct page *pg)
{
	copy_page(vto, vfrom);

	/*
	 * We should be able to use the following optimisation, however
	 * there are two problems.
	 * Firstly a bug in some versions of binutils meant PLT sections
	 * were not marked executable.
	 * Secondly the first word in the GOT section is blrl, used
	 * to establish the GOT address. Until recently the GOT was
	 * not marked executable.
	 * - Anton
	 */
#if 0
	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
		return;
#endif

	flush_dcache_page(pg);
}

void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
			     unsigned long addr, int len)
{
	unsigned long maddr;

	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	flush_icache_range(maddr, maddr + len);
	kunmap(page);
}
EXPORT_SYMBOL(flush_icache_user_range);

/*
 * This is called at the end of handling a user page fault, when the
 * fault has been handled by updating a PTE in the linux page tables.
 * We use it to preload an HPTE into the hash table corresponding to
 * the updated linux PTE.
 * 
 * This must always be called with the pte lock held.
 */
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
		      pte_t *ptep)
{
#ifdef CONFIG_PPC_STD_MMU
	/*
	 * We don't need to worry about _PAGE_PRESENT here because we are
	 * called with either mm->page_table_lock held or ptl lock held
	 */
	unsigned long access = 0, trap;

	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
	if (!pte_young(*ptep) || address >= TASK_SIZE)
		return;

	/* We try to figure out if we are coming from an instruction
	 * access fault and pass that down to __hash_page so we avoid
	 * double-faulting on execution of fresh text. We have to test
	 * for regs NULL since init will get here first thing at boot
	 *
	 * We also avoid filling the hash if not coming from a fault
	 */
	if (current->thread.regs == NULL)
		return;
	trap = TRAP(current->thread.regs);
	if (trap == 0x400)
		access |= _PAGE_EXEC;
	else if (trap != 0x300)
		return;
	hash_preload(vma->vm_mm, address, access, trap);
#endif /* CONFIG_PPC_STD_MMU */
#if (defined(CONFIG_PPC_BOOK3E_64) || defined(CONFIG_PPC_FSL_BOOK3E)) \
	&& defined(CONFIG_HUGETLB_PAGE)
	if (is_vm_hugetlb_page(vma))
		book3e_hugetlb_preload(vma, address, *ptep);
#endif
}

/*
 * System memory should not be in /proc/iomem but various tools expect it
 * (eg kdump).
 */
static int add_system_ram_resources(void)
{
	struct memblock_region *reg;

	for_each_memblock(memory, reg) {
		struct resource *res;
		unsigned long base = reg->base;
		unsigned long size = reg->size;

		res = kzalloc(sizeof(struct resource), GFP_KERNEL);
		WARN_ON(!res);

		if (res) {
			res->name = "System RAM";
			res->start = base;
			res->end = base + size - 1;
			res->flags = IORESOURCE_MEM;
			WARN_ON(request_resource(&iomem_resource, res) < 0);
		}
	}

	return 0;
}
subsys_initcall(add_system_ram_resources);

#ifdef CONFIG_STRICT_DEVMEM
/*
 * devmem_is_allowed(): check to see if /dev/mem access to a certain address
 * is valid. The argument is a physical page number.
 *
 * Access has to be given to non-kernel-ram areas as well, these contain the
 * PCI mmio resources as well as potential bios/acpi data regions.
 */
int devmem_is_allowed(unsigned long pfn)
{
	if (iomem_is_exclusive(pfn << PAGE_SHIFT))
		return 0;
	if (!page_is_ram(pfn))
		return 1;
	if (page_is_rtas_user_buf(pfn))
		return 1;
	return 0;
}
#endif /* CONFIG_STRICT_DEVMEM */
Commit	Line	Data
14cf11af PM	1	/*
	2	* PowerPC version
	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	4	*
	5	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	6	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	7	* Copyright (C) 1996 Paul Mackerras
14cf11af PM	8	* PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
	9	*
	10	* Derived from "arch/i386/mm/init.c"
	11	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	12	*
	13	* This program is free software; you can redistribute it and/or
	14	* modify it under the terms of the GNU General Public License
	15	* as published by the Free Software Foundation; either version
	16	* 2 of the License, or (at your option) any later version.
	17	*
	18	*/
	19
4b16f8e2	20	#include <linux/export.h>
14cf11af PM	21	#include <linux/sched.h>
	22	#include <linux/kernel.h>
	23	#include <linux/errno.h>
	24	#include <linux/string.h>
5a0e3ad6	25	#include <linux/gfp.h>
14cf11af PM	26	#include <linux/types.h>
	27	#include <linux/mm.h>
	28	#include <linux/stddef.h>
	29	#include <linux/init.h>
	30	#include <linux/bootmem.h>
	31	#include <linux/highmem.h>
	32	#include <linux/initrd.h>
	33	#include <linux/pagemap.h>
4e8ad3e8	34	#include <linux/suspend.h>
95f72d1e	35	#include <linux/memblock.h>
0895ecda	36	#include <linux/hugetlb.h>
c40dd2f7	37	#include <linux/slab.h>
14cf11af PM	38
	39	#include <asm/pgalloc.h>
	40	#include <asm/prom.h>
	41	#include <asm/io.h>
	42	#include <asm/mmu_context.h>
	43	#include <asm/pgtable.h>
	44	#include <asm/mmu.h>
	45	#include <asm/smp.h>
	46	#include <asm/machdep.h>
	47	#include <asm/btext.h>
	48	#include <asm/tlb.h>
7c8c6b97	49	#include <asm/sections.h>
db7f37de	50	#include <asm/sparsemem.h>
ab1f9dac	51	#include <asm/vdso.h>
2c419bde	52	#include <asm/fixmap.h>
a9327296	53	#include <asm/swiotlb.h>
8a3e3d31	54	#include <asm/rtas.h>
14cf11af	55
14cf11af PM	56	#include "mmu_decl.h"
	57
	58	#ifndef CPU_FTR_COHERENT_ICACHE
	59	#define CPU_FTR_COHERENT_ICACHE 0 /* XXX for now */
	60	#define CPU_FTR_NOEXECUTE 0
	61	#endif
	62
7c8c6b97 PM	63	int init_bootmem_done;
7c8c6b97 PM	64	int mem_init_done;
a84fcd46	65	unsigned long long memory_limit;
7c8c6b97	66
2c419bde KG	67	#ifdef CONFIG_HIGHMEM
2c419bde KG	68	pte_t *kmap_pte;
8040bda3	69	EXPORT_SYMBOL(kmap_pte);
2c419bde	70	pgprot_t kmap_prot;
2c419bde	71	EXPORT_SYMBOL(kmap_prot);
2c419bde KG	72
	73	static inline pte_t *virt_to_kpte(unsigned long vaddr)
	74	{
	75	return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
	76	vaddr), vaddr), vaddr);
	77	}
	78	#endif
	79
14cf11af PM	80	int page_is_ram(unsigned long pfn)
14cf11af PM	81	{
14cf11af	82	#ifndef CONFIG_PPC64 /* XXX for now */
a880e762	83	return pfn < max_pfn;
14cf11af	84	#else
a880e762	85	unsigned long paddr = (pfn << PAGE_SHIFT);
28be7072	86	struct memblock_region *reg;
14cf11af	87
28be7072 BH	88	for_each_memblock(memory, reg)
28be7072 BH	89	if (paddr >= reg->base && paddr < (reg->base + reg->size))
14cf11af	90	return 1;
14cf11af PM	91	return 0;
	92	#endif
	93	}
14cf11af	94
8b150478	95	pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
14cf11af PM	96	unsigned long size, pgprot_t vma_prot)
	97	{
	98	if (ppc_md.phys_mem_access_prot)
8b150478	99	return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
14cf11af	100
8b150478	101	if (!page_is_ram(pfn))
64b3d0e8 BH	102	vma_prot = pgprot_noncached(vma_prot);
64b3d0e8 BH	103
14cf11af PM	104	return vma_prot;
	105	}
	106	EXPORT_SYMBOL(phys_mem_access_prot);
	107
23fd0775 PM	108	#ifdef CONFIG_MEMORY_HOTPLUG
23fd0775 PM	109
bc02af93 YG	110	#ifdef CONFIG_NUMA
	111	int memory_add_physaddr_to_nid(u64 start)
	112	{
	113	return hot_add_scn_to_nid(start);
	114	}
	115	#endif
	116
fa90f70a	117	int arch_add_memory(int nid, u64 start, u64 size)
23fd0775	118	{
237a0989	119	struct pglist_data *pgdata;
23fd0775 PM	120	struct zone *zone;
	121	unsigned long start_pfn = start >> PAGE_SHIFT;
	122	unsigned long nr_pages = size >> PAGE_SHIFT;
	123
237a0989 MK	124	pgdata = NODE_DATA(nid);
237a0989 MK	125
2d0eee14	126	start = (unsigned long)__va(start);
a1194097 AB	127	if (create_section_mapping(start, start + size))
a1194097 AB	128	return -EINVAL;
54b79248	129
23fd0775 PM	130	/* this should work for most non-highmem platforms */
	131	zone = pgdata->node_zones;
	132
c04fc586	133	return __add_pages(nid, zone, start_pfn, nr_pages);
23fd0775	134	}
24d335ca WC	135
	136	#ifdef CONFIG_MEMORY_HOTREMOVE
	137	int arch_remove_memory(u64 start, u64 size)
	138	{
	139	unsigned long start_pfn = start >> PAGE_SHIFT;
	140	unsigned long nr_pages = size >> PAGE_SHIFT;
	141	struct zone *zone;
	142
	143	zone = page_zone(pfn_to_page(start_pfn));
	144	return __remove_pages(zone, start_pfn, nr_pages);
	145	}
	146	#endif
0d579944	147	#endif /* CONFIG_MEMORY_HOTPLUG */
a99824f3 BP	148
	149	/*
	150	* walk_memory_resource() needs to make sure there is no holes in a given
9d88a2eb	151	* memory range. PPC64 does not maintain the memory layout in /proc/iomem.
95f72d1e	152	* Instead it maintains it in memblock.memory structures. Walk through the
9d88a2eb	153	* memory regions, find holes and callback for contiguous regions.
a99824f3 BP	154	*/
a99824f3 BP	155	int
908eedc6 KH	156	walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
908eedc6 KH	157	void arg, int (func)(unsigned long, unsigned long, void *))
a99824f3	158	{
28be7072 BH	159	struct memblock_region *reg;
	160	unsigned long end_pfn = start_pfn + nr_pages;
	161	unsigned long tstart, tend;
9d88a2eb BP	162	int ret = -1;
9d88a2eb BP	163
28be7072	164	for_each_memblock(memory, reg) {
c7fc2de0 YL	165	tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
c7fc2de0 YL	166	tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
28be7072 BH	167	if (tstart >= tend)
	168	continue;
	169	ret = (*func)(tstart, tend - tstart, arg);
9d88a2eb BP	170	if (ret)
9d88a2eb BP	171	break;
9d88a2eb BP	172	}
9d88a2eb BP	173	return ret;
a99824f3	174	}
908eedc6	175	EXPORT_SYMBOL_GPL(walk_system_ram_range);
a99824f3	176
7c8c6b97 PM	177	/*
	178	* Initialize the bootmem system and give it all the memory we
	179	* have available. If we are using highmem, we only put the
	180	* lowmem into the bootmem system.
	181	*/
	182	#ifndef CONFIG_NEED_MULTIPLE_NODES
	183	void __init do_init_bootmem(void)
	184	{
7c8c6b97 PM	185	unsigned long start, bootmap_pages;
7c8c6b97 PM	186	unsigned long total_pages;
28be7072	187	struct memblock_region *reg;
7c8c6b97 PM	188	int boot_mapsize;
7c8c6b97 PM	189
95f72d1e YL	190	max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
95f72d1e YL	191	total_pages = (memblock_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
7c8c6b97 PM	192	#ifdef CONFIG_HIGHMEM
7c8c6b97 PM	193	total_pages = total_lowmem >> PAGE_SHIFT;
d7917ba7	194	max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97 PM	195	#endif
	196
	197	/*
	198	* Find an area to use for the bootmem bitmap. Calculate the size of
	199	* bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
	200	* Add 1 additional page in case the address isn't page-aligned.
	201	*/
	202	bootmap_pages = bootmem_bootmap_pages(total_pages);
	203
95f72d1e	204	start = memblock_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);
7c8c6b97	205
37dd2bad KG	206	min_low_pfn = MEMORY_START >> PAGE_SHIFT;
37dd2bad KG	207	boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);
7c8c6b97	208
4e8309ba CS	209	/* Place all memblock_regions in the same node and merge contiguous
	210	* memblock_regions
	211	*/
	212	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
c67c3cb4	213
7c8c6b97 PM	214	/* Add all physical memory to the bootmem map, mark each area
	215	* present.
	216	*/
7c8c6b97	217	#ifdef CONFIG_HIGHMEM
d7917ba7	218	free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);
f98eeb4e KG	219
f98eeb4e KG	220	/* reserve the sections we're already using */
28be7072 BH	221	for_each_memblock(reserved, reg) {
	222	unsigned long top = reg->base + reg->size - 1;
	223	if (top < lowmem_end_addr)
	224	reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
	225	else if (reg->base < lowmem_end_addr) {
	226	unsigned long trunc_size = lowmem_end_addr - reg->base;
	227	reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
f98eeb4e KG	228	}
f98eeb4e KG	229	}
c67c3cb4 MG	230	#else
c67c3cb4 MG	231	free_bootmem_with_active_regions(0, max_pfn);
7c8c6b97 PM	232
7c8c6b97 PM	233	/* reserve the sections we're already using */
28be7072 BH	234	for_each_memblock(reserved, reg)
28be7072 BH	235	reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
f98eeb4e	236	#endif
7c8c6b97	237	/* XXX need to clip this if using highmem? */
c67c3cb4 MG	238	sparse_memory_present_with_active_regions(0);
c67c3cb4 MG	239
7c8c6b97 PM	240	init_bootmem_done = 1;
	241	}
	242
4e8ad3e8 JB	243	/* mark pages that don't exist as nosave */
	244	static int __init mark_nonram_nosave(void)
	245	{
28be7072 BH	246	struct memblock_region reg, prev = NULL;
	247
	248	for_each_memblock(memory, reg) {
	249	if (prev &&
c7fc2de0 YL	250	memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
	251	register_nosave_region(memblock_region_memory_end_pfn(prev),
	252	memblock_region_memory_base_pfn(reg));
28be7072	253	prev = reg;
4e8ad3e8	254	}
4e8ad3e8 JB	255	return 0;
	256	}
	257
7c8c6b97 PM	258	/*
	259	* paging_init() sets up the page tables - in fact we've already done this.
	260	*/
	261	void __init paging_init(void)
	262	{
f7ba2991	263	unsigned long long total_ram = memblock_phys_mem_size();
95f72d1e	264	phys_addr_t top_of_ram = memblock_end_of_DRAM();
c67c3cb4	265	unsigned long max_zone_pfns[MAX_NR_ZONES];
7c8c6b97	266
2c419bde KG	267	#ifdef CONFIG_PPC32
	268	unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
	269	unsigned long end = __fix_to_virt(FIX_HOLE);
	270
	271	for (; v < end; v += PAGE_SIZE)
	272	map_page(v, 0, 0); /* XXX gross */
	273	#endif
	274
7c8c6b97 PM	275	#ifdef CONFIG_HIGHMEM
7c8c6b97 PM	276	map_page(PKMAP_BASE, 0, 0); /* XXX gross */
2c419bde KG	277	pkmap_page_table = virt_to_kpte(PKMAP_BASE);
	278
	279	kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
7c8c6b97 PM	280	kmap_prot = PAGE_KERNEL;
	281	#endif /* CONFIG_HIGHMEM */
	282
f7ba2991	283	printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
fb610635	284	(unsigned long long)top_of_ram, total_ram);
e110b281	285	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
2bf3016f	286	(long int)((top_of_ram - total_ram) >> 20));
6391af17	287	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
7c8c6b97	288	#ifdef CONFIG_HIGHMEM
d7917ba7	289	max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
6391af17	290	max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
7c8c6b97	291	#else
6391af17	292	max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
c67c3cb4 MG	293	#endif
c67c3cb4 MG	294	free_area_init_nodes(max_zone_pfns);
4e8ad3e8 JB	295
4e8ad3e8 JB	296	mark_nonram_nosave();
7c8c6b97 PM	297	}
	298	#endif /* ! CONFIG_NEED_MULTIPLE_NODES */
	299
	300	void __init mem_init(void)
	301	{
a9327296	302	#ifdef CONFIG_SWIOTLB
688ba1db	303	swiotlb_init(0);
a9327296 FT	304	#endif
a9327296 FT	305
7c8c6b97	306	high_memory = (void ) __va(max_low_pfn PAGE_SIZE);
602ddc70	307	set_max_mapnr(max_pfn);
0c988534	308	free_all_bootmem();
7c8c6b97 PM	309
	310	#ifdef CONFIG_HIGHMEM
	311	{
	312	unsigned long pfn, highmem_mapnr;
	313
d7917ba7	314	highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
7c8c6b97	315	for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
3d41e0f6	316	phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
7c8c6b97	317	struct page *page = pfn_to_page(pfn);
369a9d85 JL	318	if (!memblock_is_reserved(paddr))
369a9d85 JL	319	free_highmem_page(page);
7c8c6b97	320	}
7c8c6b97 PM	321	}
	322	#endif /* CONFIG_HIGHMEM */
	323
3160b097 BB	324	#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
	325	/*
	326	* If smp is enabled, next_tlbcam_idx is initialized in the cpu up
	327	* functions.... do it here for the non-smp case.
	328	*/
	329	per_cpu(next_tlbcam_idx, smp_processor_id()) =
	330	(mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
	331	#endif
	332
369a9d85	333	mem_init_print_info(NULL);
f637a49e BH	334	#ifdef CONFIG_PPC32
	335	pr_info("Kernel virtual memory layout:\n");
	336	pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP);
	337	#ifdef CONFIG_HIGHMEM
	338	pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n",
	339	PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
	340	#endif /* CONFIG_HIGHMEM */
8b31e49d BH	341	#ifdef CONFIG_NOT_COHERENT_CACHE
	342	pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
	343	IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
	344	#endif /* CONFIG_NOT_COHERENT_CACHE */
f637a49e BH	345	pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
	346	ioremap_bot, IOREMAP_TOP);
	347	pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
	348	VMALLOC_START, VMALLOC_END);
	349	#endif /* CONFIG_PPC32 */
	350
7c8c6b97	351	mem_init_done = 1;
7c8c6b97 PM	352	}
7c8c6b97 PM	353
2773fcc8 DC	354	void free_initmem(void)
2773fcc8 DC	355	{
a9c0f41b	356	ppc_md.progress = ppc_printk_progress;
5d585e5c	357	free_initmem_default(POISON_FREE_INITMEM);
2773fcc8 DC	358	}
2773fcc8 DC	359
307cfe71 BH	360	#ifdef CONFIG_BLK_DEV_INITRD
	361	void __init free_initrd_mem(unsigned long start, unsigned long end)
	362	{
dbe67df4	363	free_reserved_area((void )start, (void )end, -1, "initrd");
307cfe71 BH	364	}
	365	#endif
	366
14cf11af PM	367	/*
	368	* This is called when a page has been modified by the kernel.
	369	* It just marks the page as not i-cache clean. We do the i-cache
	370	* flush later when the page is given to a user process, if necessary.
	371	*/
	372	void flush_dcache_page(struct page *page)
	373	{
	374	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
	375	return;
	376	/* avoid an atomic op if possible */
	377	if (test_bit(PG_arch_1, &page->flags))
	378	clear_bit(PG_arch_1, &page->flags);
	379	}
	380	EXPORT_SYMBOL(flush_dcache_page);
	381
	382	void flush_dcache_icache_page(struct page *page)
	383	{
0895ecda DG	384	#ifdef CONFIG_HUGETLB_PAGE
	385	if (PageCompound(page)) {
	386	flush_dcache_icache_hugepage(page);
	387	return;
	388	}
	389	#endif
14cf11af	390	#ifdef CONFIG_BOOKE
0895ecda	391	{
2480b208	392	void *start = kmap_atomic(page);
0895ecda	393	__flush_dcache_icache(start);
2480b208	394	kunmap_atomic(start);
0895ecda	395	}
ab1f9dac	396	#elif defined(CONFIG_8xx) \|\| defined(CONFIG_PPC64)
14cf11af PM	397	/* On 8xx there is no need to kmap since highmem is not supported */
	398	__flush_dcache_icache(page_address(page));
	399	#else
	400	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
	401	#endif
14cf11af	402	}
249ba1ee	403	EXPORT_SYMBOL(flush_dcache_icache_page);
0895ecda	404
14cf11af PM	405	void clear_user_page(void page, unsigned long vaddr, struct page pg)
	406	{
	407	clear_page(page);
	408
14cf11af	409	/*
25985edc	410	* We shouldn't have to do this, but some versions of glibc
14cf11af PM	411	* require it (ld.so assumes zero filled pages are icache clean)
	412	* - Anton
	413	*/
09f5dc44	414	flush_dcache_page(pg);
14cf11af PM	415	}
	416	EXPORT_SYMBOL(clear_user_page);
	417
	418	void copy_user_page(void vto, void vfrom, unsigned long vaddr,
	419	struct page *pg)
	420	{
	421	copy_page(vto, vfrom);
	422
	423	/*
	424	* We should be able to use the following optimisation, however
	425	* there are two problems.
	426	* Firstly a bug in some versions of binutils meant PLT sections
	427	* were not marked executable.
	428	* Secondly the first word in the GOT section is blrl, used
	429	* to establish the GOT address. Until recently the GOT was
	430	* not marked executable.
	431	* - Anton
	432	*/
	433	#if 0
	434	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
	435	return;
	436	#endif
	437
09f5dc44	438	flush_dcache_page(pg);
14cf11af PM	439	}
	440
	441	void flush_icache_user_range(struct vm_area_struct vma, struct page page,
	442	unsigned long addr, int len)
	443	{
	444	unsigned long maddr;
	445
	446	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
	447	flush_icache_range(maddr, maddr + len);
	448	kunmap(page);
	449	}
	450	EXPORT_SYMBOL(flush_icache_user_range);
	451
	452	/*
	453	* This is called at the end of handling a user page fault, when the
	454	* fault has been handled by updating a PTE in the linux page tables.
	455	* We use it to preload an HPTE into the hash table corresponding to
	456	* the updated linux PTE.
	457	*
01edcd89	458	* This must always be called with the pte lock held.
14cf11af PM	459	*/
14cf11af PM	460	void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
4b3073e1	461	pte_t *ptep)
14cf11af	462	{
3c726f8d	463	#ifdef CONFIG_PPC_STD_MMU
0ac52dd7 AK	464	/*
	465	* We don't need to worry about _PAGE_PRESENT here because we are
	466	* called with either mm->page_table_lock held or ptl lock held
	467	*/
3c726f8d	468	unsigned long access = 0, trap;
14cf11af	469
14cf11af	470	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
4b3073e1	471	if (!pte_young(*ptep) \|\| address >= TASK_SIZE)
14cf11af	472	return;
14cf11af	473
3c726f8d BH	474	/* We try to figure out if we are coming from an instruction
	475	* access fault and pass that down to __hash_page so we avoid
	476	* double-faulting on execution of fresh text. We have to test
	477	* for regs NULL since init will get here first thing at boot
	478	*
	479	* We also avoid filling the hash if not coming from a fault
	480	*/
	481	if (current->thread.regs == NULL)
14cf11af	482	return;
3c726f8d BH	483	trap = TRAP(current->thread.regs);
	484	if (trap == 0x400)
	485	access \|= _PAGE_EXEC;
	486	else if (trap != 0x300)
	487	return;
	488	hash_preload(vma->vm_mm, address, access, trap);
	489	#endif /* CONFIG_PPC_STD_MMU */
41151e77 BB	490	#if (defined(CONFIG_PPC_BOOK3E_64) \|\| defined(CONFIG_PPC_FSL_BOOK3E)) \
	491	&& defined(CONFIG_HUGETLB_PAGE)
	492	if (is_vm_hugetlb_page(vma))
d93e4d7d	493	book3e_hugetlb_preload(vma, address, *ptep);
41151e77	494	#endif
14cf11af	495	}
c40dd2f7 AB	496
	497	/*
	498	* System memory should not be in /proc/iomem but various tools expect it
	499	* (eg kdump).
	500	*/
	501	static int add_system_ram_resources(void)
	502	{
	503	struct memblock_region *reg;
	504
	505	for_each_memblock(memory, reg) {
	506	struct resource *res;
	507	unsigned long base = reg->base;
	508	unsigned long size = reg->size;
	509
	510	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
	511	WARN_ON(!res);
	512
	513	if (res) {
	514	res->name = "System RAM";
	515	res->start = base;
	516	res->end = base + size - 1;
	517	res->flags = IORESOURCE_MEM;
	518	WARN_ON(request_resource(&iomem_resource, res) < 0);
	519	}
	520	}
	521
	522	return 0;
	523	}
	524	subsys_initcall(add_system_ram_resources);
1d54cf2b	525
	526	#ifdef CONFIG_STRICT_DEVMEM
	527	/*
	528	* devmem_is_allowed(): check to see if /dev/mem access to a certain address
	529	* is valid. The argument is a physical page number.
	530	*
	531	* Access has to be given to non-kernel-ram areas as well, these contain the
	532	* PCI mmio resources as well as potential bios/acpi data regions.
	533	*/
	534	int devmem_is_allowed(unsigned long pfn)
	535	{
	536	if (iomem_is_exclusive(pfn << PAGE_SHIFT))
	537	return 0;
	538	if (!page_is_ram(pfn))
	539	return 1;
8a3e3d31	540	if (page_is_rtas_user_buf(pfn))
8a3e3d31	541	return 1;
1d54cf2b	542	return 0;
	543	}
	544	#endif /* CONFIG_STRICT_DEVMEM */