[linux-2.6-block.git] / arch / powerpc / mm / tlb_nohash.c

/*
 * This file contains the routines for TLB flushing.
 * On machines where the MMU does not use a hash table to store virtual to
 * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
 * this does -not- include 603 however which shares the implementation with
 * hash based processors)
 *
 *  -- BenH
 *
 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
 *                     IBM Corp.
 *
 *  Derived from arch/ppc/mm/init.c:
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
 *    Copyright (C) 1996 Paul Mackerras
 *
 *  Derived from "arch/i386/mm/init.c"
 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version
 *  2 of the License, or (at your option) any later version.
 *
 */

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
#include <linux/hugetlb.h>

#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <asm/code-patching.h>
#include <asm/hugetlb.h>

#include "mmu_decl.h"

/*
 * This struct lists the sw-supported page sizes.  The hardawre MMU may support
 * other sizes not listed here.   The .ind field is only used on MMUs that have
 * indirect page table entries.
 */
#ifdef CONFIG_PPC_BOOK3E_MMU
#ifdef CONFIG_FSL_BOOKE
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.enc	= BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_4M] = {
		.shift	= 22,
		.enc	= BOOK3E_PAGESZ_4M,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.enc	= BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_64M] = {
		.shift	= 26,
		.enc	= BOOK3E_PAGESZ_64M,
	},
	[MMU_PAGE_256M] = {
		.shift	= 28,
		.enc	= BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
		.shift	= 30,
		.enc	= BOOK3E_PAGESZ_1GB,
	},
};
#else
struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
		.shift	= 12,
		.ind	= 20,
		.enc	= BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_16K] = {
		.shift	= 14,
		.enc	= BOOK3E_PAGESZ_16K,
	},
	[MMU_PAGE_64K] = {
		.shift	= 16,
		.ind	= 28,
		.enc	= BOOK3E_PAGESZ_64K,
	},
	[MMU_PAGE_1M] = {
		.shift	= 20,
		.enc	= BOOK3E_PAGESZ_1M,
	},
	[MMU_PAGE_16M] = {
		.shift	= 24,
		.ind	= 36,
		.enc	= BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_256M] = {
		.shift	= 28,
		.enc	= BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
		.shift	= 30,
		.enc	= BOOK3E_PAGESZ_1GB,
	},
};
#endif /* CONFIG_FSL_BOOKE */

static inline int mmu_get_tsize(int psize)
{
	return mmu_psize_defs[psize].enc;
}
#else
static inline int mmu_get_tsize(int psize)
{
	/* This isn't used on !Book3E for now */
	return 0;
}
#endif /* CONFIG_PPC_BOOK3E_MMU */

/* The variables below are currently only used on 64-bit Book3E
 * though this will probably be made common with other nohash
 * implementations at some point
 */
#ifdef CONFIG_PPC64

int mmu_linear_psize;		/* Page size used for the linear mapping */
int mmu_pte_psize;		/* Page size used for PTE pages */
int mmu_vmemmap_psize;		/* Page size used for the virtual mem map */
int book3e_htw_enabled;		/* Is HW tablewalk enabled ? */
unsigned long linear_map_top;	/* Top of linear mapping */

#endif /* CONFIG_PPC64 */

#ifdef CONFIG_PPC_FSL_BOOK3E
/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
DEFINE_PER_CPU(int, next_tlbcam_idx);
EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
#endif

/*
 * Base TLB flushing operations:
 *
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes kernel pages
 *
 *  - local_* variants of page and mm only apply to the current
 *    processor
 */

/*
 * These are the base non-SMP variants of page and mm flushing
 */
void local_flush_tlb_mm(struct mm_struct *mm)
{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
		_tlbil_pid(pid);
	preempt_enable();
}
EXPORT_SYMBOL(local_flush_tlb_mm);

void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
			    int tsize, int ind)
{
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (pid != MMU_NO_CONTEXT)
		_tlbil_va(vmaddr, pid, tsize, ind);
	preempt_enable();
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			       mmu_get_tsize(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(local_flush_tlb_page);

/*
 * And here are the SMP non-local implementations
 */
#ifdef CONFIG_SMP

static DEFINE_RAW_SPINLOCK(tlbivax_lock);

static int mm_is_core_local(struct mm_struct *mm)
{
	return cpumask_subset(mm_cpumask(mm),
			      topology_thread_cpumask(smp_processor_id()));
}

struct tlb_flush_param {
	unsigned long addr;
	unsigned int pid;
	unsigned int tsize;
	unsigned int ind;
};

static void do_flush_tlb_mm_ipi(void *param)
{
	struct tlb_flush_param *p = param;

	_tlbil_pid(p ? p->pid : 0);
}

static void do_flush_tlb_page_ipi(void *param)
{
	struct tlb_flush_param *p = param;

	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
}


/* Note on invalidations and PID:
 *
 * We snapshot the PID with preempt disabled. At this point, it can still
 * change either because:
 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
 * - we are invaliating some target that isn't currently running here
 *   and is concurrently acquiring a new PID on another CPU
 * - some other CPU is re-acquiring a lost PID for this mm
 * etc...
 *
 * However, this shouldn't be a problem as we only guarantee
 * invalidation of TLB entries present prior to this call, so we
 * don't care about the PID changing, and invalidating a stale PID
 * is generally harmless.
 */

void flush_tlb_mm(struct mm_struct *mm)
{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto no_context;
	if (!mm_is_core_local(mm)) {
		struct tlb_flush_param p = { .pid = pid };
		/* Ignores smp_processor_id() even if set. */
		smp_call_function_many(mm_cpumask(mm),
				       do_flush_tlb_mm_ipi, &p, 1);
	}
	_tlbil_pid(pid);
 no_context:
	preempt_enable();
}
EXPORT_SYMBOL(flush_tlb_mm);

void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
		      int tsize, int ind)
{
	struct cpumask *cpu_mask;
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
		goto bail;
	cpu_mask = mm_cpumask(mm);
	if (!mm_is_core_local(mm)) {
		/* If broadcast tlbivax is supported, use it */
		if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
			int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
			if (lock)
				raw_spin_lock(&tlbivax_lock);
			_tlbivax_bcast(vmaddr, pid, tsize, ind);
			if (lock)
				raw_spin_unlock(&tlbivax_lock);
			goto bail;
		} else {
			struct tlb_flush_param p = {
				.pid = pid,
				.addr = vmaddr,
				.tsize = tsize,
				.ind = ind,
			};
			/* Ignores smp_processor_id() even if set in cpu_mask */
			smp_call_function_many(cpu_mask,
					       do_flush_tlb_page_ipi, &p, 1);
		}
	}
	_tlbil_va(vmaddr, pid, tsize, ind);
 bail:
	preempt_enable();
}

void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
#ifdef CONFIG_HUGETLB_PAGE
	if (is_vm_hugetlb_page(vma))
		flush_hugetlb_page(vma, vmaddr);
#endif

	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
			 mmu_get_tsize(mmu_virtual_psize), 0);
}
EXPORT_SYMBOL(flush_tlb_page);

#endif /* CONFIG_SMP */

#ifdef CONFIG_PPC_47x
void __init early_init_mmu_47x(void)
{
#ifdef CONFIG_SMP
	unsigned long root = of_get_flat_dt_root();
	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
		mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
#endif /* CONFIG_SMP */
}
#endif /* CONFIG_PPC_47x */

/*
 * Flush kernel TLB entries in the given range
 */
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
#ifdef CONFIG_SMP
	preempt_disable();
	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
	_tlbil_pid(0);
	preempt_enable();
#else
	_tlbil_pid(0);
#endif
}
EXPORT_SYMBOL(flush_tlb_kernel_range);

/*
 * Currently, for range flushing, we just do a full mm flush. This should
 * be optimized based on a threshold on the size of the range, since
 * some implementation can stack multiple tlbivax before a tlbsync but
 * for now, we keep it that way
 */
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
		     unsigned long end)

{
	flush_tlb_mm(vma->vm_mm);
}
EXPORT_SYMBOL(flush_tlb_range);

void tlb_flush(struct mmu_gather *tlb)
{
	flush_tlb_mm(tlb->mm);
}

/*
 * Below are functions specific to the 64-bit variant of Book3E though that
 * may change in the future
 */

#ifdef CONFIG_PPC64

/*
 * Handling of virtual linear page tables or indirect TLB entries
 * flushing when PTE pages are freed
 */
void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
{
	int tsize = mmu_psize_defs[mmu_pte_psize].enc;

	if (book3e_htw_enabled) {
		unsigned long start = address & PMD_MASK;
		unsigned long end = address + PMD_SIZE;
		unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;

		/* This isn't the most optimal, ideally we would factor out the
		 * while preempt & CPU mask mucking around, or even the IPI but
		 * it will do for now
		 */
		while (start < end) {
			__flush_tlb_page(tlb->mm, start, tsize, 1);
			start += size;
		}
	} else {
		unsigned long rmask = 0xf000000000000000ul;
		unsigned long rid = (address & rmask) | 0x1000000000000000ul;
		unsigned long vpte = address & ~rmask;

#ifdef CONFIG_PPC_64K_PAGES
		vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
#else
		vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
#endif
		vpte |= rid;
		__flush_tlb_page(tlb->mm, vpte, tsize, 0);
	}
}

static void setup_page_sizes(void)
{
	unsigned int tlb0cfg;
	unsigned int tlb0ps;
	unsigned int eptcfg;
	int i, psize;

#ifdef CONFIG_PPC_FSL_BOOK3E
	unsigned int mmucfg = mfspr(SPRN_MMUCFG);

	if (((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) &&
		(mmu_has_feature(MMU_FTR_TYPE_FSL_E))) {
		unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
		unsigned int min_pg, max_pg;

		min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
		max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;

		for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
			struct mmu_psize_def *def;
			unsigned int shift;

			def = &mmu_psize_defs[psize];
			shift = def->shift;

			if (shift == 0)
				continue;

			/* adjust to be in terms of 4^shift Kb */
			shift = (shift - 10) >> 1;

			if ((shift >= min_pg) && (shift <= max_pg))
				def->flags |= MMU_PAGE_SIZE_DIRECT;
		}

		goto no_indirect;
	}
#endif

	tlb0cfg = mfspr(SPRN_TLB0CFG);
	tlb0ps = mfspr(SPRN_TLB0PS);
	eptcfg = mfspr(SPRN_EPTCFG);

	/* Look for supported direct sizes */
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
		struct mmu_psize_def *def = &mmu_psize_defs[psize];

		if (tlb0ps & (1U << (def->shift - 10)))
			def->flags |= MMU_PAGE_SIZE_DIRECT;
	}

	/* Indirect page sizes supported ? */
	if ((tlb0cfg & TLBnCFG_IND) == 0)
		goto no_indirect;

	/* Now, we only deal with one IND page size for each
	 * direct size. Hopefully all implementations today are
	 * unambiguous, but we might want to be careful in the
	 * future.
	 */
	for (i = 0; i < 3; i++) {
		unsigned int ps, sps;

		sps = eptcfg & 0x1f;
		eptcfg >>= 5;
		ps = eptcfg & 0x1f;
		eptcfg >>= 5;
		if (!ps || !sps)
			continue;
		for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
			struct mmu_psize_def *def = &mmu_psize_defs[psize];

			if (ps == (def->shift - 10))
				def->flags |= MMU_PAGE_SIZE_INDIRECT;
			if (sps == (def->shift - 10))
				def->ind = ps + 10;
		}
	}
 no_indirect:

	/* Cleanup array and print summary */
	pr_info("MMU: Supported page sizes\n");
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
		struct mmu_psize_def *def = &mmu_psize_defs[psize];
		const char *__page_type_names[] = {
			"unsupported",
			"direct",
			"indirect",
			"direct & indirect"
		};
		if (def->flags == 0) {
			def->shift = 0;	
			continue;
		}
		pr_info("  %8ld KB as %s\n", 1ul << (def->shift - 10),
			__page_type_names[def->flags & 0x3]);
	}
}

static void __patch_exception(int exc, unsigned long addr)
{
	extern unsigned int interrupt_base_book3e;
 	unsigned int *ibase = &interrupt_base_book3e;
 
	/* Our exceptions vectors start with a NOP and -then- a branch
	 * to deal with single stepping from userspace which stops on
	 * the second instruction. Thus we need to patch the second
	 * instruction of the exception, not the first one
	 */

	patch_branch(ibase + (exc / 4) + 1, addr, 0);
}

#define patch_exception(exc, name) do { \
	extern unsigned int name; \
	__patch_exception((exc), (unsigned long)&name); \
} while (0)

static void setup_mmu_htw(void)
{
	/* Check if HW tablewalk is present, and if yes, enable it by:
	 *
	 * - patching the TLB miss handlers to branch to the
	 *   one dedicates to it
	 *
	 * - setting the global book3e_htw_enabled
       	 */
	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);

	if ((tlb0cfg & TLBnCFG_IND) &&
	    (tlb0cfg & TLBnCFG_PT)) {
		patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
		patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
		book3e_htw_enabled = 1;
	}
	pr_info("MMU: Book3E HW tablewalk %s\n",
		book3e_htw_enabled ? "enabled" : "not supported");
}

/*
 * Early initialization of the MMU TLB code
 */
static void __early_init_mmu(int boot_cpu)
{
	unsigned int mas4;

	/* XXX This will have to be decided at runtime, but right
	 * now our boot and TLB miss code hard wires it. Ideally
	 * we should find out a suitable page size and patch the
	 * TLB miss code (either that or use the PACA to store
	 * the value we want)
	 */
	mmu_linear_psize = MMU_PAGE_1G;

	/* XXX This should be decided at runtime based on supported
	 * page sizes in the TLB, but for now let's assume 16M is
	 * always there and a good fit (which it probably is)
	 */
	mmu_vmemmap_psize = MMU_PAGE_16M;

	/* XXX This code only checks for TLB 0 capabilities and doesn't
	 *     check what page size combos are supported by the HW. It
	 *     also doesn't handle the case where a separate array holds
	 *     the IND entries from the array loaded by the PT.
	 */
	if (boot_cpu) {
		/* Look for supported page sizes */
		setup_page_sizes();

		/* Look for HW tablewalk support */
		setup_mmu_htw();
	}

	/* Set MAS4 based on page table setting */

	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
	if (book3e_htw_enabled) {
		mas4 |= mas4 | MAS4_INDD;
#ifdef CONFIG_PPC_64K_PAGES
		mas4 |=	BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
		mmu_pte_psize = MMU_PAGE_256M;
#else
		mas4 |=	BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
		mmu_pte_psize = MMU_PAGE_1M;
#endif
	} else {
#ifdef CONFIG_PPC_64K_PAGES
		mas4 |=	BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
#else
		mas4 |=	BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
#endif
		mmu_pte_psize = mmu_virtual_psize;
	}
	mtspr(SPRN_MAS4, mas4);

	/* Set the global containing the top of the linear mapping
	 * for use by the TLB miss code
	 */
	linear_map_top = memblock_end_of_DRAM();

#ifdef CONFIG_PPC_FSL_BOOK3E
	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
		unsigned int num_cams;

		/* use a quarter of the TLBCAM for bolted linear map */
		num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
		linear_map_top = map_mem_in_cams(linear_map_top, num_cams);

		/* limit memory so we dont have linear faults */
		memblock_enforce_memory_limit(linear_map_top);
		memblock_analyze();

		patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
		patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
	}
#endif

	/* A sync won't hurt us after mucking around with
	 * the MMU configuration
	 */
	mb();

	memblock_set_current_limit(linear_map_top);
}

void __init early_init_mmu(void)
{
	__early_init_mmu(1);
}

void __cpuinit early_init_mmu_secondary(void)
{
	__early_init_mmu(0);
}

void setup_initial_memory_limit(phys_addr_t first_memblock_base,
				phys_addr_t first_memblock_size)
{
	/* On Embedded 64-bit, we adjust the RMA size to match
	 * the bolted TLB entry. We know for now that only 1G
	 * entries are supported though that may eventually
	 * change. We crop it to the size of the first MEMBLOCK to
	 * avoid going over total available memory just in case...
	 */
	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);

	/* Finally limit subsequent allocations */
	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
}
#else /* ! CONFIG_PPC64 */
void __init early_init_mmu(void)
{
#ifdef CONFIG_PPC_47x
	early_init_mmu_47x();
#endif
}
#endif /* CONFIG_PPC64 */
Commit	Line	Data
f048aace BH	1	/*
	2	* This file contains the routines for TLB flushing.
	3	* On machines where the MMU does not use a hash table to store virtual to
	4	* physical translations (ie, SW loaded TLBs or Book3E compilant processors,
	5	* this does -not- include 603 however which shares the implementation with
	6	* hash based processors)
	7	*
	8	* -- BenH
	9	*
25d21ad6 BH	10	* Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
25d21ad6 BH	11	* IBM Corp.
f048aace BH	12	*
	13	* Derived from arch/ppc/mm/init.c:
	14	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	15	*
	16	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	17	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	18	* Copyright (C) 1996 Paul Mackerras
	19	*
	20	* Derived from "arch/i386/mm/init.c"
	21	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	22	*
	23	* This program is free software; you can redistribute it and/or
	24	* modify it under the terms of the GNU General Public License
	25	* as published by the Free Software Foundation; either version
	26	* 2 of the License, or (at your option) any later version.
	27	*
	28	*/
	29
	30	#include <linux/kernel.h>
	31	#include <linux/mm.h>
	32	#include <linux/init.h>
	33	#include <linux/highmem.h>
	34	#include <linux/pagemap.h>
	35	#include <linux/preempt.h>
	36	#include <linux/spinlock.h>
95f72d1e	37	#include <linux/memblock.h>
91b191c7	38	#include <linux/of_fdt.h>
41151e77	39	#include <linux/hugetlb.h>
f048aace BH	40
	41	#include <asm/tlbflush.h>
	42	#include <asm/tlb.h>
25d21ad6	43	#include <asm/code-patching.h>
41151e77	44	#include <asm/hugetlb.h>
f048aace BH	45
	46	#include "mmu_decl.h"
	47
41151e77 BB	48	/*
	49	* This struct lists the sw-supported page sizes. The hardawre MMU may support
	50	* other sizes not listed here. The .ind field is only used on MMUs that have
	51	* indirect page table entries.
	52	*/
	53	#ifdef CONFIG_PPC_BOOK3E_MMU
	54	#ifdef CONFIG_FSL_BOOKE
	55	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	56	[MMU_PAGE_4K] = {
	57	.shift = 12,
	58	.enc = BOOK3E_PAGESZ_4K,
	59	},
	60	[MMU_PAGE_4M] = {
	61	.shift = 22,
	62	.enc = BOOK3E_PAGESZ_4M,
	63	},
	64	[MMU_PAGE_16M] = {
	65	.shift = 24,
	66	.enc = BOOK3E_PAGESZ_16M,
	67	},
	68	[MMU_PAGE_64M] = {
	69	.shift = 26,
	70	.enc = BOOK3E_PAGESZ_64M,
	71	},
	72	[MMU_PAGE_256M] = {
	73	.shift = 28,
	74	.enc = BOOK3E_PAGESZ_256M,
	75	},
	76	[MMU_PAGE_1G] = {
	77	.shift = 30,
	78	.enc = BOOK3E_PAGESZ_1GB,
	79	},
	80	};
	81	#else
25d21ad6 BH	82	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	83	[MMU_PAGE_4K] = {
	84	.shift = 12,
f2b26c92	85	.ind = 20,
25d21ad6 BH	86	.enc = BOOK3E_PAGESZ_4K,
	87	},
	88	[MMU_PAGE_16K] = {
	89	.shift = 14,
	90	.enc = BOOK3E_PAGESZ_16K,
	91	},
	92	[MMU_PAGE_64K] = {
	93	.shift = 16,
f2b26c92	94	.ind = 28,
25d21ad6 BH	95	.enc = BOOK3E_PAGESZ_64K,
	96	},
	97	[MMU_PAGE_1M] = {
	98	.shift = 20,
	99	.enc = BOOK3E_PAGESZ_1M,
	100	},
	101	[MMU_PAGE_16M] = {
	102	.shift = 24,
f2b26c92	103	.ind = 36,
25d21ad6 BH	104	.enc = BOOK3E_PAGESZ_16M,
	105	},
	106	[MMU_PAGE_256M] = {
	107	.shift = 28,
	108	.enc = BOOK3E_PAGESZ_256M,
	109	},
	110	[MMU_PAGE_1G] = {
	111	.shift = 30,
	112	.enc = BOOK3E_PAGESZ_1GB,
	113	},
	114	};
41151e77 BB	115	#endif /* CONFIG_FSL_BOOKE */
41151e77 BB	116
25d21ad6 BH	117	static inline int mmu_get_tsize(int psize)
	118	{
	119	return mmu_psize_defs[psize].enc;
	120	}
	121	#else
	122	static inline int mmu_get_tsize(int psize)
	123	{
	124	/* This isn't used on !Book3E for now */
	125	return 0;
	126	}
41151e77	127	#endif /* CONFIG_PPC_BOOK3E_MMU */
25d21ad6 BH	128
	129	/* The variables below are currently only used on 64-bit Book3E
	130	* though this will probably be made common with other nohash
	131	* implementations at some point
	132	*/
	133	#ifdef CONFIG_PPC64
	134
	135	int mmu_linear_psize; /* Page size used for the linear mapping */
	136	int mmu_pte_psize; /* Page size used for PTE pages */
32a74949	137	int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
25d21ad6 BH	138	int book3e_htw_enabled; /* Is HW tablewalk enabled ? */
	139	unsigned long linear_map_top; /* Top of linear mapping */
	140
	141	#endif /* CONFIG_PPC64 */
	142
3160b097 BB	143	#ifdef CONFIG_PPC_FSL_BOOK3E
	144	/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
	145	DEFINE_PER_CPU(int, next_tlbcam_idx);
	146	EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
	147	#endif
	148
f048aace BH	149	/*
	150	* Base TLB flushing operations:
	151	*
	152	* - flush_tlb_mm(mm) flushes the specified mm context TLB's
	153	* - flush_tlb_page(vma, vmaddr) flushes one page
	154	* - flush_tlb_range(vma, start, end) flushes a range of pages
	155	* - flush_tlb_kernel_range(start, end) flushes kernel pages
	156	*
	157	* - local_* variants of page and mm only apply to the current
	158	* processor
	159	*/
	160
	161	/*
	162	* These are the base non-SMP variants of page and mm flushing
	163	*/
	164	void local_flush_tlb_mm(struct mm_struct *mm)
	165	{
	166	unsigned int pid;
	167
	168	preempt_disable();
	169	pid = mm->context.id;
	170	if (pid != MMU_NO_CONTEXT)
	171	_tlbil_pid(pid);
	172	preempt_enable();
	173	}
	174	EXPORT_SYMBOL(local_flush_tlb_mm);
	175
d4e167da BH	176	void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
d4e167da BH	177	int tsize, int ind)
f048aace BH	178	{
	179	unsigned int pid;
	180
	181	preempt_disable();
d4e167da	182	pid = mm ? mm->context.id : 0;
f048aace	183	if (pid != MMU_NO_CONTEXT)
d4e167da	184	_tlbil_va(vmaddr, pid, tsize, ind);
f048aace BH	185	preempt_enable();
f048aace BH	186	}
f048aace	187
d4e167da BH	188	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	189	{
	190	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
25d21ad6	191	mmu_get_tsize(mmu_virtual_psize), 0);
d4e167da BH	192	}
d4e167da BH	193	EXPORT_SYMBOL(local_flush_tlb_page);
f048aace BH	194
	195	/*
	196	* And here are the SMP non-local implementations
	197	*/
	198	#ifdef CONFIG_SMP
	199
3eb93c55	200	static DEFINE_RAW_SPINLOCK(tlbivax_lock);
f048aace	201
fcce8109 BH	202	static int mm_is_core_local(struct mm_struct *mm)
	203	{
	204	return cpumask_subset(mm_cpumask(mm),
	205	topology_thread_cpumask(smp_processor_id()));
	206	}
	207
f048aace BH	208	struct tlb_flush_param {
	209	unsigned long addr;
	210	unsigned int pid;
d4e167da BH	211	unsigned int tsize;
d4e167da BH	212	unsigned int ind;
f048aace BH	213	};
	214
	215	static void do_flush_tlb_mm_ipi(void *param)
	216	{
	217	struct tlb_flush_param *p = param;
	218
	219	_tlbil_pid(p ? p->pid : 0);
	220	}
	221
	222	static void do_flush_tlb_page_ipi(void *param)
	223	{
	224	struct tlb_flush_param *p = param;
	225
d4e167da	226	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
f048aace BH	227	}
	228
	229
	230	/* Note on invalidations and PID:
	231	*
	232	* We snapshot the PID with preempt disabled. At this point, it can still
	233	* change either because:
	234	* - our context is being stolen (PID -> NO_CONTEXT) on another CPU
	235	* - we are invaliating some target that isn't currently running here
	236	* and is concurrently acquiring a new PID on another CPU
	237	* - some other CPU is re-acquiring a lost PID for this mm
	238	* etc...
	239	*
	240	* However, this shouldn't be a problem as we only guarantee
	241	* invalidation of TLB entries present prior to this call, so we
	242	* don't care about the PID changing, and invalidating a stale PID
	243	* is generally harmless.
	244	*/
	245
	246	void flush_tlb_mm(struct mm_struct *mm)
	247	{
f048aace BH	248	unsigned int pid;
	249
	250	preempt_disable();
	251	pid = mm->context.id;
	252	if (unlikely(pid == MMU_NO_CONTEXT))
	253	goto no_context;
fcce8109	254	if (!mm_is_core_local(mm)) {
f048aace	255	struct tlb_flush_param p = { .pid = pid };
56aa4129 RR	256	/* Ignores smp_processor_id() even if set. */
	257	smp_call_function_many(mm_cpumask(mm),
	258	do_flush_tlb_mm_ipi, &p, 1);
f048aace BH	259	}
	260	_tlbil_pid(pid);
	261	no_context:
	262	preempt_enable();
	263	}
	264	EXPORT_SYMBOL(flush_tlb_mm);
	265
d4e167da BH	266	void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
d4e167da BH	267	int tsize, int ind)
f048aace	268	{
56aa4129	269	struct cpumask *cpu_mask;
f048aace BH	270	unsigned int pid;
	271
	272	preempt_disable();
d4e167da	273	pid = mm ? mm->context.id : 0;
f048aace BH	274	if (unlikely(pid == MMU_NO_CONTEXT))
f048aace BH	275	goto bail;
d4e167da	276	cpu_mask = mm_cpumask(mm);
fcce8109	277	if (!mm_is_core_local(mm)) {
f048aace BH	278	/* If broadcast tlbivax is supported, use it */
	279	if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
	280	int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
	281	if (lock)
3eb93c55	282	raw_spin_lock(&tlbivax_lock);
d4e167da	283	_tlbivax_bcast(vmaddr, pid, tsize, ind);
f048aace	284	if (lock)
3eb93c55	285	raw_spin_unlock(&tlbivax_lock);
f048aace BH	286	goto bail;
f048aace BH	287	} else {
d4e167da BH	288	struct tlb_flush_param p = {
	289	.pid = pid,
	290	.addr = vmaddr,
	291	.tsize = tsize,
	292	.ind = ind,
	293	};
56aa4129 RR	294	/* Ignores smp_processor_id() even if set in cpu_mask */
56aa4129 RR	295	smp_call_function_many(cpu_mask,
f048aace BH	296	do_flush_tlb_page_ipi, &p, 1);
	297	}
	298	}
d4e167da	299	_tlbil_va(vmaddr, pid, tsize, ind);
f048aace BH	300	bail:
	301	preempt_enable();
	302	}
d4e167da BH	303
	304	void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	305	{
41151e77 BB	306	#ifdef CONFIG_HUGETLB_PAGE
	307	if (is_vm_hugetlb_page(vma))
	308	flush_hugetlb_page(vma, vmaddr);
	309	#endif
	310
d4e167da	311	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
25d21ad6	312	mmu_get_tsize(mmu_virtual_psize), 0);
d4e167da	313	}
f048aace BH	314	EXPORT_SYMBOL(flush_tlb_page);
	315
	316	#endif /* CONFIG_SMP */
	317
91b191c7 DK	318	#ifdef CONFIG_PPC_47x
	319	void __init early_init_mmu_47x(void)
	320	{
	321	#ifdef CONFIG_SMP
	322	unsigned long root = of_get_flat_dt_root();
	323	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
	324	mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
	325	#endif /* CONFIG_SMP */
	326	}
	327	#endif /* CONFIG_PPC_47x */
	328
f048aace BH	329	/*
	330	* Flush kernel TLB entries in the given range
	331	*/
	332	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
	333	{
	334	#ifdef CONFIG_SMP
	335	preempt_disable();
	336	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
	337	_tlbil_pid(0);
	338	preempt_enable();
d6a09e0c	339	#else
f048aace	340	_tlbil_pid(0);
d6a09e0c	341	#endif
f048aace BH	342	}
	343	EXPORT_SYMBOL(flush_tlb_kernel_range);
	344
	345	/*
	346	* Currently, for range flushing, we just do a full mm flush. This should
	347	* be optimized based on a threshold on the size of the range, since
	348	* some implementation can stack multiple tlbivax before a tlbsync but
	349	* for now, we keep it that way
	350	*/
	351	void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	352	unsigned long end)
	353
	354	{
	355	flush_tlb_mm(vma->vm_mm);
	356	}
	357	EXPORT_SYMBOL(flush_tlb_range);
c7cc58a1 BH	358
	359	void tlb_flush(struct mmu_gather *tlb)
	360	{
	361	flush_tlb_mm(tlb->mm);
c7cc58a1	362	}
25d21ad6 BH	363
	364	/*
	365	* Below are functions specific to the 64-bit variant of Book3E though that
	366	* may change in the future
	367	*/
	368
	369	#ifdef CONFIG_PPC64
	370
	371	/*
	372	* Handling of virtual linear page tables or indirect TLB entries
	373	* flushing when PTE pages are freed
	374	*/
	375	void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
	376	{
	377	int tsize = mmu_psize_defs[mmu_pte_psize].enc;
	378
	379	if (book3e_htw_enabled) {
	380	unsigned long start = address & PMD_MASK;
	381	unsigned long end = address + PMD_SIZE;
	382	unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
	383
	384	/* This isn't the most optimal, ideally we would factor out the
	385	* while preempt & CPU mask mucking around, or even the IPI but
	386	* it will do for now
	387	*/
	388	while (start < end) {
	389	__flush_tlb_page(tlb->mm, start, tsize, 1);
	390	start += size;
	391	}
	392	} else {
	393	unsigned long rmask = 0xf000000000000000ul;
	394	unsigned long rid = (address & rmask) \| 0x1000000000000000ul;
	395	unsigned long vpte = address & ~rmask;
	396
	397	#ifdef CONFIG_PPC_64K_PAGES
	398	vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
	399	#else
	400	vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
	401	#endif
	402	vpte \|= rid;
	403	__flush_tlb_page(tlb->mm, vpte, tsize, 0);
	404	}
	405	}
	406
f2b26c92 BH	407	static void setup_page_sizes(void)
f2b26c92 BH	408	{
988cf86d KG	409	unsigned int tlb0cfg;
	410	unsigned int tlb0ps;
	411	unsigned int eptcfg;
f2b26c92 BH	412	int i, psize;
f2b26c92 BH	413
988cf86d KG	414	#ifdef CONFIG_PPC_FSL_BOOK3E
	415	unsigned int mmucfg = mfspr(SPRN_MMUCFG);
	416
	417	if (((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) &&
	418	(mmu_has_feature(MMU_FTR_TYPE_FSL_E))) {
	419	unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
	420	unsigned int min_pg, max_pg;
	421
	422	min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
	423	max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
	424
	425	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	426	struct mmu_psize_def *def;
	427	unsigned int shift;
	428
	429	def = &mmu_psize_defs[psize];
	430	shift = def->shift;
	431
	432	if (shift == 0)
	433	continue;
	434
	435	/* adjust to be in terms of 4^shift Kb */
	436	shift = (shift - 10) >> 1;
	437
	438	if ((shift >= min_pg) && (shift <= max_pg))
	439	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	440	}
	441
	442	goto no_indirect;
	443	}
	444	#endif
	445
	446	tlb0cfg = mfspr(SPRN_TLB0CFG);
	447	tlb0ps = mfspr(SPRN_TLB0PS);
	448	eptcfg = mfspr(SPRN_EPTCFG);
	449
f2b26c92 BH	450	/* Look for supported direct sizes */
	451	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	452	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	453
	454	if (tlb0ps & (1U << (def->shift - 10)))
	455	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	456	}
	457
	458	/* Indirect page sizes supported ? */
	459	if ((tlb0cfg & TLBnCFG_IND) == 0)
	460	goto no_indirect;
	461
	462	/* Now, we only deal with one IND page size for each
	463	* direct size. Hopefully all implementations today are
	464	* unambiguous, but we might want to be careful in the
	465	* future.
	466	*/
	467	for (i = 0; i < 3; i++) {
	468	unsigned int ps, sps;
	469
	470	sps = eptcfg & 0x1f;
	471	eptcfg >>= 5;
	472	ps = eptcfg & 0x1f;
	473	eptcfg >>= 5;
	474	if (!ps \|\| !sps)
	475	continue;
	476	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
	477	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	478
	479	if (ps == (def->shift - 10))
	480	def->flags \|= MMU_PAGE_SIZE_INDIRECT;
	481	if (sps == (def->shift - 10))
	482	def->ind = ps + 10;
	483	}
	484	}
	485	no_indirect:
	486
	487	/* Cleanup array and print summary */
	488	pr_info("MMU: Supported page sizes\n");
	489	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	490	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	491	const char *__page_type_names[] = {
	492	"unsupported",
	493	"direct",
	494	"indirect",
	495	"direct & indirect"
	496	};
	497	if (def->flags == 0) {
	498	def->shift = 0;
	499	continue;
	500	}
	501	pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10),
	502	__page_type_names[def->flags & 0x3]);
	503	}
	504	}
	505
f67f4ef5	506	static void __patch_exception(int exc, unsigned long addr)
25d21ad6 BH	507	{
25d21ad6 BH	508	extern unsigned int interrupt_base_book3e;
f67f4ef5 SW	509	unsigned int *ibase = &interrupt_base_book3e;
	510
	511	/* Our exceptions vectors start with a NOP and -then- a branch
	512	* to deal with single stepping from userspace which stops on
	513	* the second instruction. Thus we need to patch the second
	514	* instruction of the exception, not the first one
	515	*/
	516
	517	patch_branch(ibase + (exc / 4) + 1, addr, 0);
	518	}
25d21ad6	519
f67f4ef5 SW	520	#define patch_exception(exc, name) do { \
	521	extern unsigned int name; \
	522	__patch_exception((exc), (unsigned long)&name); \
	523	} while (0)
f2b26c92	524
f67f4ef5 SW	525	static void setup_mmu_htw(void)
f67f4ef5 SW	526	{
f2b26c92 BH	527	/* Check if HW tablewalk is present, and if yes, enable it by:
	528	*
	529	* - patching the TLB miss handlers to branch to the
	530	* one dedicates to it
	531	*
	532	* - setting the global book3e_htw_enabled
	533	*/
	534	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);
	535
	536	if ((tlb0cfg & TLBnCFG_IND) &&
	537	(tlb0cfg & TLBnCFG_PT)) {
f67f4ef5 SW	538	patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
f67f4ef5 SW	539	patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
f2b26c92 BH	540	book3e_htw_enabled = 1;
f2b26c92 BH	541	}
32d206eb KG	542	pr_info("MMU: Book3E HW tablewalk %s\n",
32d206eb KG	543	book3e_htw_enabled ? "enabled" : "not supported");
f2b26c92 BH	544	}
	545
	546	/*
	547	* Early initialization of the MMU TLB code
	548	*/
	549	static void __early_init_mmu(int boot_cpu)
	550	{
25d21ad6 BH	551	unsigned int mas4;
	552
	553	/* XXX This will have to be decided at runtime, but right
32a74949 BH	554	* now our boot and TLB miss code hard wires it. Ideally
	555	* we should find out a suitable page size and patch the
	556	* TLB miss code (either that or use the PACA to store
	557	* the value we want)
25d21ad6 BH	558	*/
	559	mmu_linear_psize = MMU_PAGE_1G;
	560
32a74949 BH	561	/* XXX This should be decided at runtime based on supported
	562	* page sizes in the TLB, but for now let's assume 16M is
	563	* always there and a good fit (which it probably is)
	564	*/
	565	mmu_vmemmap_psize = MMU_PAGE_16M;
25d21ad6	566
25d21ad6 BH	567	/* XXX This code only checks for TLB 0 capabilities and doesn't
	568	* check what page size combos are supported by the HW. It
	569	* also doesn't handle the case where a separate array holds
	570	* the IND entries from the array loaded by the PT.
	571	*/
	572	if (boot_cpu) {
f2b26c92 BH	573	/* Look for supported page sizes */
f2b26c92 BH	574	setup_page_sizes();
25d21ad6	575
f2b26c92 BH	576	/* Look for HW tablewalk support */
f2b26c92 BH	577	setup_mmu_htw();
25d21ad6 BH	578	}
	579
	580	/* Set MAS4 based on page table setting */
	581
	582	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
	583	if (book3e_htw_enabled) {
	584	mas4 \|= mas4 \| MAS4_INDD;
	585	#ifdef CONFIG_PPC_64K_PAGES
	586	mas4 \|= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
	587	mmu_pte_psize = MMU_PAGE_256M;
	588	#else
	589	mas4 \|= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
	590	mmu_pte_psize = MMU_PAGE_1M;
	591	#endif
	592	} else {
	593	#ifdef CONFIG_PPC_64K_PAGES
	594	mas4 \|= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
	595	#else
	596	mas4 \|= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
	597	#endif
	598	mmu_pte_psize = mmu_virtual_psize;
	599	}
	600	mtspr(SPRN_MAS4, mas4);
	601
	602	/* Set the global containing the top of the linear mapping
	603	* for use by the TLB miss code
	604	*/
95f72d1e	605	linear_map_top = memblock_end_of_DRAM();
25d21ad6	606
55fd766b KG	607	#ifdef CONFIG_PPC_FSL_BOOK3E
	608	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
	609	unsigned int num_cams;
	610
	611	/* use a quarter of the TLBCAM for bolted linear map */
	612	num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
	613	linear_map_top = map_mem_in_cams(linear_map_top, num_cams);
	614
	615	/* limit memory so we dont have linear faults */
	616	memblock_enforce_memory_limit(linear_map_top);
	617	memblock_analyze();
f67f4ef5 SW	618
	619	patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
	620	patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
55fd766b KG	621	}
	622	#endif
	623
25d21ad6 BH	624	/* A sync won't hurt us after mucking around with
	625	* the MMU configuration
	626	*/
	627	mb();
e63075a3 BH	628
e63075a3 BH	629	memblock_set_current_limit(linear_map_top);
25d21ad6 BH	630	}
	631
	632	void __init early_init_mmu(void)
	633	{
	634	__early_init_mmu(1);
	635	}
	636
	637	void __cpuinit early_init_mmu_secondary(void)
	638	{
	639	__early_init_mmu(0);
	640	}
	641
cd3db0c4 BH	642	void setup_initial_memory_limit(phys_addr_t first_memblock_base,
	643	phys_addr_t first_memblock_size)
	644	{
	645	/* On Embedded 64-bit, we adjust the RMA size to match
	646	* the bolted TLB entry. We know for now that only 1G
	647	* entries are supported though that may eventually
	648	* change. We crop it to the size of the first MEMBLOCK to
	649	* avoid going over total available memory just in case...
	650	*/
	651	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
	652
	653	/* Finally limit subsequent allocations */
4a89261b	654	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
cd3db0c4	655	}
91b191c7 DK	656	#else /* ! CONFIG_PPC64 */
	657	void __init early_init_mmu(void)
	658	{
	659	#ifdef CONFIG_PPC_47x
	660	early_init_mmu_47x();
	661	#endif
	662	}
25d21ad6	663	#endif /* CONFIG_PPC64 */