[linux-2.6-block.git] / arch / powerpc / kernel / misc_64.S

/*
 * This file contains miscellaneous low-level functions.
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *
 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
 * and Paul Mackerras.
 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 */

#include <linux/sys.h>
#include <asm/unistd.h>
#include <asm/errno.h>
#include <asm/processor.h>
#include <asm/page.h>
#include <asm/cache.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/kexec.h>
#include <asm/ptrace.h>
#include <asm/mmu.h>
#include <asm/export.h>

	.text

_GLOBAL(call_do_softirq)
	mflr	r0
	std	r0,16(r1)
	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
	mr	r1,r3
	bl	__do_softirq
	ld	r1,0(r1)
	ld	r0,16(r1)
	mtlr	r0
	blr

_GLOBAL(call_do_irq)
	mflr	r0
	std	r0,16(r1)
	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
	mr	r1,r4
	bl	__do_irq
	ld	r1,0(r1)
	ld	r0,16(r1)
	mtlr	r0
	blr

	.section	".toc","aw"
PPC64_CACHES:
	.tc		ppc64_caches[TC],ppc64_caches
	.section	".text"

/*
 * Write any modified data cache blocks out to memory
 * and invalidate the corresponding instruction cache blocks.
 *
 * flush_icache_range(unsigned long start, unsigned long stop)
 *
 *   flush all bytes from start through stop-1 inclusive
 */

_GLOBAL_TOC(flush_icache_range)
BEGIN_FTR_SECTION
	PURGE_PREFETCHED_INS
	blr
END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
/*
 * Flush the data cache to memory 
 * 
 * Different systems have different cache line sizes
 * and in some cases i-cache and d-cache line sizes differ from
 * each other.
 */
 	ld	r10,PPC64_CACHES@toc(r2)
	lwz	r7,DCACHEL1BLOCKSIZE(r10)/* Get cache block size */
	addi	r5,r7,-1
	andc	r6,r3,r5		/* round low to line bdy */
	subf	r8,r6,r4		/* compute length */
	add	r8,r8,r5		/* ensure we get enough */
	lwz	r9,DCACHEL1LOGBLOCKSIZE(r10)	/* Get log-2 of cache block size */
	srw.	r8,r8,r9		/* compute line count */
	beqlr				/* nothing to do? */
	mtctr	r8
1:	dcbst	0,r6
	add	r6,r6,r7
	bdnz	1b
	sync

/* Now invalidate the instruction cache */
	
	lwz	r7,ICACHEL1BLOCKSIZE(r10)	/* Get Icache block size */
	addi	r5,r7,-1
	andc	r6,r3,r5		/* round low to line bdy */
	subf	r8,r6,r4		/* compute length */
	add	r8,r8,r5
	lwz	r9,ICACHEL1LOGBLOCKSIZE(r10)	/* Get log-2 of Icache block size */
	srw.	r8,r8,r9		/* compute line count */
	beqlr				/* nothing to do? */
	mtctr	r8
2:	icbi	0,r6
	add	r6,r6,r7
	bdnz	2b
	isync
	blr
_ASM_NOKPROBE_SYMBOL(flush_icache_range)
EXPORT_SYMBOL(flush_icache_range)

/*
 * Like above, but only do the D-cache.
 *
 * flush_dcache_range(unsigned long start, unsigned long stop)
 *
 *    flush all bytes from start to stop-1 inclusive
 */
_GLOBAL_TOC(flush_dcache_range)

/*
 * Flush the data cache to memory 
 * 
 * Different systems have different cache line sizes
 */
 	ld	r10,PPC64_CACHES@toc(r2)
	lwz	r7,DCACHEL1BLOCKSIZE(r10)	/* Get dcache block size */
	addi	r5,r7,-1
	andc	r6,r3,r5		/* round low to line bdy */
	subf	r8,r6,r4		/* compute length */
	add	r8,r8,r5		/* ensure we get enough */
	lwz	r9,DCACHEL1LOGBLOCKSIZE(r10)	/* Get log-2 of dcache block size */
	srw.	r8,r8,r9		/* compute line count */
	beqlr				/* nothing to do? */
	mtctr	r8
0:	dcbst	0,r6
	add	r6,r6,r7
	bdnz	0b
	sync
	blr
EXPORT_SYMBOL(flush_dcache_range)

_GLOBAL(flush_inval_dcache_range)
 	ld	r10,PPC64_CACHES@toc(r2)
	lwz	r7,DCACHEL1BLOCKSIZE(r10)	/* Get dcache block size */
	addi	r5,r7,-1
	andc	r6,r3,r5		/* round low to line bdy */
	subf	r8,r6,r4		/* compute length */
	add	r8,r8,r5		/* ensure we get enough */
	lwz	r9,DCACHEL1LOGBLOCKSIZE(r10)/* Get log-2 of dcache block size */
	srw.	r8,r8,r9		/* compute line count */
	beqlr				/* nothing to do? */
	sync
	isync
	mtctr	r8
0:	dcbf	0,r6
	add	r6,r6,r7
	bdnz	0b
	sync
	isync
	blr


/*
 * Flush a particular page from the data cache to RAM.
 * Note: this is necessary because the instruction cache does *not*
 * snoop from the data cache.
 *
 *	void __flush_dcache_icache(void *page)
 */
_GLOBAL(__flush_dcache_icache)
/*
 * Flush the data cache to memory 
 * 
 * Different systems have different cache line sizes
 */

BEGIN_FTR_SECTION
	PURGE_PREFETCHED_INS
	blr
END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)

/* Flush the dcache */
 	ld	r7,PPC64_CACHES@toc(r2)
	clrrdi	r3,r3,PAGE_SHIFT           	    /* Page align */
	lwz	r4,DCACHEL1BLOCKSPERPAGE(r7)	/* Get # dcache blocks per page */
	lwz	r5,DCACHEL1BLOCKSIZE(r7)	/* Get dcache block size */
	mr	r6,r3
	mtctr	r4
0:	dcbst	0,r6
	add	r6,r6,r5
	bdnz	0b
	sync

/* Now invalidate the icache */	

	lwz	r4,ICACHEL1BLOCKSPERPAGE(r7)	/* Get # icache blocks per page */
	lwz	r5,ICACHEL1BLOCKSIZE(r7)	/* Get icache block size */
	mtctr	r4
1:	icbi	0,r3
	add	r3,r3,r5
	bdnz	1b
	isync
	blr

_GLOBAL(__bswapdi2)
EXPORT_SYMBOL(__bswapdi2)
	srdi	r8,r3,32
	rlwinm	r7,r3,8,0xffffffff
	rlwimi	r7,r3,24,0,7
	rlwinm	r9,r8,8,0xffffffff
	rlwimi	r7,r3,24,16,23
	rlwimi	r9,r8,24,0,7
	rlwimi	r9,r8,24,16,23
	sldi	r7,r7,32
	or	r3,r7,r9
	blr


#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
_GLOBAL(rmci_on)
	sync
	isync
	li	r3,0x100
	rldicl	r3,r3,32,0
	mfspr	r5,SPRN_HID4
	or	r5,r5,r3
	sync
	mtspr	SPRN_HID4,r5
	isync
	slbia
	isync
	sync
	blr

_GLOBAL(rmci_off)
	sync
	isync
	li	r3,0x100
	rldicl	r3,r3,32,0
	mfspr	r5,SPRN_HID4
	andc	r5,r5,r3
	sync
	mtspr	SPRN_HID4,r5
	isync
	slbia
	isync
	sync
	blr
#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */

#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)

/*
 * Do an IO access in real mode
 */
_GLOBAL(real_readb)
	mfmsr	r7
	ori	r0,r7,MSR_DR
	xori	r0,r0,MSR_DR
	sync
	mtmsrd	r0
	sync
	isync
	mfspr	r6,SPRN_HID4
	rldicl	r5,r6,32,0
	ori	r5,r5,0x100
	rldicl	r5,r5,32,0
	sync
	mtspr	SPRN_HID4,r5
	isync
	slbia
	isync
	lbz	r3,0(r3)
	sync
	mtspr	SPRN_HID4,r6
	isync
	slbia
	isync
	mtmsrd	r7
	sync
	isync
	blr

	/*
 * Do an IO access in real mode
 */
_GLOBAL(real_writeb)
	mfmsr	r7
	ori	r0,r7,MSR_DR
	xori	r0,r0,MSR_DR
	sync
	mtmsrd	r0
	sync
	isync
	mfspr	r6,SPRN_HID4
	rldicl	r5,r6,32,0
	ori	r5,r5,0x100
	rldicl	r5,r5,32,0
	sync
	mtspr	SPRN_HID4,r5
	isync
	slbia
	isync
	stb	r3,0(r4)
	sync
	mtspr	SPRN_HID4,r6
	isync
	slbia
	isync
	mtmsrd	r7
	sync
	isync
	blr
#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */

#ifdef CONFIG_PPC_PASEMI

_GLOBAL(real_205_readb)
	mfmsr	r7
	ori	r0,r7,MSR_DR
	xori	r0,r0,MSR_DR
	sync
	mtmsrd	r0
	sync
	isync
	LBZCIX(R3,R0,R3)
	isync
	mtmsrd	r7
	sync
	isync
	blr

_GLOBAL(real_205_writeb)
	mfmsr	r7
	ori	r0,r7,MSR_DR
	xori	r0,r0,MSR_DR
	sync
	mtmsrd	r0
	sync
	isync
	STBCIX(R3,R0,R4)
	isync
	mtmsrd	r7
	sync
	isync
	blr

#endif /* CONFIG_PPC_PASEMI */


#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
/*
 * SCOM access functions for 970 (FX only for now)
 *
 * unsigned long scom970_read(unsigned int address);
 * void scom970_write(unsigned int address, unsigned long value);
 *
 * The address passed in is the 24 bits register address. This code
 * is 970 specific and will not check the status bits, so you should
 * know what you are doing.
 */
_GLOBAL(scom970_read)
	/* interrupts off */
	mfmsr	r4
	ori	r0,r4,MSR_EE
	xori	r0,r0,MSR_EE
	mtmsrd	r0,1

	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
	 * (including parity). On current CPUs they must be 0'd,
	 * and finally or in RW bit
	 */
	rlwinm	r3,r3,8,0,15
	ori	r3,r3,0x8000

	/* do the actual scom read */
	sync
	mtspr	SPRN_SCOMC,r3
	isync
	mfspr	r3,SPRN_SCOMD
	isync
	mfspr	r0,SPRN_SCOMC
	isync

	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
	 * that's the best we can do). Not implemented yet as we don't use
	 * the scom on any of the bogus CPUs yet, but may have to be done
	 * ultimately
	 */

	/* restore interrupts */
	mtmsrd	r4,1
	blr


_GLOBAL(scom970_write)
	/* interrupts off */
	mfmsr	r5
	ori	r0,r5,MSR_EE
	xori	r0,r0,MSR_EE
	mtmsrd	r0,1

	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
	 * (including parity). On current CPUs they must be 0'd.
	 */

	rlwinm	r3,r3,8,0,15

	sync
	mtspr	SPRN_SCOMD,r4      /* write data */
	isync
	mtspr	SPRN_SCOMC,r3      /* write command */
	isync
	mfspr	3,SPRN_SCOMC
	isync

	/* restore interrupts */
	mtmsrd	r5,1
	blr
#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */

/* kexec_wait(phys_cpu)
 *
 * wait for the flag to change, indicating this kernel is going away but
 * the slave code for the next one is at addresses 0 to 100.
 *
 * This is used by all slaves, even those that did not find a matching
 * paca in the secondary startup code.
 *
 * Physical (hardware) cpu id should be in r3.
 */
_GLOBAL(kexec_wait)
	bl	1f
1:	mflr	r5
	addi	r5,r5,kexec_flag-1b

99:	HMT_LOW
#ifdef CONFIG_KEXEC_CORE	/* use no memory without kexec */
	lwz	r4,0(r5)
	cmpwi	0,r4,0
	beq	99b
#ifdef CONFIG_PPC_BOOK3S_64
	li	r10,0x60
	mfmsr	r11
	clrrdi	r11,r11,1	/* Clear MSR_LE */
	mtsrr0	r10
	mtsrr1	r11
	rfid
#else
	/* Create TLB entry in book3e_secondary_core_init */
	li	r4,0
	ba	0x60
#endif
#endif

/* this can be in text because we won't change it until we are
 * running in real anyways
 */
kexec_flag:
	.long	0


#ifdef CONFIG_KEXEC_CORE
#ifdef CONFIG_PPC_BOOK3E
/*
 * BOOK3E has no real MMU mode, so we have to setup the initial TLB
 * for a core to identity map v:0 to p:0.  This current implementation
 * assumes that 1G is enough for kexec.
 */
kexec_create_tlb:
	/*
	 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
	 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
	 */
	PPC_TLBILX_ALL(0,R0)
	sync
	isync

	mfspr	r10,SPRN_TLB1CFG
	andi.	r10,r10,TLBnCFG_N_ENTRY	/* Extract # entries */
	subi	r10,r10,1	/* Last entry: no conflict with kernel text */
	lis	r9,MAS0_TLBSEL(1)@h
	rlwimi	r9,r10,16,4,15		/* Setup MAS0 = TLBSEL | ESEL(r9) */

/* Set up a temp identity mapping v:0 to p:0 and return to it. */
#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
#define M_IF_NEEDED	MAS2_M
#else
#define M_IF_NEEDED	0
#endif
	mtspr	SPRN_MAS0,r9

	lis	r9,(MAS1_VALID|MAS1_IPROT)@h
	ori	r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
	mtspr	SPRN_MAS1,r9

	LOAD_REG_IMMEDIATE(r9, 0x0 | M_IF_NEEDED)
	mtspr	SPRN_MAS2,r9

	LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
	mtspr	SPRN_MAS3,r9
	li	r9,0
	mtspr	SPRN_MAS7,r9

	tlbwe
	isync
	blr
#endif

/* kexec_smp_wait(void)
 *
 * call with interrupts off
 * note: this is a terminal routine, it does not save lr
 *
 * get phys id from paca
 * switch to real mode
 * mark the paca as no longer used
 * join other cpus in kexec_wait(phys_id)
 */
_GLOBAL(kexec_smp_wait)
	lhz	r3,PACAHWCPUID(r13)
	bl	real_mode

	li	r4,KEXEC_STATE_REAL_MODE
	stb	r4,PACAKEXECSTATE(r13)
	SYNC

	b	kexec_wait

/*
 * switch to real mode (turn mmu off)
 * we use the early kernel trick that the hardware ignores bits
 * 0 and 1 (big endian) of the effective address in real mode
 *
 * don't overwrite r3 here, it is live for kexec_wait above.
 */
real_mode:	/* assume normal blr return */
#ifdef CONFIG_PPC_BOOK3E
	/* Create an identity mapping. */
	b	kexec_create_tlb
#else
1:	li	r9,MSR_RI
	li	r10,MSR_DR|MSR_IR
	mflr	r11		/* return address to SRR0 */
	mfmsr	r12
	andc	r9,r12,r9
	andc	r10,r12,r10

	mtmsrd	r9,1
	mtspr	SPRN_SRR1,r10
	mtspr	SPRN_SRR0,r11
	rfid
#endif

/*
 * kexec_sequence(newstack, start, image, control, clear_all(),
	          copy_with_mmu_off)
 *
 * does the grungy work with stack switching and real mode switches
 * also does simple calls to other code
 */

_GLOBAL(kexec_sequence)
	mflr	r0
	std	r0,16(r1)

	/* switch stacks to newstack -- &kexec_stack.stack */
	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
	mr	r1,r3

	li	r0,0
	std	r0,16(r1)

BEGIN_FTR_SECTION
	/*
	 * This is the best time to turn AMR/IAMR off.
	 * key 0 is used in radix for supervisor<->user
	 * protection, but on hash key 0 is reserved
	 * ideally we want to enter with a clean state.
	 * NOTE, we rely on r0 being 0 from above.
	 */
	mtspr	SPRN_IAMR,r0
BEGIN_FTR_SECTION_NESTED(42)
	mtspr	SPRN_AMOR,r0
END_FTR_SECTION_NESTED_IFSET(CPU_FTR_HVMODE, 42)
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)

	/* save regs for local vars on new stack.
	 * yes, we won't go back, but ...
	 */
	std	r31,-8(r1)
	std	r30,-16(r1)
	std	r29,-24(r1)
	std	r28,-32(r1)
	std	r27,-40(r1)
	std	r26,-48(r1)
	std	r25,-56(r1)

	stdu	r1,-STACK_FRAME_OVERHEAD-64(r1)

	/* save args into preserved regs */
	mr	r31,r3			/* newstack (both) */
	mr	r30,r4			/* start (real) */
	mr	r29,r5			/* image (virt) */
	mr	r28,r6			/* control, unused */
	mr	r27,r7			/* clear_all() fn desc */
	mr	r26,r8			/* copy_with_mmu_off */
	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */

	/* disable interrupts, we are overwriting kernel data next */
#ifdef CONFIG_PPC_BOOK3E
	wrteei	0
#else
	mfmsr	r3
	rlwinm	r3,r3,0,17,15
	mtmsrd	r3,1
#endif

	/* We need to turn the MMU off unless we are in hash mode
	 * under a hypervisor
	 */
	cmpdi	r26,0
	beq	1f
	bl	real_mode
1:
	/* copy dest pages, flush whole dest image */
	mr	r3,r29
	bl	kexec_copy_flush	/* (image) */

	/* turn off mmu now if not done earlier */
	cmpdi	r26,0
	bne	1f
	bl	real_mode

	/* copy  0x100 bytes starting at start to 0 */
1:	li	r3,0
	mr	r4,r30		/* start, aka phys mem offset */
	li	r5,0x100
	li	r6,0
	bl	copy_and_flush	/* (dest, src, copy limit, start offset) */
1:	/* assume normal blr return */

	/* release other cpus to the new kernel secondary start at 0x60 */
	mflr	r5
	li	r6,1
	stw	r6,kexec_flag-1b(5)

	cmpdi	r27,0
	beq	1f

	/* clear out hardware hash page table and tlb */
#ifdef PPC64_ELF_ABI_v1
	ld	r12,0(r27)		/* deref function descriptor */
#else
	mr	r12,r27
#endif
	mtctr	r12
	bctrl				/* mmu_hash_ops.hpte_clear_all(void); */

/*
 *   kexec image calling is:
 *      the first 0x100 bytes of the entry point are copied to 0
 *
 *      all slaves branch to slave = 0x60 (absolute)
 *              slave(phys_cpu_id);
 *
 *      master goes to start = entry point
 *              start(phys_cpu_id, start, 0);
 *
 *
 *   a wrapper is needed to call existing kernels, here is an approximate
 *   description of one method:
 *
 * v2: (2.6.10)
 *   start will be near the boot_block (maybe 0x100 bytes before it?)
 *   it will have a 0x60, which will b to boot_block, where it will wait
 *   and 0 will store phys into struct boot-block and load r3 from there,
 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
 *
 * v1: (2.6.9)
 *    boot block will have all cpus scanning device tree to see if they
 *    are the boot cpu ?????
 *    other device tree differences (prop sizes, va vs pa, etc)...
 */
1:	mr	r3,r25	# my phys cpu
	mr	r4,r30	# start, aka phys mem offset
	mtlr	4
	li	r5,0
	blr	/* image->start(physid, image->start, 0); */
#endif /* CONFIG_KEXEC_CORE */
Commit	Line	Data
9994a338	1	/*
9994a338 PM	2	* This file contains miscellaneous low-level functions.
	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	4	*
	5	* Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
	6	* and Paul Mackerras.
	7	* Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
127efeb2 SR	8	* PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
127efeb2 SR	9	*
9994a338 PM	10	* This program is free software; you can redistribute it and/or
	11	* modify it under the terms of the GNU General Public License
	12	* as published by the Free Software Foundation; either version
	13	* 2 of the License, or (at your option) any later version.
	14	*
	15	*/
	16
9994a338 PM	17	#include <linux/sys.h>
	18	#include <asm/unistd.h>
	19	#include <asm/errno.h>
	20	#include <asm/processor.h>
	21	#include <asm/page.h>
	22	#include <asm/cache.h>
	23	#include <asm/ppc_asm.h>
	24	#include <asm/asm-offsets.h>
	25	#include <asm/cputable.h>
6cb7bfeb	26	#include <asm/thread_info.h>
1fc711f7	27	#include <asm/kexec.h>
46f52210	28	#include <asm/ptrace.h>
cf904e30	29	#include <asm/mmu.h>
9445aa1a	30	#include <asm/export.h>
9994a338 PM	31
	32	.text
	33
9994a338 PM	34	_GLOBAL(call_do_softirq)
	35	mflr r0
	36	std r0,16(r1)
4ae2dcb6	37	stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
9994a338	38	mr r1,r3
b1576fec	39	bl __do_softirq
9994a338 PM	40	ld r1,0(r1)
	41	ld r0,16(r1)
	42	mtlr r0
	43	blr
	44
0366a1c7	45	_GLOBAL(call_do_irq)
9994a338 PM	46	mflr r0
9994a338 PM	47	std r0,16(r1)
0366a1c7 BH	48	stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
0366a1c7 BH	49	mr r1,r4
b1576fec	50	bl __do_irq
9994a338 PM	51	ld r1,0(r1)
	52	ld r0,16(r1)
	53	mtlr r0
	54	blr
9994a338	55
9994a338 PM	56	.section ".toc","aw"
	57	PPC64_CACHES:
	58	.tc ppc64_caches[TC],ppc64_caches
	59	.section ".text"
	60
	61	/*
	62	* Write any modified data cache blocks out to memory
	63	* and invalidate the corresponding instruction cache blocks.
	64	*
	65	* flush_icache_range(unsigned long start, unsigned long stop)
	66	*
	67	* flush all bytes from start through stop-1 inclusive
	68	*/
	69
8f5f525d	70	_GLOBAL_TOC(flush_icache_range)
abb29c3b	71	BEGIN_FTR_SECTION
0ce63670	72	PURGE_PREFETCHED_INS
abb29c3b KH	73	blr
abb29c3b KH	74	END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
9994a338 PM	75	/*
	76	* Flush the data cache to memory
	77	*
	78	* Different systems have different cache line sizes
	79	* and in some cases i-cache and d-cache line sizes differ from
	80	* each other.
	81	*/
	82	ld r10,PPC64_CACHES@toc(r2)
bd067f83	83	lwz r7,DCACHEL1BLOCKSIZE(r10)/* Get cache block size */
9994a338 PM	84	addi r5,r7,-1
	85	andc r6,r3,r5 /* round low to line bdy */
	86	subf r8,r6,r4 /* compute length */
	87	add r8,r8,r5 /* ensure we get enough */
bd067f83	88	lwz r9,DCACHEL1LOGBLOCKSIZE(r10) /* Get log-2 of cache block size */
9994a338 PM	89	srw. r8,r8,r9 /* compute line count */
	90	beqlr /* nothing to do? */
	91	mtctr r8
	92	1: dcbst 0,r6
	93	add r6,r6,r7
	94	bdnz 1b
	95	sync
	96
	97	/* Now invalidate the instruction cache */
	98
bd067f83	99	lwz r7,ICACHEL1BLOCKSIZE(r10) /* Get Icache block size */
9994a338 PM	100	addi r5,r7,-1
	101	andc r6,r3,r5 /* round low to line bdy */
	102	subf r8,r6,r4 /* compute length */
	103	add r8,r8,r5
bd067f83	104	lwz r9,ICACHEL1LOGBLOCKSIZE(r10) /* Get log-2 of Icache block size */
9994a338 PM	105	srw. r8,r8,r9 /* compute line count */
	106	beqlr /* nothing to do? */
	107	mtctr r8
	108	2: icbi 0,r6
	109	add r6,r6,r7
	110	bdnz 2b
	111	isync
	112	blr
6f698df1	113	_ASM_NOKPROBE_SYMBOL(flush_icache_range)
9445aa1a	114	EXPORT_SYMBOL(flush_icache_range)
6f698df1	115
9994a338 PM	116	/*
	117	* Like above, but only do the D-cache.
	118	*
	119	* flush_dcache_range(unsigned long start, unsigned long stop)
	120	*
	121	* flush all bytes from start to stop-1 inclusive
	122	*/
8f5f525d	123	_GLOBAL_TOC(flush_dcache_range)
9994a338 PM	124
	125	/*
	126	* Flush the data cache to memory
	127	*
	128	* Different systems have different cache line sizes
	129	*/
	130	ld r10,PPC64_CACHES@toc(r2)
bd067f83	131	lwz r7,DCACHEL1BLOCKSIZE(r10) /* Get dcache block size */
9994a338 PM	132	addi r5,r7,-1
	133	andc r6,r3,r5 /* round low to line bdy */
	134	subf r8,r6,r4 /* compute length */
	135	add r8,r8,r5 /* ensure we get enough */
bd067f83	136	lwz r9,DCACHEL1LOGBLOCKSIZE(r10) /* Get log-2 of dcache block size */
9994a338 PM	137	srw. r8,r8,r9 /* compute line count */
	138	beqlr /* nothing to do? */
	139	mtctr r8
	140	0: dcbst 0,r6
	141	add r6,r6,r7
	142	bdnz 0b
	143	sync
	144	blr
9445aa1a	145	EXPORT_SYMBOL(flush_dcache_range)
9994a338	146
9994a338 PM	147	_GLOBAL(flush_inval_dcache_range)
9994a338 PM	148	ld r10,PPC64_CACHES@toc(r2)
bd067f83	149	lwz r7,DCACHEL1BLOCKSIZE(r10) /* Get dcache block size */
9994a338 PM	150	addi r5,r7,-1
	151	andc r6,r3,r5 /* round low to line bdy */
	152	subf r8,r6,r4 /* compute length */
	153	add r8,r8,r5 /* ensure we get enough */
bd067f83	154	lwz r9,DCACHEL1LOGBLOCKSIZE(r10)/* Get log-2 of dcache block size */
9994a338 PM	155	srw. r8,r8,r9 /* compute line count */
	156	beqlr /* nothing to do? */
	157	sync
	158	isync
	159	mtctr r8
	160	0: dcbf 0,r6
	161	add r6,r6,r7
	162	bdnz 0b
	163	sync
	164	isync
	165	blr
	166
	167
	168	/*
	169	* Flush a particular page from the data cache to RAM.
	170	* Note: this is necessary because the instruction cache does not
	171	* snoop from the data cache.
	172	*
	173	* void __flush_dcache_icache(void *page)
	174	*/
	175	_GLOBAL(__flush_dcache_icache)
	176	/*
	177	* Flush the data cache to memory
	178	*
	179	* Different systems have different cache line sizes
	180	*/
	181
0ce63670 KH	182	BEGIN_FTR_SECTION
	183	PURGE_PREFETCHED_INS
	184	blr
	185	END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
	186
9994a338 PM	187	/* Flush the dcache */
	188	ld r7,PPC64_CACHES@toc(r2)
	189	clrrdi r3,r3,PAGE_SHIFT /* Page align */
bd067f83 BH	190	lwz r4,DCACHEL1BLOCKSPERPAGE(r7) /* Get # dcache blocks per page */
bd067f83 BH	191	lwz r5,DCACHEL1BLOCKSIZE(r7) /* Get dcache block size */
9994a338 PM	192	mr r6,r3
	193	mtctr r4
	194	0: dcbst 0,r6
	195	add r6,r6,r5
	196	bdnz 0b
	197	sync
	198
	199	/* Now invalidate the icache */
	200
bd067f83 BH	201	lwz r4,ICACHEL1BLOCKSPERPAGE(r7) /* Get # icache blocks per page */
bd067f83 BH	202	lwz r5,ICACHEL1BLOCKSIZE(r7) /* Get icache block size */
9994a338 PM	203	mtctr r4
	204	1: icbi 0,r3
	205	add r3,r3,r5
	206	bdnz 1b
	207	isync
	208	blr
9994a338	209
ca9d7aea	210	_GLOBAL(__bswapdi2)
9445aa1a	211	EXPORT_SYMBOL(__bswapdi2)
ca9d7aea DW	212	srdi r8,r3,32
	213	rlwinm r7,r3,8,0xffffffff
	214	rlwimi r7,r3,24,0,7
	215	rlwinm r9,r8,8,0xffffffff
	216	rlwimi r7,r3,24,16,23
	217	rlwimi r9,r8,24,0,7
	218	rlwimi r9,r8,24,16,23
	219	sldi r7,r7,32
	220	or r3,r7,r9
	221	blr
3f639ee8	222
7191b615	223
2d6f0c3a	224	#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
7191b615 BH	225	_GLOBAL(rmci_on)
	226	sync
	227	isync
	228	li r3,0x100
	229	rldicl r3,r3,32,0
	230	mfspr r5,SPRN_HID4
	231	or r5,r5,r3
	232	sync
	233	mtspr SPRN_HID4,r5
	234	isync
	235	slbia
	236	isync
	237	sync
	238	blr
	239
	240	_GLOBAL(rmci_off)
	241	sync
	242	isync
	243	li r3,0x100
	244	rldicl r3,r3,32,0
	245	mfspr r5,SPRN_HID4
	246	andc r5,r5,r3
	247	sync
	248	mtspr SPRN_HID4,r5
	249	isync
	250	slbia
	251	isync
	252	sync
	253	blr
2d6f0c3a ME	254	#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
	255
	256	#if defined(CONFIG_PPC_PMAC) \|\| defined(CONFIG_PPC_MAPLE)
7191b615	257
9994a338 PM	258	/*
	259	* Do an IO access in real mode
	260	*/
	261	_GLOBAL(real_readb)
	262	mfmsr r7
	263	ori r0,r7,MSR_DR
	264	xori r0,r0,MSR_DR
	265	sync
	266	mtmsrd r0
	267	sync
	268	isync
	269	mfspr r6,SPRN_HID4
	270	rldicl r5,r6,32,0
	271	ori r5,r5,0x100
	272	rldicl r5,r5,32,0
	273	sync
	274	mtspr SPRN_HID4,r5
	275	isync
	276	slbia
	277	isync
	278	lbz r3,0(r3)
	279	sync
	280	mtspr SPRN_HID4,r6
	281	isync
	282	slbia
	283	isync
	284	mtmsrd r7
	285	sync
	286	isync
	287	blr
	288
	289	/*
	290	* Do an IO access in real mode
	291	*/
	292	_GLOBAL(real_writeb)
	293	mfmsr r7
	294	ori r0,r7,MSR_DR
	295	xori r0,r0,MSR_DR
	296	sync
	297	mtmsrd r0
	298	sync
	299	isync
	300	mfspr r6,SPRN_HID4
	301	rldicl r5,r6,32,0
	302	ori r5,r5,0x100
	303	rldicl r5,r5,32,0
	304	sync
	305	mtspr SPRN_HID4,r5
	306	isync
	307	slbia
	308	isync
	309	stb r3,0(r4)
	310	sync
	311	mtspr SPRN_HID4,r6
	312	isync
	313	slbia
	314	isync
	315	mtmsrd r7
	316	sync
	317	isync
	318	blr
	319	#endif /* defined(CONFIG_PPC_PMAC) \|\| defined(CONFIG_PPC_MAPLE) */
	320
39c870d5 OJ	321	#ifdef CONFIG_PPC_PASEMI
39c870d5 OJ	322
39c870d5 OJ	323	_GLOBAL(real_205_readb)
	324	mfmsr r7
	325	ori r0,r7,MSR_DR
	326	xori r0,r0,MSR_DR
	327	sync
	328	mtmsrd r0
	329	sync
	330	isync
e55174e9	331	LBZCIX(R3,R0,R3)
39c870d5 OJ	332	isync
	333	mtmsrd r7
	334	sync
	335	isync
	336	blr
	337
	338	_GLOBAL(real_205_writeb)
	339	mfmsr r7
	340	ori r0,r7,MSR_DR
	341	xori r0,r0,MSR_DR
	342	sync
	343	mtmsrd r0
	344	sync
	345	isync
e55174e9	346	STBCIX(R3,R0,R4)
39c870d5 OJ	347	isync
	348	mtmsrd r7
	349	sync
	350	isync
	351	blr
	352
	353	#endif /* CONFIG_PPC_PASEMI */
	354
	355
e48f7eb2	356	#if defined(CONFIG_CPU_FREQ_PMAC64) \|\| defined(CONFIG_CPU_FREQ_MAPLE)
4350147a BH	357	/*
	358	* SCOM access functions for 970 (FX only for now)
	359	*
	360	* unsigned long scom970_read(unsigned int address);
	361	* void scom970_write(unsigned int address, unsigned long value);
	362	*
	363	* The address passed in is the 24 bits register address. This code
	364	* is 970 specific and will not check the status bits, so you should
	365	* know what you are doing.
	366	*/
	367	_GLOBAL(scom970_read)
	368	/* interrupts off */
	369	mfmsr r4
	370	ori r0,r4,MSR_EE
	371	xori r0,r0,MSR_EE
	372	mtmsrd r0,1
	373
	374	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
	375	* (including parity). On current CPUs they must be 0'd,
	376	* and finally or in RW bit
	377	*/
	378	rlwinm r3,r3,8,0,15
	379	ori r3,r3,0x8000
	380
	381	/* do the actual scom read */
	382	sync
	383	mtspr SPRN_SCOMC,r3
	384	isync
	385	mfspr r3,SPRN_SCOMD
	386	isync
	387	mfspr r0,SPRN_SCOMC
	388	isync
	389
	390	/* XXX: fixup result on some buggy 970's (ouch ! we lost a bit, bah
	391	* that's the best we can do). Not implemented yet as we don't use
	392	* the scom on any of the bogus CPUs yet, but may have to be done
	393	* ultimately
	394	*/
	395
	396	/* restore interrupts */
	397	mtmsrd r4,1
	398	blr
	399
	400
	401	_GLOBAL(scom970_write)
	402	/* interrupts off */
	403	mfmsr r5
	404	ori r0,r5,MSR_EE
	405	xori r0,r0,MSR_EE
	406	mtmsrd r0,1
	407
	408	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
	409	* (including parity). On current CPUs they must be 0'd.
	410	*/
	411
	412	rlwinm r3,r3,8,0,15
	413
	414	sync
	415	mtspr SPRN_SCOMD,r4 /* write data */
	416	isync
	417	mtspr SPRN_SCOMC,r3 /* write command */
	418	isync
	419	mfspr 3,SPRN_SCOMC
	420	isync
421
422	/* restore interrupts */
423	mtmsrd r5,1
424	blr
e48f7eb2	425	#endif /* CONFIG_CPU_FREQ_PMAC64 \|\| CONFIG_CPU_FREQ_MAPLE */
4350147a	426
9994a338 PM	427	/* kexec_wait(phys_cpu)
	428	*
	429	* wait for the flag to change, indicating this kernel is going away but
	430	* the slave code for the next one is at addresses 0 to 100.
	431	*
3d2cea73 MM	432	* This is used by all slaves, even those that did not find a matching
3d2cea73 MM	433	* paca in the secondary startup code.
9994a338 PM	434	*
	435	* Physical (hardware) cpu id should be in r3.
	436	*/
	437	_GLOBAL(kexec_wait)
	438	bl 1f
	439	1: mflr r5
	440	addi r5,r5,kexec_flag-1b
	441
	442	99: HMT_LOW
da665885	443	#ifdef CONFIG_KEXEC_CORE /* use no memory without kexec */
9994a338 PM	444	lwz r4,0(r5)
9994a338 PM	445	cmpwi 0,r4,0
ffebf5f3 SMJ	446	beq 99b
	447	#ifdef CONFIG_PPC_BOOK3S_64
	448	li r10,0x60
	449	mfmsr r11
	450	clrrdi r11,r11,1 /* Clear MSR_LE */
	451	mtsrr0 r10
	452	mtsrr1 r11
	453	rfid
	454	#else
ae73e4cc SW	455	/* Create TLB entry in book3e_secondary_core_init */
ae73e4cc SW	456	li r4,0
ffebf5f3 SMJ	457	ba 0x60
ffebf5f3 SMJ	458	#endif
9994a338	459	#endif
9994a338 PM	460
	461	/* this can be in text because we won't change it until we are
	462	* running in real anyways
	463	*/
	464	kexec_flag:
	465	.long 0
	466
	467
da665885	468	#ifdef CONFIG_KEXEC_CORE
cf904e30 TC	469	#ifdef CONFIG_PPC_BOOK3E
	470	/*
	471	* BOOK3E has no real MMU mode, so we have to setup the initial TLB
	472	* for a core to identity map v:0 to p:0. This current implementation
	473	* assumes that 1G is enough for kexec.
	474	*/
	475	kexec_create_tlb:
	476	/*
	477	* Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
	478	* IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
	479	*/
	480	PPC_TLBILX_ALL(0,R0)
	481	sync
	482	isync
	483
	484	mfspr r10,SPRN_TLB1CFG
	485	andi. r10,r10,TLBnCFG_N_ENTRY /* Extract # entries */
	486	subi r10,r10,1 /* Last entry: no conflict with kernel text */
	487	lis r9,MAS0_TLBSEL(1)@h
	488	rlwimi r9,r10,16,4,15 /* Setup MAS0 = TLBSEL \| ESEL(r9) */
	489
	490	/* Set up a temp identity mapping v:0 to p:0 and return to it. */
	491	#if defined(CONFIG_SMP) \|\| defined(CONFIG_PPC_E500MC)
	492	#define M_IF_NEEDED MAS2_M
	493	#else
	494	#define M_IF_NEEDED 0
	495	#endif
	496	mtspr SPRN_MAS0,r9
	497
	498	lis r9,(MAS1_VALID\|MAS1_IPROT)@h
	499	ori r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
	500	mtspr SPRN_MAS1,r9
	501
	502	LOAD_REG_IMMEDIATE(r9, 0x0 \| M_IF_NEEDED)
	503	mtspr SPRN_MAS2,r9
	504
	505	LOAD_REG_IMMEDIATE(r9, 0x0 \| MAS3_SR \| MAS3_SW \| MAS3_SX)
	506	mtspr SPRN_MAS3,r9
	507	li r9,0
	508	mtspr SPRN_MAS7,r9
	509
	510	tlbwe
	511	isync
	512	blr
	513	#endif
9994a338 PM	514
	515	/* kexec_smp_wait(void)
	516	*
	517	* call with interrupts off
	518	* note: this is a terminal routine, it does not save lr
	519	*
	520	* get phys id from paca
9994a338	521	* switch to real mode
3d2cea73	522	* mark the paca as no longer used
9994a338 PM	523	* join other cpus in kexec_wait(phys_id)
	524	*/
	525	_GLOBAL(kexec_smp_wait)
	526	lhz r3,PACAHWCPUID(r13)
9994a338	527	bl real_mode
3d2cea73 MM	528
	529	li r4,KEXEC_STATE_REAL_MODE
	530	stb r4,PACAKEXECSTATE(r13)
	531	SYNC
	532
b1576fec	533	b kexec_wait
9994a338 PM	534
	535	/*
	536	* switch to real mode (turn mmu off)
	537	* we use the early kernel trick that the hardware ignores bits
	538	* 0 and 1 (big endian) of the effective address in real mode
	539	*
	540	* don't overwrite r3 here, it is live for kexec_wait above.
	541	*/
	542	real_mode: /* assume normal blr return */
cf904e30 TC	543	#ifdef CONFIG_PPC_BOOK3E
	544	/* Create an identity mapping. */
	545	b kexec_create_tlb
	546	#else
9994a338 PM	547	1: li r9,MSR_RI
	548	li r10,MSR_DR\|MSR_IR
	549	mflr r11 /* return address to SRR0 */
	550	mfmsr r12
	551	andc r9,r12,r9
	552	andc r10,r12,r10
	553
	554	mtmsrd r9,1
	555	mtspr SPRN_SRR1,r10
	556	mtspr SPRN_SRR0,r11
	557	rfid
cf904e30	558	#endif
9994a338 PM	559
9994a338 PM	560	/*
b970b41e BH	561	* kexec_sequence(newstack, start, image, control, clear_all(),
b970b41e BH	562	copy_with_mmu_off)
9994a338 PM	563	*
	564	* does the grungy work with stack switching and real mode switches
	565	* also does simple calls to other code
	566	*/
	567
	568	_GLOBAL(kexec_sequence)
	569	mflr r0
	570	std r0,16(r1)
	571
	572	/* switch stacks to newstack -- &kexec_stack.stack */
4ae2dcb6	573	stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
9994a338 PM	574	mr r1,r3
	575
	576	li r0,0
	577	std r0,16(r1)
	578
1e2a516e BS	579	BEGIN_FTR_SECTION
	580	/*
	581	* This is the best time to turn AMR/IAMR off.
	582	* key 0 is used in radix for supervisor<->user
	583	* protection, but on hash key 0 is reserved
	584	* ideally we want to enter with a clean state.
	585	* NOTE, we rely on r0 being 0 from above.
	586	*/
	587	mtspr SPRN_IAMR,r0
2621e945	588	BEGIN_FTR_SECTION_NESTED(42)
1e2a516e	589	mtspr SPRN_AMOR,r0
2621e945	590	END_FTR_SECTION_NESTED_IFSET(CPU_FTR_HVMODE, 42)
1e2a516e BS	591	END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
1e2a516e BS	592
9994a338 PM	593	/* save regs for local vars on new stack.
	594	* yes, we won't go back, but ...
	595	*/
	596	std r31,-8(r1)
	597	std r30,-16(r1)
	598	std r29,-24(r1)
	599	std r28,-32(r1)
	600	std r27,-40(r1)
	601	std r26,-48(r1)
	602	std r25,-56(r1)
	603
4ae2dcb6	604	stdu r1,-STACK_FRAME_OVERHEAD-64(r1)
9994a338 PM	605
	606	/* save args into preserved regs */
	607	mr r31,r3 /* newstack (both) */
	608	mr r30,r4 /* start (real) */
	609	mr r29,r5 /* image (virt) */
	610	mr r28,r6 /* control, unused */
	611	mr r27,r7 /* clear_all() fn desc */
b970b41e	612	mr r26,r8 /* copy_with_mmu_off */
9994a338 PM	613	lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */
	614
	615	/* disable interrupts, we are overwriting kernel data next */
96eea642 TC	616	#ifdef CONFIG_PPC_BOOK3E
	617	wrteei 0
	618	#else
9994a338 PM	619	mfmsr r3
	620	rlwinm r3,r3,0,17,15
	621	mtmsrd r3,1
96eea642	622	#endif
9994a338	623
b970b41e BH	624	/* We need to turn the MMU off unless we are in hash mode
	625	* under a hypervisor
	626	*/
	627	cmpdi r26,0
	628	beq 1f
	629	bl real_mode
	630	1:
9994a338 PM	631	/* copy dest pages, flush whole dest image */
9994a338 PM	632	mr r3,r29
b1576fec	633	bl kexec_copy_flush /* (image) */
9994a338	634
b970b41e BH	635	/* turn off mmu now if not done earlier */
	636	cmpdi r26,0
	637	bne 1f
9994a338 PM	638	bl real_mode
9994a338 PM	639
ee46a90b	640	/* copy 0x100 bytes starting at start to 0 */
b970b41e	641	1: li r3,0
ee46a90b MM	642	mr r4,r30 /* start, aka phys mem offset */
	643	li r5,0x100
	644	li r6,0
b1576fec	645	bl copy_and_flush /* (dest, src, copy limit, start offset) */
ee46a90b MM	646	1: /* assume normal blr return */
	647
	648	/* release other cpus to the new kernel secondary start at 0x60 */
	649	mflr r5
	650	li r6,1
	651	stw r6,kexec_flag-1b(5)
	652
fc48bad5 BH	653	cmpdi r27,0
	654	beq 1f
	655
9994a338	656	/* clear out hardware hash page table and tlb */
f55d9665	657	#ifdef PPC64_ELF_ABI_v1
cc7efbf9 AB	658	ld r12,0(r27) /* deref function descriptor */
	659	#else
	660	mr r12,r27
	661	#endif
	662	mtctr r12
7025776e	663	bctrl /* mmu_hash_ops.hpte_clear_all(void); */
9994a338 PM	664
	665	/*
	666	* kexec image calling is:
	667	* the first 0x100 bytes of the entry point are copied to 0
	668	*
	669	* all slaves branch to slave = 0x60 (absolute)
	670	* slave(phys_cpu_id);
	671	*
	672	* master goes to start = entry point
	673	* start(phys_cpu_id, start, 0);
	674	*
	675	*
	676	* a wrapper is needed to call existing kernels, here is an approximate
	677	* description of one method:
	678	*
	679	* v2: (2.6.10)
	680	* start will be near the boot_block (maybe 0x100 bytes before it?)
	681	* it will have a 0x60, which will b to boot_block, where it will wait
	682	* and 0 will store phys into struct boot-block and load r3 from there,
	683	* copy kernel 0-0x100 and tell slaves to back down to 0x60 again
	684	*
	685	* v1: (2.6.9)
	686	* boot block will have all cpus scanning device tree to see if they
	687	* are the boot cpu ?????
	688	* other device tree differences (prop sizes, va vs pa, etc)...
	689	*/
fc48bad5	690	1: mr r3,r25 # my phys cpu
9994a338 PM	691	mr r4,r30 # start, aka phys mem offset
	692	mtlr 4
	693	li r5,0
1767c8f3	694	blr /* image->start(physid, image->start, 0); */
da665885	695	#endif /* CONFIG_KEXEC_CORE */