[linux-2.6-block.git] / arch / mips / kernel / genex.S

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 * Copyright (C) 2002, 2007  Maciej W. Rozycki
 * Copyright (C) 2001, 2012 MIPS Technologies, Inc.  All rights reserved.
 */
#include <linux/init.h>

#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/cacheops.h>
#include <asm/irqflags.h>
#include <asm/regdef.h>
#include <asm/fpregdef.h>
#include <asm/mipsregs.h>
#include <asm/stackframe.h>
#include <asm/war.h>
#include <asm/thread_info.h>

	__INIT

/*
 * General exception vector for all other CPUs.
 *
 * Be careful when changing this, it has to be at most 128 bytes
 * to fit into space reserved for the exception handler.
 */
NESTED(except_vec3_generic, 0, sp)
	.set	push
	.set	noat
#if R5432_CP0_INTERRUPT_WAR
	mfc0	k0, CP0_INDEX
#endif
	mfc0	k1, CP0_CAUSE
	andi	k1, k1, 0x7c
#ifdef CONFIG_64BIT
	dsll	k1, k1, 1
#endif
	PTR_L	k0, exception_handlers(k1)
	jr	k0
	.set	pop
	END(except_vec3_generic)

/*
 * General exception handler for CPUs with virtual coherency exception.
 *
 * Be careful when changing this, it has to be at most 256 (as a special
 * exception) bytes to fit into space reserved for the exception handler.
 */
NESTED(except_vec3_r4000, 0, sp)
	.set	push
	.set	arch=r4000
	.set	noat
	mfc0	k1, CP0_CAUSE
	li	k0, 31<<2
	andi	k1, k1, 0x7c
	.set	push
	.set	noreorder
	.set	nomacro
	beq	k1, k0, handle_vced
	 li	k0, 14<<2
	beq	k1, k0, handle_vcei
#ifdef CONFIG_64BIT
	 dsll	k1, k1, 1
#endif
	.set	pop
	PTR_L	k0, exception_handlers(k1)
	jr	k0

	/*
	 * Big shit, we now may have two dirty primary cache lines for the same
	 * physical address.  We can safely invalidate the line pointed to by
	 * c0_badvaddr because after return from this exception handler the
	 * load / store will be re-executed.
	 */
handle_vced:
	MFC0	k0, CP0_BADVADDR
	li	k1, -4					# Is this ...
	and	k0, k1					# ... really needed?
	mtc0	zero, CP0_TAGLO
	cache	Index_Store_Tag_D, (k0)
	cache	Hit_Writeback_Inv_SD, (k0)
#ifdef CONFIG_PROC_FS
	PTR_LA	k0, vced_count
	lw	k1, (k0)
	addiu	k1, 1
	sw	k1, (k0)
#endif
	eret

handle_vcei:
	MFC0	k0, CP0_BADVADDR
	cache	Hit_Writeback_Inv_SD, (k0)		# also cleans pi
#ifdef CONFIG_PROC_FS
	PTR_LA	k0, vcei_count
	lw	k1, (k0)
	addiu	k1, 1
	sw	k1, (k0)
#endif
	eret
	.set	pop
	END(except_vec3_r4000)

	__FINIT

	.align	5	/* 32 byte rollback region */
LEAF(__r4k_wait)
	.set	push
	.set	noreorder
	/* start of rollback region */
	LONG_L	t0, TI_FLAGS($28)
	nop
	andi	t0, _TIF_NEED_RESCHED
	bnez	t0, 1f
	 nop
	nop
	nop
#ifdef CONFIG_CPU_MICROMIPS
	nop
	nop
	nop
	nop
#endif
	.set	arch=r4000
	wait
	/* end of rollback region (the region size must be power of two) */
1:
	jr	ra
	nop
	.set	pop
	END(__r4k_wait)

	.macro	BUILD_ROLLBACK_PROLOGUE handler
	FEXPORT(rollback_\handler)
	.set	push
	.set	noat
	MFC0	k0, CP0_EPC
	PTR_LA	k1, __r4k_wait
	ori	k0, 0x1f	/* 32 byte rollback region */
	xori	k0, 0x1f
	bne	k0, k1, 9f
	MTC0	k0, CP0_EPC
9:
	.set pop
	.endm

	.align	5
BUILD_ROLLBACK_PROLOGUE handle_int
NESTED(handle_int, PT_SIZE, sp)
#ifdef CONFIG_TRACE_IRQFLAGS
	/*
	 * Check to see if the interrupted code has just disabled
	 * interrupts and ignore this interrupt for now if so.
	 *
	 * local_irq_disable() disables interrupts and then calls
	 * trace_hardirqs_off() to track the state. If an interrupt is taken
	 * after interrupts are disabled but before the state is updated
	 * it will appear to restore_all that it is incorrectly returning with
	 * interrupts disabled
	 */
	.set	push
	.set	noat
	mfc0	k0, CP0_STATUS
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
	and	k0, ST0_IEP
	bnez	k0, 1f

	mfc0	k0, CP0_EPC
	.set	noreorder
	j	k0
	rfe
#else
	and	k0, ST0_IE
	bnez	k0, 1f

	eret
#endif
1:
	.set pop
#endif
	SAVE_ALL
	CLI
	TRACE_IRQS_OFF

	LONG_L	s0, TI_REGS($28)
	LONG_S	sp, TI_REGS($28)
	PTR_LA	ra, ret_from_irq
	PTR_LA  v0, plat_irq_dispatch
	jr	v0
#ifdef CONFIG_CPU_MICROMIPS
	nop
#endif
	END(handle_int)

	__INIT

/*
 * Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
 * This is a dedicated interrupt exception vector which reduces the
 * interrupt processing overhead.  The jump instruction will be replaced
 * at the initialization time.
 *
 * Be careful when changing this, it has to be at most 128 bytes
 * to fit into space reserved for the exception handler.
 */
NESTED(except_vec4, 0, sp)
1:	j	1b			/* Dummy, will be replaced */
	END(except_vec4)

/*
 * EJTAG debug exception handler.
 * The EJTAG debug exception entry point is 0xbfc00480, which
 * normally is in the boot PROM, so the boot PROM must do an
 * unconditional jump to this vector.
 */
NESTED(except_vec_ejtag_debug, 0, sp)
	j	ejtag_debug_handler
#ifdef CONFIG_CPU_MICROMIPS
	 nop
#endif
	END(except_vec_ejtag_debug)

	__FINIT

/*
 * Vectored interrupt handler.
 * This prototype is copied to ebase + n*IntCtl.VS and patched
 * to invoke the handler
 */
BUILD_ROLLBACK_PROLOGUE except_vec_vi
NESTED(except_vec_vi, 0, sp)
	SAVE_SOME
	SAVE_AT
	.set	push
	.set	noreorder
	PTR_LA	v1, except_vec_vi_handler
FEXPORT(except_vec_vi_lui)
	lui	v0, 0		/* Patched */
	jr	v1
FEXPORT(except_vec_vi_ori)
	 ori	v0, 0		/* Patched */
	.set	pop
	END(except_vec_vi)
EXPORT(except_vec_vi_end)

/*
 * Common Vectored Interrupt code
 * Complete the register saves and invoke the handler which is passed in $v0
 */
NESTED(except_vec_vi_handler, 0, sp)
	SAVE_TEMP
	SAVE_STATIC
	CLI
#ifdef CONFIG_TRACE_IRQFLAGS
	move	s0, v0
	TRACE_IRQS_OFF
	move	v0, s0
#endif

	LONG_L	s0, TI_REGS($28)
	LONG_S	sp, TI_REGS($28)
	PTR_LA	ra, ret_from_irq
	jr	v0
	END(except_vec_vi_handler)

/*
 * EJTAG debug exception handler.
 */
NESTED(ejtag_debug_handler, PT_SIZE, sp)
	.set	push
	.set	noat
	MTC0	k0, CP0_DESAVE
	mfc0	k0, CP0_DEBUG

	sll	k0, k0, 30	# Check for SDBBP.
	bgez	k0, ejtag_return

	PTR_LA	k0, ejtag_debug_buffer
	LONG_S	k1, 0(k0)
	SAVE_ALL
	move	a0, sp
	jal	ejtag_exception_handler
	RESTORE_ALL
	PTR_LA	k0, ejtag_debug_buffer
	LONG_L	k1, 0(k0)

ejtag_return:
	MFC0	k0, CP0_DESAVE
	.set	mips32
	deret
	.set pop
	END(ejtag_debug_handler)

/*
 * This buffer is reserved for the use of the EJTAG debug
 * handler.
 */
	.data
EXPORT(ejtag_debug_buffer)
	.fill	LONGSIZE
	.previous

	__INIT

/*
 * NMI debug exception handler for MIPS reference boards.
 * The NMI debug exception entry point is 0xbfc00000, which
 * normally is in the boot PROM, so the boot PROM must do a
 * unconditional jump to this vector.
 */
NESTED(except_vec_nmi, 0, sp)
	j	nmi_handler
#ifdef CONFIG_CPU_MICROMIPS
	 nop
#endif
	END(except_vec_nmi)

	__FINIT

NESTED(nmi_handler, PT_SIZE, sp)
	.set	push
	.set	noat
	/*
	 * Clear ERL - restore segment mapping
	 * Clear BEV - required for page fault exception handler to work
	 */
	mfc0	k0, CP0_STATUS
	ori     k0, k0, ST0_EXL
	li	k1, ~(ST0_BEV | ST0_ERL)
	and     k0, k0, k1
	mtc0    k0, CP0_STATUS
	_ehb
	SAVE_ALL
	move	a0, sp
	jal	nmi_exception_handler
	/* nmi_exception_handler never returns */
	.set	pop
	END(nmi_handler)

	.macro	__build_clear_none
	.endm

	.macro	__build_clear_sti
	TRACE_IRQS_ON
	STI
	.endm

	.macro	__build_clear_cli
	CLI
	TRACE_IRQS_OFF
	.endm

	.macro	__build_clear_fpe
	.set	push
	/* gas fails to assemble cfc1 for some archs (octeon).*/ \
	.set	mips1
	cfc1	a1, fcr31
	li	a2, ~(0x3f << 12)
	and	a2, a1
	ctc1	a2, fcr31
	.set	pop
	TRACE_IRQS_ON
	STI
	.endm

	.macro	__build_clear_ade
	MFC0	t0, CP0_BADVADDR
	PTR_S	t0, PT_BVADDR(sp)
	KMODE
	.endm

	.macro	__BUILD_silent exception
	.endm

	/* Gas tries to parse the PRINT argument as a string containing
	   string escapes and emits bogus warnings if it believes to
	   recognize an unknown escape code.  So make the arguments
	   start with an n and gas will believe \n is ok ...  */
	.macro	__BUILD_verbose nexception
	LONG_L	a1, PT_EPC(sp)
#ifdef CONFIG_32BIT
	PRINT("Got \nexception at %08lx\012")
#endif
#ifdef CONFIG_64BIT
	PRINT("Got \nexception at %016lx\012")
#endif
	.endm

	.macro	__BUILD_count exception
	LONG_L	t0,exception_count_\exception
	LONG_ADDIU t0, 1
	LONG_S	t0,exception_count_\exception
	.comm	exception_count\exception, 8, 8
	.endm

	.macro	__BUILD_HANDLER exception handler clear verbose ext
	.align	5
	NESTED(handle_\exception, PT_SIZE, sp)
	.set	noat
	SAVE_ALL
	FEXPORT(handle_\exception\ext)
	__BUILD_clear_\clear
	.set	at
	__BUILD_\verbose \exception
	move	a0, sp
	PTR_LA	ra, ret_from_exception
	j	do_\handler
	END(handle_\exception)
	.endm

	.macro	BUILD_HANDLER exception handler clear verbose
	__BUILD_HANDLER \exception \handler \clear \verbose _int
	.endm

	BUILD_HANDLER adel ade ade silent		/* #4  */
	BUILD_HANDLER ades ade ade silent		/* #5  */
	BUILD_HANDLER ibe be cli silent			/* #6  */
	BUILD_HANDLER dbe be cli silent			/* #7  */
	BUILD_HANDLER bp bp sti silent			/* #9  */
	BUILD_HANDLER ri ri sti silent			/* #10 */
	BUILD_HANDLER cpu cpu sti silent		/* #11 */
	BUILD_HANDLER ov ov sti silent			/* #12 */
	BUILD_HANDLER tr tr sti silent			/* #13 */
	BUILD_HANDLER msa_fpe msa_fpe sti silent	/* #14 */
	BUILD_HANDLER fpe fpe fpe silent		/* #15 */
	BUILD_HANDLER ftlb ftlb none silent		/* #16 */
	BUILD_HANDLER msa msa sti silent		/* #21 */
	BUILD_HANDLER mdmx mdmx sti silent		/* #22 */
#ifdef	CONFIG_HARDWARE_WATCHPOINTS
	/*
	 * For watch, interrupts will be enabled after the watch
	 * registers are read.
	 */
	BUILD_HANDLER watch watch cli silent		/* #23 */
#else
	BUILD_HANDLER watch watch sti verbose		/* #23 */
#endif
	BUILD_HANDLER mcheck mcheck cli verbose		/* #24 */
	BUILD_HANDLER mt mt sti silent			/* #25 */
	BUILD_HANDLER dsp dsp sti silent		/* #26 */
	BUILD_HANDLER reserved reserved sti verbose	/* others */

	.align	5
	LEAF(handle_ri_rdhwr_vivt)
	.set	push
	.set	noat
	.set	noreorder
	/* check if TLB contains a entry for EPC */
	MFC0	k1, CP0_ENTRYHI
	andi	k1, 0xff	/* ASID_MASK */
	MFC0	k0, CP0_EPC
	PTR_SRL k0, _PAGE_SHIFT + 1
	PTR_SLL k0, _PAGE_SHIFT + 1
	or	k1, k0
	MTC0	k1, CP0_ENTRYHI
	mtc0_tlbw_hazard
	tlbp
	tlb_probe_hazard
	mfc0	k1, CP0_INDEX
	.set	pop
	bltz	k1, handle_ri	/* slow path */
	/* fall thru */
	END(handle_ri_rdhwr_vivt)

	LEAF(handle_ri_rdhwr)
	.set	push
	.set	noat
	.set	noreorder
	/* MIPS32:    0x7c03e83b: rdhwr v1,$29 */
	/* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
	MFC0	k1, CP0_EPC
#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2)
	and     k0, k1, 1
	beqz    k0, 1f
	xor     k1, k0
	lhu     k0, (k1)
	lhu     k1, 2(k1)
	ins     k1, k0, 16, 16
	lui     k0, 0x007d
	b       docheck
	ori     k0, 0x6b3c
1:
	lui     k0, 0x7c03
	lw      k1, (k1)
	ori     k0, 0xe83b
#else
	andi    k0, k1, 1
	bnez    k0, handle_ri
	lui     k0, 0x7c03
	lw      k1, (k1)
	ori     k0, 0xe83b
#endif
	.set    reorder
docheck:
	bne	k0, k1, handle_ri	/* if not ours */

isrdhwr:
	/* The insn is rdhwr.  No need to check CAUSE.BD here. */
	get_saved_sp	/* k1 := current_thread_info */
	.set	noreorder
	MFC0	k0, CP0_EPC
#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX)
	ori	k1, _THREAD_MASK
	xori	k1, _THREAD_MASK
	LONG_L	v1, TI_TP_VALUE(k1)
	LONG_ADDIU	k0, 4
	jr	k0
	 rfe
#else
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
	LONG_ADDIU	k0, 4		/* stall on $k0 */
#else
	.set	at=v1
	LONG_ADDIU	k0, 4
	.set	noat
#endif
	MTC0	k0, CP0_EPC
	/* I hope three instructions between MTC0 and ERET are enough... */
	ori	k1, _THREAD_MASK
	xori	k1, _THREAD_MASK
	LONG_L	v1, TI_TP_VALUE(k1)
	.set	arch=r4000
	eret
	.set	mips0
#endif
	.set	pop
	END(handle_ri_rdhwr)

#ifdef CONFIG_64BIT
/* A temporary overflow handler used by check_daddi(). */

	__INIT

	BUILD_HANDLER  daddi_ov daddi_ov none silent	/* #12 */
#endif
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
	7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
619b6e18	8	* Copyright (C) 2002, 2007 Maciej W. Rozycki
2a0b24f5	9	* Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved.
1da177e4	10	*/
1da177e4 LT	11	#include <linux/init.h>
	12
	13	#include <asm/asm.h>
41c594ab	14	#include <asm/asmmacro.h>
1da177e4	15	#include <asm/cacheops.h>
192ef366	16	#include <asm/irqflags.h>
1da177e4 LT	17	#include <asm/regdef.h>
	18	#include <asm/fpregdef.h>
	19	#include <asm/mipsregs.h>
	20	#include <asm/stackframe.h>
	21	#include <asm/war.h>
c65a5480	22	#include <asm/thread_info.h>
1da177e4	23
1da177e4 LT	24	__INIT
1da177e4 LT	25
1da177e4 LT	26	/*
	27	* General exception vector for all other CPUs.
	28	*
	29	* Be careful when changing this, it has to be at most 128 bytes
	30	* to fit into space reserved for the exception handler.
	31	*/
	32	NESTED(except_vec3_generic, 0, sp)
	33	.set push
	34	.set noat
	35	#if R5432_CP0_INTERRUPT_WAR
	36	mfc0 k0, CP0_INDEX
	37	#endif
	38	mfc0 k1, CP0_CAUSE
	39	andi k1, k1, 0x7c
875d43e7	40	#ifdef CONFIG_64BIT
1da177e4 LT	41	dsll k1, k1, 1
	42	#endif
	43	PTR_L k0, exception_handlers(k1)
	44	jr k0
	45	.set pop
	46	END(except_vec3_generic)
	47
	48	/*
	49	* General exception handler for CPUs with virtual coherency exception.
	50	*
	51	* Be careful when changing this, it has to be at most 256 (as a special
	52	* exception) bytes to fit into space reserved for the exception handler.
	53	*/
	54	NESTED(except_vec3_r4000, 0, sp)
	55	.set push
a809d460	56	.set arch=r4000
1da177e4 LT	57	.set noat
	58	mfc0 k1, CP0_CAUSE
	59	li k0, 31<<2
	60	andi k1, k1, 0x7c
	61	.set push
	62	.set noreorder
	63	.set nomacro
	64	beq k1, k0, handle_vced
	65	li k0, 14<<2
	66	beq k1, k0, handle_vcei
875d43e7	67	#ifdef CONFIG_64BIT
69903d65	68	dsll k1, k1, 1
1da177e4 LT	69	#endif
	70	.set pop
	71	PTR_L k0, exception_handlers(k1)
	72	jr k0
	73
	74	/*
	75	* Big shit, we now may have two dirty primary cache lines for the same
69903d65	76	* physical address. We can safely invalidate the line pointed to by
1da177e4 LT	77	* c0_badvaddr because after return from this exception handler the
	78	* load / store will be re-executed.
	79	*/
	80	handle_vced:
69903d65	81	MFC0 k0, CP0_BADVADDR
1da177e4 LT	82	li k1, -4 # Is this ...
	83	and k0, k1 # ... really needed?
	84	mtc0 zero, CP0_TAGLO
69903d65 TS	85	cache Index_Store_Tag_D, (k0)
69903d65 TS	86	cache Hit_Writeback_Inv_SD, (k0)
1da177e4 LT	87	#ifdef CONFIG_PROC_FS
	88	PTR_LA k0, vced_count
	89	lw k1, (k0)
	90	addiu k1, 1
	91	sw k1, (k0)
	92	#endif
	93	eret
	94
	95	handle_vcei:
	96	MFC0 k0, CP0_BADVADDR
	97	cache Hit_Writeback_Inv_SD, (k0) # also cleans pi
	98	#ifdef CONFIG_PROC_FS
	99	PTR_LA k0, vcei_count
	100	lw k1, (k0)
	101	addiu k1, 1
	102	sw k1, (k0)
	103	#endif
	104	eret
	105	.set pop
	106	END(except_vec3_r4000)
	107
e4ac58af RB	108	__FINIT
e4ac58af RB	109
c65a5480	110	.align 5 /* 32 byte rollback region */
087d990b	111	LEAF(__r4k_wait)
c65a5480 AN	112	.set push
	113	.set noreorder
	114	/* start of rollback region */
	115	LONG_L t0, TI_FLAGS($28)
	116	nop
	117	andi t0, _TIF_NEED_RESCHED
	118	bnez t0, 1f
	119	nop
	120	nop
	121	nop
2a0b24f5 SH	122	#ifdef CONFIG_CPU_MICROMIPS
	123	nop
	124	nop
	125	nop
	126	nop
	127	#endif
a809d460	128	.set arch=r4000
c65a5480 AN	129	wait
c65a5480 AN	130	/* end of rollback region (the region size must be power of two) */
c65a5480 AN	131	1:
c65a5480 AN	132	jr ra
2a0b24f5 SH	133	nop
2a0b24f5 SH	134	.set pop
087d990b	135	END(__r4k_wait)
c65a5480 AN	136
	137	.macro BUILD_ROLLBACK_PROLOGUE handler
	138	FEXPORT(rollback_\handler)
	139	.set push
	140	.set noat
	141	MFC0 k0, CP0_EPC
087d990b	142	PTR_LA k1, __r4k_wait
c65a5480 AN	143	ori k0, 0x1f /* 32 byte rollback region */
	144	xori k0, 0x1f
	145	bne k0, k1, 9f
	146	MTC0 k0, CP0_EPC
	147	9:
	148	.set pop
	149	.endm
	150
70342287	151	.align 5
c65a5480	152	BUILD_ROLLBACK_PROLOGUE handle_int
e4ac58af	153	NESTED(handle_int, PT_SIZE, sp)
fe99f1b1 CD	154	#ifdef CONFIG_TRACE_IRQFLAGS
	155	/*
	156	* Check to see if the interrupted code has just disabled
	157	* interrupts and ignore this interrupt for now if so.
	158	*
	159	* local_irq_disable() disables interrupts and then calls
	160	* trace_hardirqs_off() to track the state. If an interrupt is taken
	161	* after interrupts are disabled but before the state is updated
	162	* it will appear to restore_all that it is incorrectly returning with
	163	* interrupts disabled
	164	*/
	165	.set push
	166	.set noat
	167	mfc0 k0, CP0_STATUS
	168	#if defined(CONFIG_CPU_R3000) \|\| defined(CONFIG_CPU_TX39XX)
	169	and k0, ST0_IEP
	170	bnez k0, 1f
	171
c6563e85	172	mfc0 k0, CP0_EPC
fe99f1b1 CD	173	.set noreorder
	174	j k0
	175	rfe
	176	#else
	177	and k0, ST0_IE
	178	bnez k0, 1f
	179
	180	eret
	181	#endif
	182	1:
	183	.set pop
	184	#endif
e4ac58af RB	185	SAVE_ALL
e4ac58af RB	186	CLI
192ef366	187	TRACE_IRQS_OFF
e4ac58af	188
937a8015 RB	189	LONG_L s0, TI_REGS($28)
937a8015 RB	190	LONG_S sp, TI_REGS($28)
f431baa5	191	PTR_LA ra, ret_from_irq
2a0b24f5 SH	192	PTR_LA v0, plat_irq_dispatch
	193	jr v0
	194	#ifdef CONFIG_CPU_MICROMIPS
	195	nop
	196	#endif
e4ac58af RB	197	END(handle_int)
	198
	199	__INIT
	200
1da177e4 LT	201	/*
	202	* Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
	203	* This is a dedicated interrupt exception vector which reduces the
	204	* interrupt processing overhead. The jump instruction will be replaced
	205	* at the initialization time.
	206	*
	207	* Be careful when changing this, it has to be at most 128 bytes
	208	* to fit into space reserved for the exception handler.
	209	*/
	210	NESTED(except_vec4, 0, sp)
	211	1: j 1b /* Dummy, will be replaced */
	212	END(except_vec4)
	213
	214	/*
	215	* EJTAG debug exception handler.
	216	* The EJTAG debug exception entry point is 0xbfc00480, which
2a0b24f5	217	* normally is in the boot PROM, so the boot PROM must do an
1da177e4 LT	218	* unconditional jump to this vector.
	219	*/
	220	NESTED(except_vec_ejtag_debug, 0, sp)
	221	j ejtag_debug_handler
2a0b24f5 SH	222	#ifdef CONFIG_CPU_MICROMIPS
	223	nop
	224	#endif
1da177e4 LT	225	END(except_vec_ejtag_debug)
	226
	227	__FINIT
	228
e01402b1 RB	229	/*
	230	* Vectored interrupt handler.
	231	* This prototype is copied to ebase + n*IntCtl.VS and patched
	232	* to invoke the handler
	233	*/
c65a5480	234	BUILD_ROLLBACK_PROLOGUE except_vec_vi
e01402b1 RB	235	NESTED(except_vec_vi, 0, sp)
	236	SAVE_SOME
	237	SAVE_AT
	238	.set push
	239	.set noreorder
2a0b24f5	240	PTR_LA v1, except_vec_vi_handler
7df42461	241	FEXPORT(except_vec_vi_lui)
e01402b1	242	lui v0, 0 /* Patched */
2a0b24f5	243	jr v1
7df42461	244	FEXPORT(except_vec_vi_ori)
e01402b1 RB	245	ori v0, 0 /* Patched */
	246	.set pop
	247	END(except_vec_vi)
	248	EXPORT(except_vec_vi_end)
	249
	250	/*
	251	* Common Vectored Interrupt code
	252	* Complete the register saves and invoke the handler which is passed in $v0
	253	*/
	254	NESTED(except_vec_vi_handler, 0, sp)
	255	SAVE_TEMP
	256	SAVE_STATIC
	257	CLI
8c364435 RB	258	#ifdef CONFIG_TRACE_IRQFLAGS
8c364435 RB	259	move s0, v0
192ef366	260	TRACE_IRQS_OFF
8c364435 RB	261	move v0, s0
8c364435 RB	262	#endif
937a8015 RB	263
	264	LONG_L s0, TI_REGS($28)
	265	LONG_S sp, TI_REGS($28)
23126692	266	PTR_LA ra, ret_from_irq
f431baa5	267	jr v0
e01402b1 RB	268	END(except_vec_vi_handler)
e01402b1 RB	269
1da177e4 LT	270	/*
	271	* EJTAG debug exception handler.
	272	*/
	273	NESTED(ejtag_debug_handler, PT_SIZE, sp)
	274	.set push
	275	.set noat
	276	MTC0 k0, CP0_DESAVE
	277	mfc0 k0, CP0_DEBUG
	278
	279	sll k0, k0, 30 # Check for SDBBP.
	280	bgez k0, ejtag_return
	281
	282	PTR_LA k0, ejtag_debug_buffer
	283	LONG_S k1, 0(k0)
	284	SAVE_ALL
	285	move a0, sp
	286	jal ejtag_exception_handler
	287	RESTORE_ALL
	288	PTR_LA k0, ejtag_debug_buffer
	289	LONG_L k1, 0(k0)
	290
	291	ejtag_return:
	292	MFC0 k0, CP0_DESAVE
	293	.set mips32
	294	deret
	295	.set pop
	296	END(ejtag_debug_handler)
	297
	298	/*
	299	* This buffer is reserved for the use of the EJTAG debug
	300	* handler.
	301	*/
	302	.data
	303	EXPORT(ejtag_debug_buffer)
	304	.fill LONGSIZE
	305	.previous
	306
	307	__INIT
	308
	309	/*
	310	* NMI debug exception handler for MIPS reference boards.
	311	* The NMI debug exception entry point is 0xbfc00000, which
	312	* normally is in the boot PROM, so the boot PROM must do a
	313	* unconditional jump to this vector.
	314	*/
	315	NESTED(except_vec_nmi, 0, sp)
	316	j nmi_handler
2a0b24f5 SH	317	#ifdef CONFIG_CPU_MICROMIPS
	318	nop
	319	#endif
1da177e4 LT	320	END(except_vec_nmi)
	321
	322	__FINIT
	323
	324	NESTED(nmi_handler, PT_SIZE, sp)
	325	.set push
	326	.set noat
83e4da1e LY	327	/*
	328	* Clear ERL - restore segment mapping
	329	* Clear BEV - required for page fault exception handler to work
	330	*/
	331	mfc0 k0, CP0_STATUS
	332	ori k0, k0, ST0_EXL
	333	li k1, ~(ST0_BEV \| ST0_ERL)
	334	and k0, k0, k1
	335	mtc0 k0, CP0_STATUS
	336	_ehb
1da177e4	337	SAVE_ALL
70342287	338	move a0, sp
1da177e4	339	jal nmi_exception_handler
83e4da1e	340	/* nmi_exception_handler never returns */
1da177e4 LT	341	.set pop
	342	END(nmi_handler)
	343
	344	.macro __build_clear_none
	345	.endm
	346
	347	.macro __build_clear_sti
192ef366	348	TRACE_IRQS_ON
1da177e4 LT	349	STI
	350	.endm
	351
	352	.macro __build_clear_cli
	353	CLI
192ef366	354	TRACE_IRQS_OFF
1da177e4 LT	355	.endm
	356
	357	.macro __build_clear_fpe
25c30003 DD	358	.set push
	359	/* gas fails to assemble cfc1 for some archs (octeon).*/ \
	360	.set mips1
1da177e4 LT	361	cfc1 a1, fcr31
	362	li a2, ~(0x3f << 12)
	363	and a2, a1
	364	ctc1 a2, fcr31
25c30003	365	.set pop
192ef366	366	TRACE_IRQS_ON
1da177e4 LT	367	STI
	368	.endm
	369
	370	.macro __build_clear_ade
	371	MFC0 t0, CP0_BADVADDR
	372	PTR_S t0, PT_BVADDR(sp)
	373	KMODE
	374	.endm
	375
	376	.macro __BUILD_silent exception
	377	.endm
	378
	379	/* Gas tries to parse the PRINT argument as a string containing
	380	string escapes and emits bogus warnings if it believes to
	381	recognize an unknown escape code. So make the arguments
	382	start with an n and gas will believe \n is ok ... */
70342287	383	.macro __BUILD_verbose nexception
1da177e4	384	LONG_L a1, PT_EPC(sp)
766160c2	385	#ifdef CONFIG_32BIT
1da177e4	386	PRINT("Got \nexception at %08lx\012")
42a3b4f2	387	#endif
766160c2	388	#ifdef CONFIG_64BIT
1da177e4	389	PRINT("Got \nexception at %016lx\012")
42a3b4f2	390	#endif
1da177e4 LT	391	.endm
	392
	393	.macro __BUILD_count exception
	394	LONG_L t0,exception_count_\exception
	395	LONG_ADDIU t0, 1
	396	LONG_S t0,exception_count_\exception
	397	.comm exception_count\exception, 8, 8
	398	.endm
	399
	400	.macro __BUILD_HANDLER exception handler clear verbose ext
	401	.align 5
	402	NESTED(handle_\exception, PT_SIZE, sp)
	403	.set noat
	404	SAVE_ALL
	405	FEXPORT(handle_\exception\ext)
	406	__BUILD_clear_\clear
	407	.set at
	408	__BUILD_\verbose \exception
	409	move a0, sp
23126692 AN	410	PTR_LA ra, ret_from_exception
23126692 AN	411	j do_\handler
1da177e4 LT	412	END(handle_\exception)
	413	.endm
	414
	415	.macro BUILD_HANDLER exception handler clear verbose
70342287	416	__BUILD_HANDLER \exception \handler \clear \verbose _int
1da177e4 LT	417	.endm
	418
	419	BUILD_HANDLER adel ade ade silent /* #4 */
	420	BUILD_HANDLER ades ade ade silent /* #5 */
	421	BUILD_HANDLER ibe be cli silent /* #6 */
	422	BUILD_HANDLER dbe be cli silent /* #7 */
	423	BUILD_HANDLER bp bp sti silent /* #9 */
	424	BUILD_HANDLER ri ri sti silent /* #10 */
	425	BUILD_HANDLER cpu cpu sti silent /* #11 */
	426	BUILD_HANDLER ov ov sti silent /* #12 */
	427	BUILD_HANDLER tr tr sti silent /* #13 */
2bcb3fbc	428	BUILD_HANDLER msa_fpe msa_fpe sti silent /* #14 */
1da177e4	429	BUILD_HANDLER fpe fpe fpe silent /* #15 */
75b5b5e0	430	BUILD_HANDLER ftlb ftlb none silent /* #16 */
1db1af84	431	BUILD_HANDLER msa msa sti silent /* #21 */
1da177e4	432	BUILD_HANDLER mdmx mdmx sti silent /* #22 */
70342287	433	#ifdef CONFIG_HARDWARE_WATCHPOINTS
8bc6d05b DD	434	/*
	435	* For watch, interrupts will be enabled after the watch
	436	* registers are read.
	437	*/
	438	BUILD_HANDLER watch watch cli silent /* #23 */
b67b2b70	439	#else
1da177e4	440	BUILD_HANDLER watch watch sti verbose /* #23 */
b67b2b70	441	#endif
1da177e4	442	BUILD_HANDLER mcheck mcheck cli verbose /* #24 */
e35a5e35	443	BUILD_HANDLER mt mt sti silent /* #25 */
e50c0a8f	444	BUILD_HANDLER dsp dsp sti silent /* #26 */
1da177e4 LT	445	BUILD_HANDLER reserved reserved sti verbose /* others */
1da177e4 LT	446
5b10496b AN	447	.align 5
5b10496b AN	448	LEAF(handle_ri_rdhwr_vivt)
5b10496b AN	449	.set push
	450	.set noat
	451	.set noreorder
	452	/* check if TLB contains a entry for EPC */
	453	MFC0 k1, CP0_ENTRYHI
48c4ac97	454	andi k1, 0xff /* ASID_MASK */
5b10496b	455	MFC0 k0, CP0_EPC
70342287 RB	456	PTR_SRL k0, _PAGE_SHIFT + 1
70342287 RB	457	PTR_SLL k0, _PAGE_SHIFT + 1
5b10496b AN	458	or k1, k0
	459	MTC0 k1, CP0_ENTRYHI
	460	mtc0_tlbw_hazard
	461	tlbp
	462	tlb_probe_hazard
	463	mfc0 k1, CP0_INDEX
	464	.set pop
	465	bltz k1, handle_ri /* slow path */
	466	/* fall thru */
5b10496b AN	467	END(handle_ri_rdhwr_vivt)
	468
	469	LEAF(handle_ri_rdhwr)
	470	.set push
	471	.set noat
	472	.set noreorder
2a0b24f5 SH	473	/* MIPS32: 0x7c03e83b: rdhwr v1,$29 */
2a0b24f5 SH	474	/* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
5b10496b	475	MFC0 k1, CP0_EPC
2a0b24f5 SH	476	#if defined(CONFIG_CPU_MICROMIPS) \|\| defined(CONFIG_CPU_MIPS32_R2) \|\| defined(CONFIG_CPU_MIPS64_R2)
	477	and k0, k1, 1
	478	beqz k0, 1f
	479	xor k1, k0
	480	lhu k0, (k1)
	481	lhu k1, 2(k1)
	482	ins k1, k0, 16, 16
	483	lui k0, 0x007d
	484	b docheck
	485	ori k0, 0x6b3c
	486	1:
	487	lui k0, 0x7c03
	488	lw k1, (k1)
	489	ori k0, 0xe83b
	490	#else
	491	andi k0, k1, 1
	492	bnez k0, handle_ri
	493	lui k0, 0x7c03
	494	lw k1, (k1)
	495	ori k0, 0xe83b
	496	#endif
	497	.set reorder
	498	docheck:
5b10496b	499	bne k0, k1, handle_ri /* if not ours */
2a0b24f5 SH	500
2a0b24f5 SH	501	isrdhwr:
5b10496b AN	502	/* The insn is rdhwr. No need to check CAUSE.BD here. */
	503	get_saved_sp /* k1 := current_thread_info */
	504	.set noreorder
	505	MFC0 k0, CP0_EPC
	506	#if defined(CONFIG_CPU_R3000) \|\| defined(CONFIG_CPU_TX39XX)
	507	ori k1, _THREAD_MASK
	508	xori k1, _THREAD_MASK
	509	LONG_L v1, TI_TP_VALUE(k1)
	510	LONG_ADDIU k0, 4
	511	jr k0
	512	rfe
	513	#else
619b6e18	514	#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
5b10496b	515	LONG_ADDIU k0, 4 /* stall on $k0 */
619b6e18 MR	516	#else
	517	.set at=v1
	518	LONG_ADDIU k0, 4
	519	.set noat
	520	#endif
5b10496b AN	521	MTC0 k0, CP0_EPC
	522	/* I hope three instructions between MTC0 and ERET are enough... */
	523	ori k1, _THREAD_MASK
	524	xori k1, _THREAD_MASK
	525	LONG_L v1, TI_TP_VALUE(k1)
a809d460	526	.set arch=r4000
5b10496b AN	527	eret
	528	.set mips0
	529	#endif
	530	.set pop
	531	END(handle_ri_rdhwr)
	532
875d43e7	533	#ifdef CONFIG_64BIT
1da177e4 LT	534	/* A temporary overflow handler used by check_daddi(). */
	535
	536	__INIT
	537
	538	BUILD_HANDLER daddi_ov daddi_ov none silent /* #12 */
	539	#endif