[linux-2.6-block.git] / arch / sparc64 / mm / ultra.S

/* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $
 * ultra.S: Don't expand these all over the place...
 *
 * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com)
 */

#include <linux/config.h>
#include <asm/asi.h>
#include <asm/pgtable.h>
#include <asm/page.h>
#include <asm/spitfire.h>
#include <asm/mmu_context.h>
#include <asm/mmu.h>
#include <asm/pil.h>
#include <asm/head.h>
#include <asm/thread_info.h>
#include <asm/cacheflush.h>

	/* Basically, most of the Spitfire vs. Cheetah madness
	 * has to do with the fact that Cheetah does not support
	 * IMMU flushes out of the secondary context.  Someone needs
	 * to throw a south lake birthday party for the folks
	 * in Microelectronics who refused to fix this shit.
	 */

	/* This file is meant to be read efficiently by the CPU, not humans.
	 * Staraj sie tego nikomu nie pierdolnac...
	 */
	.text
	.align		32
	.globl		__flush_tlb_mm
__flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
	ldxa		[%o1] ASI_DMMU, %g2
	cmp		%g2, %o0
	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
	 mov		0x50, %g3
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
	retl
	 flush		%g6
	nop
	nop
	nop
	nop
	nop
	nop
	nop
	nop
	nop
	nop

	.align		32
	.globl		__flush_tlb_pending
__flush_tlb_pending:
	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
	rdpr		%pstate, %g7
	sllx		%o1, 3, %o1
	andn		%g7, PSTATE_IE, %g2
	wrpr		%g2, %pstate
	mov		SECONDARY_CONTEXT, %o4
	ldxa		[%o4] ASI_DMMU, %g2
	stxa		%o0, [%o4] ASI_DMMU
1:	sub		%o1, (1 << 3), %o1
	ldx		[%o2 + %o1], %o3
	andcc		%o3, 1, %g0
	andn		%o3, 1, %o3
	be,pn		%icc, 2f
	 or		%o3, 0x10, %o3
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
	membar		#Sync
	brnz,pt		%o1, 1b
	 nop
	stxa		%g2, [%o4] ASI_DMMU
	flush		%g6
	retl
	 wrpr		%g7, 0x0, %pstate
	nop
	nop
	nop
	nop

	.align		32
	.globl		__flush_tlb_kernel_range
__flush_tlb_kernel_range:	/* %o0=start, %o1=end */
	cmp		%o0, %o1
	be,pn		%xcc, 2f
	 sethi		%hi(PAGE_SIZE), %o4
	sub		%o1, %o0, %o3
	sub		%o3, %o4, %o3
	or		%o0, 0x20, %o0		! Nucleus
1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
	membar		#Sync
	brnz,pt		%o3, 1b
	 sub		%o3, %o4, %o3
2:	retl
	 flush		%g6

__spitfire_flush_tlb_mm_slow:
	rdpr		%pstate, %g1
	wrpr		%g1, PSTATE_IE, %pstate
	stxa		%o0, [%o1] ASI_DMMU
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
	flush		%g6
	stxa		%g2, [%o1] ASI_DMMU
	flush		%g6
	retl
	 wrpr		%g1, 0, %pstate

/*
 * The following code flushes one page_size worth.
 */
#if (PAGE_SHIFT == 13)
#define ITAG_MASK 0xfe
#elif (PAGE_SHIFT == 16)
#define ITAG_MASK 0x7fe
#else
#error unsupported PAGE_SIZE
#endif
	.section .kprobes.text, "ax"
	.align		32
	.globl		__flush_icache_page
__flush_icache_page:	/* %o0 = phys_page */
	membar		#StoreStore
	srlx		%o0, PAGE_SHIFT, %o0
	sethi		%uhi(PAGE_OFFSET), %g1
	sllx		%o0, PAGE_SHIFT, %o0
	sethi		%hi(PAGE_SIZE), %g2
	sllx		%g1, 32, %g1
	add		%o0, %g1, %o0
1:	subcc		%g2, 32, %g2
	bne,pt		%icc, 1b
	 flush		%o0 + %g2
	retl
	 nop

#ifdef DCACHE_ALIASING_POSSIBLE

#if (PAGE_SHIFT != 13)
#error only page shift of 13 is supported by dcache flush
#endif

#define DTAG_MASK 0x3

	.align		64
	.globl		__flush_dcache_page
__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
	sethi		%uhi(PAGE_OFFSET), %g1
	sllx		%g1, 32, %g1
	sub		%o0, %g1, %o0
	clr		%o4
	srlx		%o0, 11, %o0
	sethi		%hi(1 << 14), %o2
1:	ldxa		[%o4] ASI_DCACHE_TAG, %o3	! LSU	Group
	add		%o4, (1 << 5), %o4		! IEU0
	ldxa		[%o4] ASI_DCACHE_TAG, %g1	! LSU	Group
	add		%o4, (1 << 5), %o4		! IEU0
	ldxa		[%o4] ASI_DCACHE_TAG, %g2	! LSU	Group	o3 available
	add		%o4, (1 << 5), %o4		! IEU0
	andn		%o3, DTAG_MASK, %o3		! IEU1
	ldxa		[%o4] ASI_DCACHE_TAG, %g3	! LSU	Group
	add		%o4, (1 << 5), %o4		! IEU0
	andn		%g1, DTAG_MASK, %g1		! IEU1
	cmp		%o0, %o3			! IEU1	Group
	be,a,pn		%xcc, dflush1			! CTI
	 sub		%o4, (4 << 5), %o4		! IEU0	(Group)
	cmp		%o0, %g1			! IEU1	Group
	andn		%g2, DTAG_MASK, %g2		! IEU0
	be,a,pn		%xcc, dflush2			! CTI
	 sub		%o4, (3 << 5), %o4		! IEU0	(Group)
	cmp		%o0, %g2			! IEU1	Group
	andn		%g3, DTAG_MASK, %g3		! IEU0
	be,a,pn		%xcc, dflush3			! CTI
	 sub		%o4, (2 << 5), %o4		! IEU0	(Group)
	cmp		%o0, %g3			! IEU1	Group
	be,a,pn		%xcc, dflush4			! CTI
	 sub		%o4, (1 << 5), %o4		! IEU0
2:	cmp		%o4, %o2			! IEU1	Group
	bne,pt		%xcc, 1b			! CTI
	 nop						! IEU0

	/* The I-cache does not snoop local stores so we
	 * better flush that too when necessary.
	 */
	brnz,pt		%o1, __flush_icache_page
	 sllx		%o0, 11, %o0
	retl
	 nop

dflush1:stxa		%g0, [%o4] ASI_DCACHE_TAG
	add		%o4, (1 << 5), %o4
dflush2:stxa		%g0, [%o4] ASI_DCACHE_TAG
	add		%o4, (1 << 5), %o4
dflush3:stxa		%g0, [%o4] ASI_DCACHE_TAG
	add		%o4, (1 << 5), %o4
dflush4:stxa		%g0, [%o4] ASI_DCACHE_TAG
	add		%o4, (1 << 5), %o4
	membar		#Sync
	ba,pt		%xcc, 2b
	 nop
#endif /* DCACHE_ALIASING_POSSIBLE */

	.previous .text
	.align		32
__prefill_dtlb:
	rdpr		%pstate, %g7
	wrpr		%g7, PSTATE_IE, %pstate
	mov		TLB_TAG_ACCESS, %g1
	stxa		%o5, [%g1] ASI_DMMU
	stxa		%o2, [%g0] ASI_DTLB_DATA_IN
	flush		%g6
	retl
	 wrpr		%g7, %pstate
__prefill_itlb:
	rdpr		%pstate, %g7
	wrpr		%g7, PSTATE_IE, %pstate
	mov		TLB_TAG_ACCESS, %g1
	stxa		%o5, [%g1] ASI_IMMU
	stxa		%o2, [%g0] ASI_ITLB_DATA_IN
	flush		%g6
	retl
	 wrpr		%g7, %pstate

	.globl		__update_mmu_cache
__update_mmu_cache:	/* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */
	srlx		%o1, PAGE_SHIFT, %o1
	andcc		%o3, FAULT_CODE_DTLB, %g0
	sllx		%o1, PAGE_SHIFT, %o5
	bne,pt		%xcc, __prefill_dtlb
	 or		%o5, %o0, %o5
	ba,a,pt		%xcc, __prefill_itlb

	/* Cheetah specific versions, patched at boot time. */
__cheetah_flush_tlb_mm: /* 18 insns */
	rdpr		%pstate, %g7
	andn		%g7, PSTATE_IE, %g2
	wrpr		%g2, 0x0, %pstate
	wrpr		%g0, 1, %tl
	mov		PRIMARY_CONTEXT, %o2
	mov		0x40, %g3
	ldxa		[%o2] ASI_DMMU, %g2
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
	stxa		%o0, [%o2] ASI_DMMU
	stxa		%g0, [%g3] ASI_DMMU_DEMAP
	stxa		%g0, [%g3] ASI_IMMU_DEMAP
	stxa		%g2, [%o2] ASI_DMMU
	flush		%g6
	wrpr		%g0, 0, %tl
	retl
	 wrpr		%g7, 0x0, %pstate

__cheetah_flush_tlb_pending:	/* 26 insns */
	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
	rdpr		%pstate, %g7
	sllx		%o1, 3, %o1
	andn		%g7, PSTATE_IE, %g2
	wrpr		%g2, 0x0, %pstate
	wrpr		%g0, 1, %tl
	mov		PRIMARY_CONTEXT, %o4
	ldxa		[%o4] ASI_DMMU, %g2
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
	stxa		%o0, [%o4] ASI_DMMU
1:	sub		%o1, (1 << 3), %o1
	ldx		[%o2 + %o1], %o3
	andcc		%o3, 1, %g0
	be,pn		%icc, 2f
	 andn		%o3, 1, %o3
	stxa		%g0, [%o3] ASI_IMMU_DEMAP
2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP	
	membar		#Sync
	brnz,pt		%o1, 1b
	 nop
	stxa		%g2, [%o4] ASI_DMMU
	flush		%g6
	wrpr		%g0, 0, %tl
	retl
	 wrpr		%g7, 0x0, %pstate

#ifdef DCACHE_ALIASING_POSSIBLE
flush_dcpage_cheetah: /* 11 insns */
	sethi		%uhi(PAGE_OFFSET), %g1
	sllx		%g1, 32, %g1
	sub		%o0, %g1, %o0
	sethi		%hi(PAGE_SIZE), %o4
1:	subcc		%o4, (1 << 5), %o4
	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
	membar		#Sync
	bne,pt		%icc, 1b
	 nop
	retl		/* I-cache flush never needed on Cheetah, see callers. */
	 nop
#endif /* DCACHE_ALIASING_POSSIBLE */

cheetah_patch_one:
1:	lduw		[%o1], %g1
	stw		%g1, [%o0]
	flush		%o0
	subcc		%o2, 1, %o2
	add		%o1, 4, %o1
	bne,pt		%icc, 1b
	 add		%o0, 4, %o0
	retl
	 nop

	.globl		cheetah_patch_cachetlbops
cheetah_patch_cachetlbops:
	save		%sp, -128, %sp

	sethi		%hi(__flush_tlb_mm), %o0
	or		%o0, %lo(__flush_tlb_mm), %o0
	sethi		%hi(__cheetah_flush_tlb_mm), %o1
	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
	call		cheetah_patch_one
	 mov		18, %o2

	sethi		%hi(__flush_tlb_pending), %o0
	or		%o0, %lo(__flush_tlb_pending), %o0
	sethi		%hi(__cheetah_flush_tlb_pending), %o1
	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
	call		cheetah_patch_one
	 mov		26, %o2

#ifdef DCACHE_ALIASING_POSSIBLE
	sethi		%hi(__flush_dcache_page), %o0
	or		%o0, %lo(__flush_dcache_page), %o0
	sethi		%hi(flush_dcpage_cheetah), %o1
	or		%o1, %lo(flush_dcpage_cheetah), %o1
	call		cheetah_patch_one
	 mov		11, %o2
#endif /* DCACHE_ALIASING_POSSIBLE */

	ret
	 restore

#ifdef CONFIG_SMP
	/* These are all called by the slaves of a cross call, at
	 * trap level 1, with interrupts fully disabled.
	 *
	 * Register usage:
	 *   %g5	mm->context	(all tlb flushes)
	 *   %g1	address arg 1	(tlb page and range flushes)
	 *   %g7	address arg 2	(tlb range flush only)
	 *
	 *   %g6	ivector table, don't touch
	 *   %g2	scratch 1
	 *   %g3	scratch 2
	 *   %g4	scratch 3
	 *
	 * TODO: Make xcall TLB range flushes use the tricks above... -DaveM
	 */
	.align		32
	.globl		xcall_flush_tlb_mm
xcall_flush_tlb_mm:
	mov		PRIMARY_CONTEXT, %g2
	ldxa		[%g2] ASI_DMMU, %g3
	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
	stxa		%g5, [%g2] ASI_DMMU
	mov		0x40, %g4
	stxa		%g0, [%g4] ASI_DMMU_DEMAP
	stxa		%g0, [%g4] ASI_IMMU_DEMAP
	stxa		%g3, [%g2] ASI_DMMU
	retry

	.globl		xcall_flush_tlb_pending
xcall_flush_tlb_pending:
	/* %g5=context, %g1=nr, %g7=vaddrs[] */
	sllx		%g1, 3, %g1
	mov		PRIMARY_CONTEXT, %g4
	ldxa		[%g4] ASI_DMMU, %g2
	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
	or		%g5, %g4, %g5
	mov		PRIMARY_CONTEXT, %g4
	stxa		%g5, [%g4] ASI_DMMU
1:	sub		%g1, (1 << 3), %g1
	ldx		[%g7 + %g1], %g5
	andcc		%g5, 0x1, %g0
	be,pn		%icc, 2f

	 andn		%g5, 0x1, %g5
	stxa		%g0, [%g5] ASI_IMMU_DEMAP
2:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
	membar		#Sync
	brnz,pt		%g1, 1b
	 nop
	stxa		%g2, [%g4] ASI_DMMU
	retry

	.globl		xcall_flush_tlb_kernel_range
xcall_flush_tlb_kernel_range:
	sethi		%hi(PAGE_SIZE - 1), %g2
	or		%g2, %lo(PAGE_SIZE - 1), %g2
	andn		%g1, %g2, %g1
	andn		%g7, %g2, %g7
	sub		%g7, %g1, %g3
	add		%g2, 1, %g2
	sub		%g3, %g2, %g3
	or		%g1, 0x20, %g1		! Nucleus
1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
	membar		#Sync
	brnz,pt		%g3, 1b
	 sub		%g3, %g2, %g3
	retry
	nop
	nop

	/* This runs in a very controlled environment, so we do
	 * not need to worry about BH races etc.
	 */
	.globl		xcall_sync_tick
xcall_sync_tick:
	rdpr		%pstate, %g2
	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
	rdpr		%pil, %g2
	wrpr		%g0, 15, %pil
	sethi		%hi(109f), %g7
	b,pt		%xcc, etrap_irq
109:	 or		%g7, %lo(109b), %g7
	call		smp_synchronize_tick_client
	 nop
	clr		%l6
	b		rtrap_xcall
	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1

	/* NOTE: This is SPECIAL!!  We do etrap/rtrap however
	 *       we choose to deal with the "BH's run with
	 *       %pil==15" problem (described in asm/pil.h)
	 *       by just invoking rtrap directly past where
	 *       BH's are checked for.
	 *
	 *       We do it like this because we do not want %pil==15
	 *       lockups to prevent regs being reported.
	 */
	.globl		xcall_report_regs
xcall_report_regs:
	rdpr		%pstate, %g2
	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
	rdpr		%pil, %g2
	wrpr		%g0, 15, %pil
	sethi		%hi(109f), %g7
	b,pt		%xcc, etrap_irq
109:	 or		%g7, %lo(109b), %g7
	call		__show_regs
	 add		%sp, PTREGS_OFF, %o0
	clr		%l6
	/* Has to be a non-v9 branch due to the large distance. */
	b		rtrap_xcall
	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1

#ifdef DCACHE_ALIASING_POSSIBLE
	.align		32
	.globl		xcall_flush_dcache_page_cheetah
xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
	sethi		%hi(PAGE_SIZE), %g3
1:	subcc		%g3, (1 << 5), %g3
	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
	membar		#Sync
	bne,pt		%icc, 1b
	 nop
	retry
	nop
#endif /* DCACHE_ALIASING_POSSIBLE */

	.globl		xcall_flush_dcache_page_spitfire
xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
				     %g7 == kernel page virtual address
				     %g5 == (page->mapping != NULL)  */
#ifdef DCACHE_ALIASING_POSSIBLE
	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
1:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
	andcc		%g2, 0x3, %g0
	be,pn		%xcc, 2f
	 andn		%g2, 0x3, %g2
	cmp		%g2, %g1

	bne,pt		%xcc, 2f
	 nop
	stxa		%g0, [%g3] ASI_DCACHE_TAG
	membar		#Sync
2:	cmp		%g3, 0
	bne,pt		%xcc, 1b
	 sub		%g3, (1 << 5), %g3

	brz,pn		%g5, 2f
#endif /* DCACHE_ALIASING_POSSIBLE */
	 sethi		%hi(PAGE_SIZE), %g3

1:	flush		%g7
	subcc		%g3, (1 << 5), %g3
	bne,pt		%icc, 1b
	 add		%g7, (1 << 5), %g7

2:	retry
	nop
	nop

	.globl		xcall_promstop
xcall_promstop:
	rdpr		%pstate, %g2
	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
	rdpr		%pil, %g2
	wrpr		%g0, 15, %pil
	sethi		%hi(109f), %g7
	b,pt		%xcc, etrap_irq
109:	 or		%g7, %lo(109b), %g7
	flushw
	call		prom_stopself
	 nop
	/* We should not return, just spin if we do... */
1:	b,a,pt		%xcc, 1b
	nop

	.data

errata32_hwbug:
	.xword	0

	.text

	/* These two are not performance critical... */
	.globl		xcall_flush_tlb_all_spitfire
xcall_flush_tlb_all_spitfire:
	/* Spitfire Errata #32 workaround. */
	sethi		%hi(errata32_hwbug), %g4
	stx		%g0, [%g4 + %lo(errata32_hwbug)]

	clr		%g2
	clr		%g3
1:	ldxa		[%g3] ASI_DTLB_DATA_ACCESS, %g4
	and		%g4, _PAGE_L, %g5
	brnz,pn		%g5, 2f
	 mov		TLB_TAG_ACCESS, %g7

	stxa		%g0, [%g7] ASI_DMMU
	membar		#Sync
	stxa		%g0, [%g3] ASI_DTLB_DATA_ACCESS
	membar		#Sync

	/* Spitfire Errata #32 workaround. */
	sethi		%hi(errata32_hwbug), %g4
	stx		%g0, [%g4 + %lo(errata32_hwbug)]

2:	ldxa		[%g3] ASI_ITLB_DATA_ACCESS, %g4
	and		%g4, _PAGE_L, %g5
	brnz,pn		%g5, 2f
	 mov		TLB_TAG_ACCESS, %g7

	stxa		%g0, [%g7] ASI_IMMU
	membar		#Sync
	stxa		%g0, [%g3] ASI_ITLB_DATA_ACCESS
	membar		#Sync

	/* Spitfire Errata #32 workaround. */
	sethi		%hi(errata32_hwbug), %g4
	stx		%g0, [%g4 + %lo(errata32_hwbug)]

2:	add		%g2, 1, %g2
	cmp		%g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
	ble,pt		%icc, 1b
	 sll		%g2, 3, %g3
	flush		%g6
	retry

	.globl		xcall_flush_tlb_all_cheetah
xcall_flush_tlb_all_cheetah:
	mov		0x80, %g2
	stxa		%g0, [%g2] ASI_DMMU_DEMAP
	stxa		%g0, [%g2] ASI_IMMU_DEMAP
	retry

	/* These just get rescheduled to PIL vectors. */
	.globl		xcall_call_function
xcall_call_function:
	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
	retry

	.globl		xcall_receive_signal
xcall_receive_signal:
	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
	retry

	.globl		xcall_capture
xcall_capture:
	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
	retry

#endif /* CONFIG_SMP */
Commit	Line	Data
1da177e4 LT	1	/* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $
	2	* ultra.S: Don't expand these all over the place...
	3	*
	4	* Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com)
	5	*/
	6
	7	#include <linux/config.h>
	8	#include <asm/asi.h>
	9	#include <asm/pgtable.h>
	10	#include <asm/page.h>
	11	#include <asm/spitfire.h>
	12	#include <asm/mmu_context.h>
2ef27778	13	#include <asm/mmu.h>
1da177e4 LT	14	#include <asm/pil.h>
	15	#include <asm/head.h>
	16	#include <asm/thread_info.h>
	17	#include <asm/cacheflush.h>
	18
	19	/* Basically, most of the Spitfire vs. Cheetah madness
	20	* has to do with the fact that Cheetah does not support
	21	* IMMU flushes out of the secondary context. Someone needs
	22	* to throw a south lake birthday party for the folks
	23	* in Microelectronics who refused to fix this shit.
	24	*/
	25
	26	/* This file is meant to be read efficiently by the CPU, not humans.
	27	* Staraj sie tego nikomu nie pierdolnac...
	28	*/
	29	.text
	30	.align 32
	31	.globl __flush_tlb_mm
	32	__flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
	33	ldxa [%o1] ASI_DMMU, %g2
	34	cmp %g2, %o0
	35	bne,pn %icc, __spitfire_flush_tlb_mm_slow
	36	mov 0x50, %g3
	37	stxa %g0, [%g3] ASI_DMMU_DEMAP
	38	stxa %g0, [%g3] ASI_IMMU_DEMAP
	39	retl
	40	flush %g6
	41	nop
	42	nop
	43	nop
	44	nop
	45	nop
	46	nop
	47	nop
	48	nop
2ef27778 DM	49	nop
2ef27778 DM	50	nop
1da177e4 LT	51
	52	.align 32
	53	.globl __flush_tlb_pending
	54	__flush_tlb_pending:
	55	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
	56	rdpr %pstate, %g7
	57	sllx %o1, 3, %o1
	58	andn %g7, PSTATE_IE, %g2
	59	wrpr %g2, %pstate
	60	mov SECONDARY_CONTEXT, %o4
	61	ldxa [%o4] ASI_DMMU, %g2
	62	stxa %o0, [%o4] ASI_DMMU
	63	1: sub %o1, (1 << 3), %o1
	64	ldx [%o2 + %o1], %o3
	65	andcc %o3, 1, %g0
	66	andn %o3, 1, %o3
	67	be,pn %icc, 2f
	68	or %o3, 0x10, %o3
	69	stxa %g0, [%o3] ASI_IMMU_DEMAP
	70	2: stxa %g0, [%o3] ASI_DMMU_DEMAP
	71	membar #Sync
	72	brnz,pt %o1, 1b
	73	nop
	74	stxa %g2, [%o4] ASI_DMMU
	75	flush %g6
	76	retl
	77	wrpr %g7, 0x0, %pstate
fef43da4	78	nop
2ef27778 DM	79	nop
	80	nop
	81	nop
1da177e4 LT	82
	83	.align 32
	84	.globl __flush_tlb_kernel_range
	85	__flush_tlb_kernel_range: /* %o0=start, %o1=end */
	86	cmp %o0, %o1
	87	be,pn %xcc, 2f
	88	sethi %hi(PAGE_SIZE), %o4
	89	sub %o1, %o0, %o3
	90	sub %o3, %o4, %o3
	91	or %o0, 0x20, %o0 ! Nucleus
	92	1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
	93	stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
	94	membar #Sync
	95	brnz,pt %o3, 1b
	96	sub %o3, %o4, %o3
	97	2: retl
	98	flush %g6
	99
	100	__spitfire_flush_tlb_mm_slow:
	101	rdpr %pstate, %g1
	102	wrpr %g1, PSTATE_IE, %pstate
	103	stxa %o0, [%o1] ASI_DMMU
	104	stxa %g0, [%g3] ASI_DMMU_DEMAP
	105	stxa %g0, [%g3] ASI_IMMU_DEMAP
	106	flush %g6
	107	stxa %g2, [%o1] ASI_DMMU
	108	flush %g6
	109	retl
	110	wrpr %g1, 0, %pstate
	111
	112	/*
	113	* The following code flushes one page_size worth.
	114	*/
	115	#if (PAGE_SHIFT == 13)
	116	#define ITAG_MASK 0xfe
	117	#elif (PAGE_SHIFT == 16)
	118	#define ITAG_MASK 0x7fe
	119	#else
	120	#error unsupported PAGE_SIZE
	121	#endif
83005161	122	.section .kprobes.text, "ax"
1da177e4 LT	123	.align 32
	124	.globl __flush_icache_page
	125	__flush_icache_page: /* %o0 = phys_page */
	126	membar #StoreStore
	127	srlx %o0, PAGE_SHIFT, %o0
	128	sethi %uhi(PAGE_OFFSET), %g1
	129	sllx %o0, PAGE_SHIFT, %o0
	130	sethi %hi(PAGE_SIZE), %g2
	131	sllx %g1, 32, %g1
	132	add %o0, %g1, %o0
	133	1: subcc %g2, 32, %g2
	134	bne,pt %icc, 1b
	135	flush %o0 + %g2
	136	retl
	137	nop
	138
	139	#ifdef DCACHE_ALIASING_POSSIBLE
	140
	141	#if (PAGE_SHIFT != 13)
	142	#error only page shift of 13 is supported by dcache flush
	143	#endif
	144
	145	#define DTAG_MASK 0x3
	146
	147	.align 64
	148	.globl __flush_dcache_page
	149	__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
	150	sethi %uhi(PAGE_OFFSET), %g1
	151	sllx %g1, 32, %g1
	152	sub %o0, %g1, %o0
	153	clr %o4
	154	srlx %o0, 11, %o0
	155	sethi %hi(1 << 14), %o2
	156	1: ldxa [%o4] ASI_DCACHE_TAG, %o3 ! LSU Group
	157	add %o4, (1 << 5), %o4 ! IEU0
	158	ldxa [%o4] ASI_DCACHE_TAG, %g1 ! LSU Group
	159	add %o4, (1 << 5), %o4 ! IEU0
	160	ldxa [%o4] ASI_DCACHE_TAG, %g2 ! LSU Group o3 available
	161	add %o4, (1 << 5), %o4 ! IEU0
	162	andn %o3, DTAG_MASK, %o3 ! IEU1
	163	ldxa [%o4] ASI_DCACHE_TAG, %g3 ! LSU Group
	164	add %o4, (1 << 5), %o4 ! IEU0
	165	andn %g1, DTAG_MASK, %g1 ! IEU1
	166	cmp %o0, %o3 ! IEU1 Group
	167	be,a,pn %xcc, dflush1 ! CTI
	168	sub %o4, (4 << 5), %o4 ! IEU0 (Group)
	169	cmp %o0, %g1 ! IEU1 Group
	170	andn %g2, DTAG_MASK, %g2 ! IEU0
	171	be,a,pn %xcc, dflush2 ! CTI
	172	sub %o4, (3 << 5), %o4 ! IEU0 (Group)
	173	cmp %o0, %g2 ! IEU1 Group
	174	andn %g3, DTAG_MASK, %g3 ! IEU0
	175	be,a,pn %xcc, dflush3 ! CTI
	176	sub %o4, (2 << 5), %o4 ! IEU0 (Group)
	177	cmp %o0, %g3 ! IEU1 Group
	178	be,a,pn %xcc, dflush4 ! CTI
	179	sub %o4, (1 << 5), %o4 ! IEU0
	180	2: cmp %o4, %o2 ! IEU1 Group
	181	bne,pt %xcc, 1b ! CTI
	182	nop ! IEU0
	183
	184	/* The I-cache does not snoop local stores so we
	185	* better flush that too when necessary.
	186	*/
187	brnz,pt %o1, __flush_icache_page
188	sllx %o0, 11, %o0
189	retl
190	nop
191
192	dflush1:stxa %g0, [%o4] ASI_DCACHE_TAG
193	add %o4, (1 << 5), %o4
194	dflush2:stxa %g0, [%o4] ASI_DCACHE_TAG
195	add %o4, (1 << 5), %o4
196	dflush3:stxa %g0, [%o4] ASI_DCACHE_TAG
197	add %o4, (1 << 5), %o4
198	dflush4:stxa %g0, [%o4] ASI_DCACHE_TAG
199	add %o4, (1 << 5), %o4
200	membar #Sync
201	ba,pt %xcc, 2b
202	nop
203	#endif /* DCACHE_ALIASING_POSSIBLE */
204
05e14cb3	205	.previous .text
1da177e4 LT	206	.align 32
	207	__prefill_dtlb:
	208	rdpr %pstate, %g7
	209	wrpr %g7, PSTATE_IE, %pstate
	210	mov TLB_TAG_ACCESS, %g1
	211	stxa %o5, [%g1] ASI_DMMU
	212	stxa %o2, [%g0] ASI_DTLB_DATA_IN
	213	flush %g6
	214	retl
	215	wrpr %g7, %pstate
	216	__prefill_itlb:
	217	rdpr %pstate, %g7
	218	wrpr %g7, PSTATE_IE, %pstate
	219	mov TLB_TAG_ACCESS, %g1
	220	stxa %o5, [%g1] ASI_IMMU
	221	stxa %o2, [%g0] ASI_ITLB_DATA_IN
	222	flush %g6
	223	retl
	224	wrpr %g7, %pstate
	225
	226	.globl __update_mmu_cache
	227	__update_mmu_cache: /* %o0=hw_context, %o1=address, %o2=pte, %o3=fault_code */
	228	srlx %o1, PAGE_SHIFT, %o1
	229	andcc %o3, FAULT_CODE_DTLB, %g0
	230	sllx %o1, PAGE_SHIFT, %o5
	231	bne,pt %xcc, __prefill_dtlb
	232	or %o5, %o0, %o5
	233	ba,a,pt %xcc, __prefill_itlb
	234
2ef27778 DM	235	/* Cheetah specific versions, patched at boot time. */
2ef27778 DM	236	__cheetah_flush_tlb_mm: /* 18 insns */
1da177e4 LT	237	rdpr %pstate, %g7
	238	andn %g7, PSTATE_IE, %g2
	239	wrpr %g2, 0x0, %pstate
	240	wrpr %g0, 1, %tl
	241	mov PRIMARY_CONTEXT, %o2
	242	mov 0x40, %g3
	243	ldxa [%o2] ASI_DMMU, %g2
2ef27778 DM	244	srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1
	245	sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1
	246	or %o0, %o1, %o0 /* Preserve nucleus page size fields */
1da177e4 LT	247	stxa %o0, [%o2] ASI_DMMU
	248	stxa %g0, [%g3] ASI_DMMU_DEMAP
	249	stxa %g0, [%g3] ASI_IMMU_DEMAP
	250	stxa %g2, [%o2] ASI_DMMU
	251	flush %g6
	252	wrpr %g0, 0, %tl
	253	retl
	254	wrpr %g7, 0x0, %pstate
	255
2ef27778	256	__cheetah_flush_tlb_pending: /* 26 insns */
1da177e4 LT	257	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
	258	rdpr %pstate, %g7
	259	sllx %o1, 3, %o1
	260	andn %g7, PSTATE_IE, %g2
	261	wrpr %g2, 0x0, %pstate
	262	wrpr %g0, 1, %tl
	263	mov PRIMARY_CONTEXT, %o4
	264	ldxa [%o4] ASI_DMMU, %g2
2ef27778 DM	265	srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
	266	sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
	267	or %o0, %o3, %o0 /* Preserve nucleus page size fields */
1da177e4 LT	268	stxa %o0, [%o4] ASI_DMMU
	269	1: sub %o1, (1 << 3), %o1
	270	ldx [%o2 + %o1], %o3
	271	andcc %o3, 1, %g0
	272	be,pn %icc, 2f
	273	andn %o3, 1, %o3
	274	stxa %g0, [%o3] ASI_IMMU_DEMAP
	275	2: stxa %g0, [%o3] ASI_DMMU_DEMAP
b445e26c	276	membar #Sync
1da177e4	277	brnz,pt %o1, 1b
b445e26c	278	nop
1da177e4 LT	279	stxa %g2, [%o4] ASI_DMMU
	280	flush %g6
	281	wrpr %g0, 0, %tl
	282	retl
	283	wrpr %g7, 0x0, %pstate
	284
	285	#ifdef DCACHE_ALIASING_POSSIBLE
	286	flush_dcpage_cheetah: /* 11 insns */
	287	sethi %uhi(PAGE_OFFSET), %g1
	288	sllx %g1, 32, %g1
	289	sub %o0, %g1, %o0
	290	sethi %hi(PAGE_SIZE), %o4
	291	1: subcc %o4, (1 << 5), %o4
	292	stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
	293	membar #Sync
	294	bne,pt %icc, 1b
	295	nop
	296	retl /* I-cache flush never needed on Cheetah, see callers. */
	297	nop
	298	#endif /* DCACHE_ALIASING_POSSIBLE */
	299
	300	cheetah_patch_one:
	301	1: lduw [%o1], %g1
	302	stw %g1, [%o0]
	303	flush %o0
	304	subcc %o2, 1, %o2
	305	add %o1, 4, %o1
	306	bne,pt %icc, 1b
	307	add %o0, 4, %o0
	308	retl
	309	nop
	310
	311	.globl cheetah_patch_cachetlbops
	312	cheetah_patch_cachetlbops:
	313	save %sp, -128, %sp
	314
	315	sethi %hi(__flush_tlb_mm), %o0
	316	or %o0, %lo(__flush_tlb_mm), %o0
	317	sethi %hi(__cheetah_flush_tlb_mm), %o1
	318	or %o1, %lo(__cheetah_flush_tlb_mm), %o1
	319	call cheetah_patch_one
2ef27778	320	mov 18, %o2
1da177e4 LT	321
	322	sethi %hi(__flush_tlb_pending), %o0
	323	or %o0, %lo(__flush_tlb_pending), %o0
	324	sethi %hi(__cheetah_flush_tlb_pending), %o1
	325	or %o1, %lo(__cheetah_flush_tlb_pending), %o1
	326	call cheetah_patch_one
2ef27778	327	mov 26, %o2
1da177e4 LT	328
	329	#ifdef DCACHE_ALIASING_POSSIBLE
	330	sethi %hi(__flush_dcache_page), %o0
	331	or %o0, %lo(__flush_dcache_page), %o0
	332	sethi %hi(flush_dcpage_cheetah), %o1
	333	or %o1, %lo(flush_dcpage_cheetah), %o1
	334	call cheetah_patch_one
	335	mov 11, %o2
	336	#endif /* DCACHE_ALIASING_POSSIBLE */
	337
	338	ret
	339	restore
	340
	341	#ifdef CONFIG_SMP
	342	/* These are all called by the slaves of a cross call, at
	343	* trap level 1, with interrupts fully disabled.
	344	*
	345	* Register usage:
	346	* %g5 mm->context (all tlb flushes)
	347	* %g1 address arg 1 (tlb page and range flushes)
	348	* %g7 address arg 2 (tlb range flush only)
	349	*
	350	* %g6 ivector table, don't touch
	351	* %g2 scratch 1
	352	* %g3 scratch 2
	353	* %g4 scratch 3
	354	*
	355	* TODO: Make xcall TLB range flushes use the tricks above... -DaveM
	356	*/
	357	.align 32
	358	.globl xcall_flush_tlb_mm
	359	xcall_flush_tlb_mm:
	360	mov PRIMARY_CONTEXT, %g2
1da177e4	361	ldxa [%g2] ASI_DMMU, %g3
2ef27778 DM	362	srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
	363	sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
	364	or %g5, %g4, %g5 /* Preserve nucleus page size fields */
1da177e4	365	stxa %g5, [%g2] ASI_DMMU
2ef27778	366	mov 0x40, %g4
1da177e4 LT	367	stxa %g0, [%g4] ASI_DMMU_DEMAP
	368	stxa %g0, [%g4] ASI_IMMU_DEMAP
	369	stxa %g3, [%g2] ASI_DMMU
	370	retry
	371
	372	.globl xcall_flush_tlb_pending
	373	xcall_flush_tlb_pending:
	374	/* %g5=context, %g1=nr, %g7=vaddrs[] */
	375	sllx %g1, 3, %g1
	376	mov PRIMARY_CONTEXT, %g4
	377	ldxa [%g4] ASI_DMMU, %g2
2ef27778 DM	378	srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
	379	sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
	380	or %g5, %g4, %g5
	381	mov PRIMARY_CONTEXT, %g4
1da177e4 LT	382	stxa %g5, [%g4] ASI_DMMU
	383	1: sub %g1, (1 << 3), %g1
	384	ldx [%g7 + %g1], %g5
	385	andcc %g5, 0x1, %g0
	386	be,pn %icc, 2f
	387
	388	andn %g5, 0x1, %g5
	389	stxa %g0, [%g5] ASI_IMMU_DEMAP
	390	2: stxa %g0, [%g5] ASI_DMMU_DEMAP
	391	membar #Sync
	392	brnz,pt %g1, 1b
	393	nop
	394	stxa %g2, [%g4] ASI_DMMU
	395	retry
	396
	397	.globl xcall_flush_tlb_kernel_range
	398	xcall_flush_tlb_kernel_range:
	399	sethi %hi(PAGE_SIZE - 1), %g2
	400	or %g2, %lo(PAGE_SIZE - 1), %g2
	401	andn %g1, %g2, %g1
	402	andn %g7, %g2, %g7
	403	sub %g7, %g1, %g3
	404	add %g2, 1, %g2
	405	sub %g3, %g2, %g3
	406	or %g1, 0x20, %g1 ! Nucleus
	407	1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
	408	stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
	409	membar #Sync
	410	brnz,pt %g3, 1b
	411	sub %g3, %g2, %g3
	412	retry
	413	nop
	414	nop
	415
	416	/* This runs in a very controlled environment, so we do
	417	* not need to worry about BH races etc.
	418	*/
	419	.globl xcall_sync_tick
	420	xcall_sync_tick:
	421	rdpr %pstate, %g2
	422	wrpr %g2, PSTATE_IG \| PSTATE_AG, %pstate
	423	rdpr %pil, %g2
	424	wrpr %g0, 15, %pil
	425	sethi %hi(109f), %g7
	426	b,pt %xcc, etrap_irq
	427	109: or %g7, %lo(109b), %g7
	428	call smp_synchronize_tick_client
	429	nop
	430	clr %l6
	431	b rtrap_xcall
	432	ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
	433
	434	/* NOTE: This is SPECIAL!! We do etrap/rtrap however
	435	* we choose to deal with the "BH's run with
	436	* %pil==15" problem (described in asm/pil.h)
	437	* by just invoking rtrap directly past where
	438	* BH's are checked for.
	439	*
	440	* We do it like this because we do not want %pil==15
	441	* lockups to prevent regs being reported.
	442	*/
	443	.globl xcall_report_regs
	444	xcall_report_regs:
	445	rdpr %pstate, %g2
446	wrpr %g2, PSTATE_IG \| PSTATE_AG, %pstate
447	rdpr %pil, %g2
448	wrpr %g0, 15, %pil
449	sethi %hi(109f), %g7
450	b,pt %xcc, etrap_irq
451	109: or %g7, %lo(109b), %g7
452	call __show_regs
453	add %sp, PTREGS_OFF, %o0
454	clr %l6
455	/* Has to be a non-v9 branch due to the large distance. */
456	b rtrap_xcall
457	ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
458
459	#ifdef DCACHE_ALIASING_POSSIBLE
460	.align 32
461	.globl xcall_flush_dcache_page_cheetah
462	xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
463	sethi %hi(PAGE_SIZE), %g3
464	1: subcc %g3, (1 << 5), %g3
465	stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
466	membar #Sync
467	bne,pt %icc, 1b
468	nop
469	retry
470	nop
471	#endif /* DCACHE_ALIASING_POSSIBLE */
472
473	.globl xcall_flush_dcache_page_spitfire
474	xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
475	%g7 == kernel page virtual address
476	%g5 == (page->mapping != NULL) */
477	#ifdef DCACHE_ALIASING_POSSIBLE
478	srlx %g1, (13 - 2), %g1 ! Form tag comparitor
479	sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
480	sub %g3, (1 << 5), %g3 ! D$ linesize == 32
481	1: ldxa [%g3] ASI_DCACHE_TAG, %g2
482	andcc %g2, 0x3, %g0
483	be,pn %xcc, 2f
484	andn %g2, 0x3, %g2
485	cmp %g2, %g1
486
487	bne,pt %xcc, 2f
488	nop
489	stxa %g0, [%g3] ASI_DCACHE_TAG
490	membar #Sync
491	2: cmp %g3, 0
492	bne,pt %xcc, 1b
493	sub %g3, (1 << 5), %g3
494
495	brz,pn %g5, 2f
496	#endif /* DCACHE_ALIASING_POSSIBLE */
497	sethi %hi(PAGE_SIZE), %g3
498
499	1: flush %g7
500	subcc %g3, (1 << 5), %g3
501	bne,pt %icc, 1b
502	add %g7, (1 << 5), %g7
503
504	2: retry
505	nop
506	nop
507
508	.globl xcall_promstop
509	xcall_promstop:
510	rdpr %pstate, %g2
511	wrpr %g2, PSTATE_IG \| PSTATE_AG, %pstate
512	rdpr %pil, %g2
513	wrpr %g0, 15, %pil
514	sethi %hi(109f), %g7
515	b,pt %xcc, etrap_irq
516	109: or %g7, %lo(109b), %g7
517	flushw
518	call prom_stopself
519	nop
520	/* We should not return, just spin if we do... */
521	1: b,a,pt %xcc, 1b
522	nop
523
524	.data
525
526	errata32_hwbug:
527	.xword 0
528
529	.text
530
531	/* These two are not performance critical... */
532	.globl xcall_flush_tlb_all_spitfire
533	xcall_flush_tlb_all_spitfire:
534	/* Spitfire Errata #32 workaround. */
535	sethi %hi(errata32_hwbug), %g4
536	stx %g0, [%g4 + %lo(errata32_hwbug)]
537
538	clr %g2
539	clr %g3
540	1: ldxa [%g3] ASI_DTLB_DATA_ACCESS, %g4
541	and %g4, _PAGE_L, %g5
542	brnz,pn %g5, 2f
543	mov TLB_TAG_ACCESS, %g7
544
545	stxa %g0, [%g7] ASI_DMMU
546	membar #Sync
547	stxa %g0, [%g3] ASI_DTLB_DATA_ACCESS
548	membar #Sync
549
550	/* Spitfire Errata #32 workaround. */
551	sethi %hi(errata32_hwbug), %g4
552	stx %g0, [%g4 + %lo(errata32_hwbug)]
553
554	2: ldxa [%g3] ASI_ITLB_DATA_ACCESS, %g4
555	and %g4, _PAGE_L, %g5
556	brnz,pn %g5, 2f
557	mov TLB_TAG_ACCESS, %g7
558
559	stxa %g0, [%g7] ASI_IMMU
560	membar #Sync
561	stxa %g0, [%g3] ASI_ITLB_DATA_ACCESS
562	membar #Sync
563
564	/* Spitfire Errata #32 workaround. */
565	sethi %hi(errata32_hwbug), %g4
566	stx %g0, [%g4 + %lo(errata32_hwbug)]
567
568	2: add %g2, 1, %g2
569	cmp %g2, SPITFIRE_HIGHEST_LOCKED_TLBENT
570	ble,pt %icc, 1b
571	sll %g2, 3, %g3
572	flush %g6
573	retry
574
575	.globl xcall_flush_tlb_all_cheetah
576	xcall_flush_tlb_all_cheetah:
577	mov 0x80, %g2
578	stxa %g0, [%g2] ASI_DMMU_DEMAP
579	stxa %g0, [%g2] ASI_IMMU_DEMAP
580	retry
581
582	/* These just get rescheduled to PIL vectors. */
583	.globl xcall_call_function
584	xcall_call_function:
585	wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
586	retry
587
588	.globl xcall_receive_signal
589	xcall_receive_signal:
590	wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
591	retry
592
593	.globl xcall_capture
594	xcall_capture:
595	wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint
596	retry
597
598	#endif /* CONFIG_SMP */