[linux-2.6-block.git] / arch / sparc / lib / copy_page.S

/* SPDX-License-Identifier: GPL-2.0 */
/* clear_page.S: UltraSparc optimized copy page.
 *
 * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
 */

#include <asm/visasm.h>
#include <asm/thread_info.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/spitfire.h>
#include <asm/head.h>
#include <asm/export.h>

	/* What we used to do was lock a TLB entry into a specific
	 * TLB slot, clear the page with interrupts disabled, then
	 * restore the original TLB entry.  This was great for
	 * disturbing the TLB as little as possible, but it meant
	 * we had to keep interrupts disabled for a long time.
	 *
	 * Now, we simply use the normal TLB loading mechanism,
	 * and this makes the cpu choose a slot all by itself.
	 * Then we do a normal TLB flush on exit.  We need only
	 * disable preemption during the clear.
	 */

#define	DCACHE_SIZE	(PAGE_SIZE * 2)

#if (PAGE_SHIFT == 13)
#define PAGE_SIZE_REM	0x80
#elif (PAGE_SHIFT == 16)
#define PAGE_SIZE_REM	0x100
#else
#error Wrong PAGE_SHIFT specified
#endif

#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7)	\
	fsrc2	%reg0, %f48; 	fsrc2	%reg1, %f50;		\
	fsrc2	%reg2, %f52; 	fsrc2	%reg3, %f54;		\
	fsrc2	%reg4, %f56; 	fsrc2	%reg5, %f58;		\
	fsrc2	%reg6, %f60; 	fsrc2	%reg7, %f62;

	.text

	.align		32
	.globl		copy_user_page
	.type		copy_user_page,#function
	EXPORT_SYMBOL(copy_user_page)
copy_user_page:		/* %o0=dest, %o1=src, %o2=vaddr */
	lduw		[%g6 + TI_PRE_COUNT], %o4
	sethi		%hi(PAGE_OFFSET), %g2
	sethi		%hi(PAGE_SIZE), %o3

	ldx		[%g2 + %lo(PAGE_OFFSET)], %g2
	sethi		%hi(PAGE_KERNEL_LOCKED), %g3

	ldx		[%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
	sub		%o0, %g2, %g1		! dest paddr

	sub		%o1, %g2, %g2		! src paddr

	and		%o2, %o3, %o0		! vaddr D-cache alias bit
	or		%g1, %g3, %g1		! dest TTE data

	or		%g2, %g3, %g2		! src TTE data
	sethi		%hi(TLBTEMP_BASE), %o3

	sethi		%hi(DCACHE_SIZE), %o1
	add		%o0, %o3, %o0		! dest TTE vaddr

	add		%o4, 1, %o2
	add		%o0, %o1, %o1		! src TTE vaddr

	/* Disable preemption.  */
	mov		TLB_TAG_ACCESS, %g3
	stw		%o2, [%g6 + TI_PRE_COUNT]

	/* Load TLB entries.  */
	rdpr		%pstate, %o2
	wrpr		%o2, PSTATE_IE, %pstate
	stxa		%o0, [%g3] ASI_DMMU
	stxa		%g1, [%g0] ASI_DTLB_DATA_IN
	membar		#Sync
	stxa		%o1, [%g3] ASI_DMMU
	stxa		%g2, [%g0] ASI_DTLB_DATA_IN
	membar		#Sync
	wrpr		%o2, 0x0, %pstate

cheetah_copy_page_insn:
	ba,pt		%xcc, 9f
	 nop

1:
	VISEntryHalf
	membar		#StoreLoad | #StoreStore | #LoadStore
	sethi		%hi((PAGE_SIZE/64)-2), %o2
	mov		%o0, %g1
	prefetch	[%o1 + 0x000], #one_read
	or		%o2, %lo((PAGE_SIZE/64)-2), %o2
	prefetch	[%o1 + 0x040], #one_read
	prefetch	[%o1 + 0x080], #one_read
	prefetch	[%o1 + 0x0c0], #one_read
	ldd		[%o1 + 0x000], %f0
	prefetch	[%o1 + 0x100], #one_read
	ldd		[%o1 + 0x008], %f2
	prefetch	[%o1 + 0x140], #one_read
	ldd		[%o1 + 0x010], %f4
	prefetch	[%o1 + 0x180], #one_read
	fsrc2		%f0, %f16
	ldd		[%o1 + 0x018], %f6
	fsrc2		%f2, %f18
	ldd		[%o1 + 0x020], %f8
	fsrc2		%f4, %f20
	ldd		[%o1 + 0x028], %f10
	fsrc2		%f6, %f22
	ldd		[%o1 + 0x030], %f12
	fsrc2		%f8, %f24
	ldd		[%o1 + 0x038], %f14
	fsrc2		%f10, %f26
	ldd		[%o1 + 0x040], %f0
1:	ldd		[%o1 + 0x048], %f2
	fsrc2		%f12, %f28
	ldd		[%o1 + 0x050], %f4
	fsrc2		%f14, %f30
	stda		%f16, [%o0] ASI_BLK_P
	ldd		[%o1 + 0x058], %f6
	fsrc2		%f0, %f16
	ldd		[%o1 + 0x060], %f8
	fsrc2		%f2, %f18
	ldd		[%o1 + 0x068], %f10
	fsrc2		%f4, %f20
	ldd		[%o1 + 0x070], %f12
	fsrc2		%f6, %f22
	ldd		[%o1 + 0x078], %f14
	fsrc2		%f8, %f24
	ldd		[%o1 + 0x080], %f0
	prefetch	[%o1 + 0x180], #one_read
	fsrc2		%f10, %f26
	subcc		%o2, 1, %o2
	add		%o0, 0x40, %o0
	bne,pt		%xcc, 1b
	 add		%o1, 0x40, %o1

	ldd		[%o1 + 0x048], %f2
	fsrc2		%f12, %f28
	ldd		[%o1 + 0x050], %f4
	fsrc2		%f14, %f30
	stda		%f16, [%o0] ASI_BLK_P
	ldd		[%o1 + 0x058], %f6
	fsrc2		%f0, %f16
	ldd		[%o1 + 0x060], %f8
	fsrc2		%f2, %f18
	ldd		[%o1 + 0x068], %f10
	fsrc2		%f4, %f20
	ldd		[%o1 + 0x070], %f12
	fsrc2		%f6, %f22
	add		%o0, 0x40, %o0
	ldd		[%o1 + 0x078], %f14
	fsrc2		%f8, %f24
	fsrc2		%f10, %f26
	fsrc2		%f12, %f28
	fsrc2		%f14, %f30
	stda		%f16, [%o0] ASI_BLK_P
	membar		#Sync
	VISExitHalf
	ba,pt		%xcc, 5f
	 nop

9:
	VISEntry
	ldub		[%g6 + TI_FAULT_CODE], %g3
	mov		%o0, %g1
	cmp		%g3, 0
	rd		%asi, %g3
	be,a,pt		%icc, 1f
	 wr		%g0, ASI_BLK_P, %asi
	wr		%g0, ASI_BLK_COMMIT_P, %asi
1:	ldda		[%o1] ASI_BLK_P, %f0
	add		%o1, 0x40, %o1
	ldda		[%o1] ASI_BLK_P, %f16
	add		%o1, 0x40, %o1
	sethi		%hi(PAGE_SIZE), %o2
1:	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] %asi
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] %asi
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
	ldda		[%o1] ASI_BLK_P, %f16
	stda		%f48, [%o0] %asi
	sub		%o2, 0x40, %o2
	add		%o1, 0x40, %o1
	cmp		%o2, PAGE_SIZE_REM
	bne,pt		%xcc, 1b
	 add		%o0, 0x40, %o0
#if (PAGE_SHIFT == 16)
	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	ldda		[%o1] ASI_BLK_P, %f32
	stda		%f48, [%o0] %asi
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	ldda		[%o1] ASI_BLK_P, %f0
	stda		%f48, [%o0] %asi
	add		%o1, 0x40, %o1
	sub		%o2, 0x40, %o2
	add		%o0, 0x40, %o0
	membar		#Sync
	stda		%f32, [%o0] %asi
	add		%o0, 0x40, %o0
	stda		%f0, [%o0] %asi
#else
	membar		#Sync
	stda		%f0, [%o0] %asi
	add		%o0, 0x40, %o0
	stda		%f16, [%o0] %asi
#endif
	membar		#Sync
	wr		%g3, 0x0, %asi
	VISExit

5:
	stxa		%g0, [%g1] ASI_DMMU_DEMAP
	membar		#Sync

	sethi		%hi(DCACHE_SIZE), %g2
	stxa		%g0, [%g1 + %g2] ASI_DMMU_DEMAP
	membar		#Sync

	retl
	 stw		%o4, [%g6 + TI_PRE_COUNT]

	.size		copy_user_page, .-copy_user_page

	.globl		cheetah_patch_copy_page
cheetah_patch_copy_page:
	sethi		%hi(0x01000000), %o1	! NOP
	sethi		%hi(cheetah_copy_page_insn), %o0
	or		%o0, %lo(cheetah_copy_page_insn), %o0
	stw		%o1, [%o0]
	membar		#StoreStore
	flush		%o0
	retl
	 nop
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	/* clear_page.S: UltraSparc optimized copy page.
	3	*
	4	* Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com)
	5	* Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
	6	*/
	7
	8	#include <asm/visasm.h>
	9	#include <asm/thread_info.h>
	10	#include <asm/page.h>
	11	#include <asm/pgtable.h>
	12	#include <asm/spitfire.h>
	13	#include <asm/head.h>
d3867f04	14	#include <asm/export.h>
1da177e4 LT	15
	16	/* What we used to do was lock a TLB entry into a specific
	17	* TLB slot, clear the page with interrupts disabled, then
	18	* restore the original TLB entry. This was great for
	19	* disturbing the TLB as little as possible, but it meant
	20	* we had to keep interrupts disabled for a long time.
	21	*
	22	* Now, we simply use the normal TLB loading mechanism,
	23	* and this makes the cpu choose a slot all by itself.
	24	* Then we do a normal TLB flush on exit. We need only
	25	* disable preemption during the clear.
	26	*/
	27
1da177e4 LT	28	#define DCACHE_SIZE (PAGE_SIZE * 2)
1da177e4 LT	29
f7fe9334	30	#if (PAGE_SHIFT == 13)
1da177e4	31	#define PAGE_SIZE_REM 0x80
f7fe9334	32	#elif (PAGE_SHIFT == 16)
1da177e4 LT	33	#define PAGE_SIZE_REM 0x100
	34	#else
	35	#error Wrong PAGE_SHIFT specified
	36	#endif
	37
	38	#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
6f1d827f DM	39	fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
	40	fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
	41	fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
	42	fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
1da177e4 LT	43
	44	.text
	45
	46	.align 32
	47	.globl copy_user_page
	48	.type copy_user_page,#function
d3867f04	49	EXPORT_SYMBOL(copy_user_page)
1da177e4 LT	50	copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
1da177e4 LT	51	lduw [%g6 + TI_PRE_COUNT], %o4
b2d43834	52	sethi %hi(PAGE_OFFSET), %g2
1da177e4 LT	53	sethi %hi(PAGE_SIZE), %o3
1da177e4 LT	54
b2d43834	55	ldx [%g2 + %lo(PAGE_OFFSET)], %g2
c4bce90e	56	sethi %hi(PAGE_KERNEL_LOCKED), %g3
1da177e4	57
c4bce90e	58	ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
1da177e4 LT	59	sub %o0, %g2, %g1 ! dest paddr
	60
	61	sub %o1, %g2, %g2 ! src paddr
1da177e4 LT	62
	63	and %o2, %o3, %o0 ! vaddr D-cache alias bit
	64	or %g1, %g3, %g1 ! dest TTE data
	65
	66	or %g2, %g3, %g2 ! src TTE data
	67	sethi %hi(TLBTEMP_BASE), %o3
	68
	69	sethi %hi(DCACHE_SIZE), %o1
	70	add %o0, %o3, %o0 ! dest TTE vaddr
	71
	72	add %o4, 1, %o2
	73	add %o0, %o1, %o1 ! src TTE vaddr
	74
	75	/* Disable preemption. */
	76	mov TLB_TAG_ACCESS, %g3
	77	stw %o2, [%g6 + TI_PRE_COUNT]
	78
	79	/* Load TLB entries. */
	80	rdpr %pstate, %o2
	81	wrpr %o2, PSTATE_IE, %pstate
	82	stxa %o0, [%g3] ASI_DMMU
	83	stxa %g1, [%g0] ASI_DTLB_DATA_IN
	84	membar #Sync
	85	stxa %o1, [%g3] ASI_DMMU
	86	stxa %g2, [%g0] ASI_DTLB_DATA_IN
	87	membar #Sync
	88	wrpr %o2, 0x0, %pstate
	89
dbd2fdf5	90	cheetah_copy_page_insn:
1da177e4 LT	91	ba,pt %xcc, 9f
	92	nop
	93
	94	1:
	95	VISEntryHalf
	96	membar #StoreLoad \| #StoreStore \| #LoadStore
	97	sethi %hi((PAGE_SIZE/64)-2), %o2
	98	mov %o0, %g1
	99	prefetch [%o1 + 0x000], #one_read
	100	or %o2, %lo((PAGE_SIZE/64)-2), %o2
	101	prefetch [%o1 + 0x040], #one_read
	102	prefetch [%o1 + 0x080], #one_read
	103	prefetch [%o1 + 0x0c0], #one_read
	104	ldd [%o1 + 0x000], %f0
	105	prefetch [%o1 + 0x100], #one_read
	106	ldd [%o1 + 0x008], %f2
	107	prefetch [%o1 + 0x140], #one_read
	108	ldd [%o1 + 0x010], %f4
	109	prefetch [%o1 + 0x180], #one_read
6f1d827f	110	fsrc2 %f0, %f16
1da177e4	111	ldd [%o1 + 0x018], %f6
6f1d827f	112	fsrc2 %f2, %f18
1da177e4	113	ldd [%o1 + 0x020], %f8
6f1d827f	114	fsrc2 %f4, %f20
1da177e4	115	ldd [%o1 + 0x028], %f10
6f1d827f	116	fsrc2 %f6, %f22
1da177e4	117	ldd [%o1 + 0x030], %f12
6f1d827f	118	fsrc2 %f8, %f24
1da177e4	119	ldd [%o1 + 0x038], %f14
6f1d827f	120	fsrc2 %f10, %f26
1da177e4 LT	121	ldd [%o1 + 0x040], %f0
1da177e4 LT	122	1: ldd [%o1 + 0x048], %f2
6f1d827f	123	fsrc2 %f12, %f28
1da177e4	124	ldd [%o1 + 0x050], %f4
6f1d827f	125	fsrc2 %f14, %f30
1da177e4 LT	126	stda %f16, [%o0] ASI_BLK_P
1da177e4 LT	127	ldd [%o1 + 0x058], %f6
6f1d827f	128	fsrc2 %f0, %f16
1da177e4	129	ldd [%o1 + 0x060], %f8
6f1d827f	130	fsrc2 %f2, %f18
1da177e4	131	ldd [%o1 + 0x068], %f10
6f1d827f	132	fsrc2 %f4, %f20
1da177e4	133	ldd [%o1 + 0x070], %f12
6f1d827f	134	fsrc2 %f6, %f22
1da177e4	135	ldd [%o1 + 0x078], %f14
6f1d827f	136	fsrc2 %f8, %f24
1da177e4 LT	137	ldd [%o1 + 0x080], %f0
1da177e4 LT	138	prefetch [%o1 + 0x180], #one_read
6f1d827f	139	fsrc2 %f10, %f26
1da177e4 LT	140	subcc %o2, 1, %o2
	141	add %o0, 0x40, %o0
	142	bne,pt %xcc, 1b
	143	add %o1, 0x40, %o1
	144
	145	ldd [%o1 + 0x048], %f2
6f1d827f	146	fsrc2 %f12, %f28
1da177e4	147	ldd [%o1 + 0x050], %f4
6f1d827f	148	fsrc2 %f14, %f30
1da177e4 LT	149	stda %f16, [%o0] ASI_BLK_P
1da177e4 LT	150	ldd [%o1 + 0x058], %f6
6f1d827f	151	fsrc2 %f0, %f16
1da177e4	152	ldd [%o1 + 0x060], %f8
6f1d827f	153	fsrc2 %f2, %f18
1da177e4	154	ldd [%o1 + 0x068], %f10
6f1d827f	155	fsrc2 %f4, %f20
1da177e4	156	ldd [%o1 + 0x070], %f12
6f1d827f	157	fsrc2 %f6, %f22
1da177e4 LT	158	add %o0, 0x40, %o0
1da177e4 LT	159	ldd [%o1 + 0x078], %f14
6f1d827f DM	160	fsrc2 %f8, %f24
	161	fsrc2 %f10, %f26
	162	fsrc2 %f12, %f28
	163	fsrc2 %f14, %f30
1da177e4 LT	164	stda %f16, [%o0] ASI_BLK_P
	165	membar #Sync
	166	VISExitHalf
	167	ba,pt %xcc, 5f
	168	nop
	169
	170	9:
	171	VISEntry
	172	ldub [%g6 + TI_FAULT_CODE], %g3
	173	mov %o0, %g1
	174	cmp %g3, 0
	175	rd %asi, %g3
	176	be,a,pt %icc, 1f
	177	wr %g0, ASI_BLK_P, %asi
	178	wr %g0, ASI_BLK_COMMIT_P, %asi
	179	1: ldda [%o1] ASI_BLK_P, %f0
	180	add %o1, 0x40, %o1
	181	ldda [%o1] ASI_BLK_P, %f16
	182	add %o1, 0x40, %o1
	183	sethi %hi(PAGE_SIZE), %o2
	184	1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	185	ldda [%o1] ASI_BLK_P, %f32
	186	stda %f48, [%o0] %asi
	187	add %o1, 0x40, %o1
	188	sub %o2, 0x40, %o2
	189	add %o0, 0x40, %o0
	190	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	191	ldda [%o1] ASI_BLK_P, %f0
	192	stda %f48, [%o0] %asi
	193	add %o1, 0x40, %o1
	194	sub %o2, 0x40, %o2
	195	add %o0, 0x40, %o0
	196	TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
	197	ldda [%o1] ASI_BLK_P, %f16
	198	stda %f48, [%o0] %asi
	199	sub %o2, 0x40, %o2
	200	add %o1, 0x40, %o1
	201	cmp %o2, PAGE_SIZE_REM
	202	bne,pt %xcc, 1b
	203	add %o0, 0x40, %o0
f7fe9334	204	#if (PAGE_SHIFT == 16)
1da177e4 LT	205	TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
	206	ldda [%o1] ASI_BLK_P, %f32
	207	stda %f48, [%o0] %asi
	208	add %o1, 0x40, %o1
	209	sub %o2, 0x40, %o2
	210	add %o0, 0x40, %o0
	211	TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
	212	ldda [%o1] ASI_BLK_P, %f0
	213	stda %f48, [%o0] %asi
	214	add %o1, 0x40, %o1
	215	sub %o2, 0x40, %o2
	216	add %o0, 0x40, %o0
	217	membar #Sync
	218	stda %f32, [%o0] %asi
	219	add %o0, 0x40, %o0
	220	stda %f0, [%o0] %asi
	221	#else
	222	membar #Sync
	223	stda %f0, [%o0] %asi
	224	add %o0, 0x40, %o0
	225	stda %f16, [%o0] %asi
	226	#endif
	227	membar #Sync
	228	wr %g3, 0x0, %asi
	229	VISExit
	230
	231	5:
	232	stxa %g0, [%g1] ASI_DMMU_DEMAP
	233	membar #Sync
	234
	235	sethi %hi(DCACHE_SIZE), %g2
	236	stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP
	237	membar #Sync
	238
	239	retl
	240	stw %o4, [%g6 + TI_PRE_COUNT]
	241
	242	.size copy_user_page, .-copy_user_page
dbd2fdf5 DM	243
	244	.globl cheetah_patch_copy_page
	245	cheetah_patch_copy_page:
	246	sethi %hi(0x01000000), %o1 ! NOP
	247	sethi %hi(cheetah_copy_page_insn), %o0
	248	or %o0, %lo(cheetah_copy_page_insn), %o0
	249	stw %o1, [%o0]
	250	membar #StoreStore
	251	flush %o0
	252	retl
	253	nop