[linux-2.6-block.git] / arch / alpha / lib / ev6-copy_user.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * arch/alpha/lib/ev6-copy_user.S
 *
 * 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
 *
 * Copy to/from user space, handling exceptions as we go..  This
 * isn't exactly pretty.
 *
 * This is essentially the same as "memcpy()", but with a few twists.
 * Notably, we have to make sure that $0 is always up-to-date and
 * contains the right "bytes left to copy" value (and that it is updated
 * only _after_ a successful copy). There is also some rather minor
 * exception setup stuff..
 *
 * Much of the information about 21264 scheduling/coding comes from:
 *	Compiler Writer's Guide for the Alpha 21264
 *	abbreviated as 'CWG' in other comments here
 *	ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
 * Scheduling notation:
 *	E	- either cluster
 *	U	- upper subcluster; U0 - subcluster U0; U1 - subcluster U1
 *	L	- lower subcluster; L0 - subcluster L0; L1 - subcluster L1
 */

#include <asm/export.h>
/* Allow an exception for an insn; exit if we get one.  */
#define EXI(x,y...)			\
	99: x,##y;			\
	.section __ex_table,"a";	\
	.long 99b - .;			\
	lda $31, $exitin-99b($31);	\
	.previous

#define EXO(x,y...)			\
	99: x,##y;			\
	.section __ex_table,"a";	\
	.long 99b - .;			\
	lda $31, $exitout-99b($31);	\
	.previous

	.set noat
	.align 4
	.globl __copy_user
	.ent __copy_user
				# Pipeline info: Slotting & Comments
__copy_user:
	.prologue 0
	mov $18, $0		# .. .. .. E
	subq $18, 32, $1	# .. .. E. ..	: Is this going to be a small copy?
	nop			# .. E  .. ..
	beq $18, $zerolength	# U  .. .. ..	: U L U L

	and $16,7,$3		# .. .. .. E	: is leading dest misalignment
	ble $1, $onebyteloop	# .. .. U  ..	: 1st branch : small amount of data
	beq $3, $destaligned	# .. U  .. ..	: 2nd (one cycle fetcher stall)
	subq $3, 8, $3		# E  .. .. ..	: L U U L : trip counter
/*
 * The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
 * This loop aligns the destination a byte at a time
 * We know we have at least one trip through this loop
 */
$aligndest:
	EXI( ldbu $1,0($17) )	# .. .. .. L	: Keep loads separate from stores
	addq $16,1,$16		# .. .. E  ..	: Section 3.8 in the CWG
	addq $3,1,$3		# .. E  .. ..	:
	nop			# E  .. .. ..	: U L U L

/*
 * the -1 is to compensate for the inc($16) done in a previous quadpack
 * which allows us zero dependencies within either quadpack in the loop
 */
	EXO( stb $1,-1($16) )	# .. .. .. L	:
	addq $17,1,$17		# .. .. E  ..	: Section 3.8 in the CWG
	subq $0,1,$0		# .. E  .. ..	:
	bne $3, $aligndest	# U  .. .. ..	: U L U L

/*
 * If we fell through into here, we have a minimum of 33 - 7 bytes
 * If we arrived via branch, we have a minimum of 32 bytes
 */
$destaligned:
	and $17,7,$1		# .. .. .. E	: Check _current_ source alignment
	bic $0,7,$4		# .. .. E  ..	: number bytes as a quadword loop
	EXI( ldq_u $3,0($17) )	# .. L  .. ..	: Forward fetch for fallthrough code
	beq $1,$quadaligned	# U  .. .. ..	: U L U L

/*
 * In the worst case, we've just executed an ldq_u here from 0($17)
 * and we'll repeat it once if we take the branch
 */

/* Misaligned quadword loop - not unrolled.  Leave it that way. */
$misquad:
	EXI( ldq_u $2,8($17) )	# .. .. .. L	:
	subq $4,8,$4		# .. .. E  ..	:
	extql $3,$17,$3		# .. U  .. ..	:
	extqh $2,$17,$1		# U  .. .. ..	: U U L L

	bis $3,$1,$1		# .. .. .. E	:
	EXO( stq $1,0($16) )	# .. .. L  ..	:
	addq $17,8,$17		# .. E  .. ..	:
	subq $0,8,$0		# E  .. .. ..	: U L L U

	addq $16,8,$16		# .. .. .. E	:
	bis $2,$2,$3		# .. .. E  ..	:
	nop			# .. E  .. ..	:
	bne $4,$misquad		# U  .. .. ..	: U L U L

	nop			# .. .. .. E
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	beq $0,$zerolength	# U  .. .. ..	: U L U L

/* We know we have at least one trip through the byte loop */
	EXI ( ldbu $2,0($17) )	# .. .. .. L	: No loads in the same quad
	addq $16,1,$16		# .. .. E  ..	: as the store (Section 3.8 in CWG)
	nop			# .. E  .. ..	:
	br $31, $dirtyentry	# L0 .. .. ..	: L U U L
/* Do the trailing byte loop load, then hop into the store part of the loop */

/*
 * A minimum of (33 - 7) bytes to do a quad at a time.
 * Based upon the usage context, it's worth the effort to unroll this loop
 * $0 - number of bytes to be moved
 * $4 - number of bytes to move as quadwords
 * $16 is current destination address
 * $17 is current source address
 */
$quadaligned:
	subq	$4, 32, $2	# .. .. .. E	: do not unroll for small stuff
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	blt	$2, $onequad	# U  .. .. ..	: U L U L

/*
 * There is a significant assumption here that the source and destination
 * addresses differ by more than 32 bytes.  In this particular case, a
 * sparsity of registers further bounds this to be a minimum of 8 bytes.
 * But if this isn't met, then the output result will be incorrect.
 * Furthermore, due to a lack of available registers, we really can't
 * unroll this to be an 8x loop (which would enable us to use the wh64
 * instruction memory hint instruction).
 */
$unroll4:
	EXI( ldq $1,0($17) )	# .. .. .. L
	EXI( ldq $2,8($17) )	# .. .. L  ..
	subq	$4,32,$4	# .. E  .. ..
	nop			# E  .. .. ..	: U U L L

	addq	$17,16,$17	# .. .. .. E
	EXO( stq $1,0($16) )	# .. .. L  ..
	EXO( stq $2,8($16) )	# .. L  .. ..
	subq	$0,16,$0	# E  .. .. ..	: U L L U

	addq	$16,16,$16	# .. .. .. E
	EXI( ldq $1,0($17) )	# .. .. L  ..
	EXI( ldq $2,8($17) )	# .. L  .. ..
	subq	$4, 32, $3	# E  .. .. ..	: U U L L : is there enough for another trip?

	EXO( stq $1,0($16) )	# .. .. .. L
	EXO( stq $2,8($16) )	# .. .. L  ..
	subq	$0,16,$0	# .. E  .. ..
	addq	$17,16,$17	# E  .. .. ..	: U L L U

	nop			# .. .. .. E
	nop			# .. .. E  ..
	addq	$16,16,$16	# .. E  .. ..
	bgt	$3,$unroll4	# U  .. .. ..	: U L U L

	nop
	nop
	nop
	beq	$4, $noquads

$onequad:
	EXI( ldq $1,0($17) )
	subq	$4,8,$4
	addq	$17,8,$17
	nop

	EXO( stq $1,0($16) )
	subq	$0,8,$0
	addq	$16,8,$16
	bne	$4,$onequad

$noquads:
	nop
	nop
	nop
	beq $0,$zerolength

/*
 * For small copies (or the tail of a larger copy), do a very simple byte loop.
 * There's no point in doing a lot of complex alignment calculations to try to
 * to quadword stuff for a small amount of data.
 *	$0 - remaining number of bytes left to copy
 *	$16 - current dest addr
 *	$17 - current source addr
 */

$onebyteloop:
	EXI ( ldbu $2,0($17) )	# .. .. .. L	: No loads in the same quad
	addq $16,1,$16		# .. .. E  ..	: as the store (Section 3.8 in CWG)
	nop			# .. E  .. ..	:
	nop			# E  .. .. ..	: U L U L

$dirtyentry:
/*
 * the -1 is to compensate for the inc($16) done in a previous quadpack
 * which allows us zero dependencies within either quadpack in the loop
 */
	EXO ( stb $2,-1($16) )	# .. .. .. L	:
	addq $17,1,$17		# .. .. E  ..	: quadpack as the load
	subq $0,1,$0		# .. E  .. ..	: change count _after_ copy
	bgt $0,$onebyteloop	# U  .. .. ..	: U L U L

$zerolength:
$exitin:
$exitout:			# Destination for exception recovery(?)
	nop			# .. .. .. E
	nop			# .. .. E  ..
	nop			# .. E  .. ..
	ret $31,($26),1		# L0 .. .. ..	: L U L U

	.end __copy_user
	EXPORT_SYMBOL(__copy_user)
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	/*
	3	* arch/alpha/lib/ev6-copy_user.S
	4	*
	5	* 21264 version contributed by Rick Gorton <rick.gorton@alpha-processor.com>
	6	*
	7	* Copy to/from user space, handling exceptions as we go.. This
	8	* isn't exactly pretty.
	9	*
	10	* This is essentially the same as "memcpy()", but with a few twists.
	11	* Notably, we have to make sure that $0 is always up-to-date and
	12	* contains the right "bytes left to copy" value (and that it is updated
	13	* only _after_ a successful copy). There is also some rather minor
	14	* exception setup stuff..
	15	*
1da177e4 LT	16	* Much of the information about 21264 scheduling/coding comes from:
	17	* Compiler Writer's Guide for the Alpha 21264
	18	* abbreviated as 'CWG' in other comments here
	19	* ftp.digital.com/pub/Digital/info/semiconductor/literature/dsc-library.html
	20	* Scheduling notation:
	21	* E - either cluster
	22	* U - upper subcluster; U0 - subcluster U0; U1 - subcluster U1
	23	* L - lower subcluster; L0 - subcluster L0; L1 - subcluster L1
	24	*/
	25
00fc0e0d	26	#include <asm/export.h>
1da177e4 LT	27	/* Allow an exception for an insn; exit if we get one. */
	28	#define EXI(x,y...) \
	29	99: x,##y; \
	30	.section __ex_table,"a"; \
	31	.long 99b - .; \
	32	lda $31, $exitin-99b($31); \
	33	.previous
	34
	35	#define EXO(x,y...) \
	36	99: x,##y; \
	37	.section __ex_table,"a"; \
	38	.long 99b - .; \
	39	lda $31, $exitout-99b($31); \
	40	.previous
	41
	42	.set noat
	43	.align 4
	44	.globl __copy_user
	45	.ent __copy_user
	46	# Pipeline info: Slotting & Comments
	47	__copy_user:
	48	.prologue 0
4606f68f RH	49	mov $18, $0 # .. .. .. E
	50	subq $18, 32, $1 # .. .. E. .. : Is this going to be a small copy?
	51	nop # .. E .. ..
	52	beq $18, $zerolength # U .. .. .. : U L U L
1da177e4	53
85250231	54	and $16,7,$3 # .. .. .. E : is leading dest misalignment
1da177e4 LT	55	ble $1, $onebyteloop # .. .. U .. : 1st branch : small amount of data
	56	beq $3, $destaligned # .. U .. .. : 2nd (one cycle fetcher stall)
	57	subq $3, 8, $3 # E .. .. .. : L U U L : trip counter
	58	/*
	59	* The fetcher stall also hides the 1 cycle cross-cluster stall for $3 (L --> U)
	60	* This loop aligns the destination a byte at a time
	61	* We know we have at least one trip through this loop
	62	*/
	63	$aligndest:
85250231 AV	64	EXI( ldbu $1,0($17) ) # .. .. .. L : Keep loads separate from stores
85250231 AV	65	addq $16,1,$16 # .. .. E .. : Section 3.8 in the CWG
1da177e4 LT	66	addq $3,1,$3 # .. E .. .. :
	67	nop # E .. .. .. : U L U L
	68
	69	/*
85250231	70	* the -1 is to compensate for the inc($16) done in a previous quadpack
1da177e4 LT	71	* which allows us zero dependencies within either quadpack in the loop
1da177e4 LT	72	*/
85250231 AV	73	EXO( stb $1,-1($16) ) # .. .. .. L :
85250231 AV	74	addq $17,1,$17 # .. .. E .. : Section 3.8 in the CWG
1da177e4 LT	75	subq $0,1,$0 # .. E .. .. :
	76	bne $3, $aligndest # U .. .. .. : U L U L
	77
	78	/*
	79	* If we fell through into here, we have a minimum of 33 - 7 bytes
	80	* If we arrived via branch, we have a minimum of 32 bytes
	81	*/
	82	$destaligned:
85250231	83	and $17,7,$1 # .. .. .. E : Check _current_ source alignment
1da177e4	84	bic $0,7,$4 # .. .. E .. : number bytes as a quadword loop
85250231	85	EXI( ldq_u $3,0($17) ) # .. L .. .. : Forward fetch for fallthrough code
1da177e4 LT	86	beq $1,$quadaligned # U .. .. .. : U L U L
	87
	88	/*
85250231	89	* In the worst case, we've just executed an ldq_u here from 0($17)
1da177e4 LT	90	* and we'll repeat it once if we take the branch
	91	*/
	92
	93	/* Misaligned quadword loop - not unrolled. Leave it that way. */
	94	$misquad:
85250231	95	EXI( ldq_u $2,8($17) ) # .. .. .. L :
1da177e4	96	subq $4,8,$4 # .. .. E .. :
85250231 AV	97	extql $3,$17,$3 # .. U .. .. :
85250231 AV	98	extqh $2,$17,$1 # U .. .. .. : U U L L
1da177e4 LT	99
1da177e4 LT	100	bis $3,$1,$1 # .. .. .. E :
85250231 AV	101	EXO( stq $1,0($16) ) # .. .. L .. :
85250231 AV	102	addq $17,8,$17 # .. E .. .. :
1da177e4 LT	103	subq $0,8,$0 # E .. .. .. : U L L U
1da177e4 LT	104
85250231	105	addq $16,8,$16 # .. .. .. E :
1da177e4 LT	106	bis $2,$2,$3 # .. .. E .. :
	107	nop # .. E .. .. :
	108	bne $4,$misquad # U .. .. .. : U L U L
	109
	110	nop # .. .. .. E
	111	nop # .. .. E ..
	112	nop # .. E .. ..
	113	beq $0,$zerolength # U .. .. .. : U L U L
	114
	115	/* We know we have at least one trip through the byte loop */
85250231 AV	116	EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
85250231 AV	117	addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
1da177e4 LT	118	nop # .. E .. .. :
	119	br $31, $dirtyentry # L0 .. .. .. : L U U L
	120	/* Do the trailing byte loop load, then hop into the store part of the loop */
	121
	122	/*
	123	* A minimum of (33 - 7) bytes to do a quad at a time.
	124	* Based upon the usage context, it's worth the effort to unroll this loop
	125	* $0 - number of bytes to be moved
	126	* $4 - number of bytes to move as quadwords
85250231 AV	127	* $16 is current destination address
85250231 AV	128	* $17 is current source address
1da177e4 LT	129	*/
	130	$quadaligned:
	131	subq $4, 32, $2 # .. .. .. E : do not unroll for small stuff
	132	nop # .. .. E ..
	133	nop # .. E .. ..
	134	blt $2, $onequad # U .. .. .. : U L U L
	135
	136	/*
	137	* There is a significant assumption here that the source and destination
	138	* addresses differ by more than 32 bytes. In this particular case, a
	139	* sparsity of registers further bounds this to be a minimum of 8 bytes.
	140	* But if this isn't met, then the output result will be incorrect.
	141	* Furthermore, due to a lack of available registers, we really can't
	142	* unroll this to be an 8x loop (which would enable us to use the wh64
	143	* instruction memory hint instruction).
	144	*/
	145	$unroll4:
85250231 AV	146	EXI( ldq $1,0($17) ) # .. .. .. L
85250231 AV	147	EXI( ldq $2,8($17) ) # .. .. L ..
1da177e4 LT	148	subq $4,32,$4 # .. E .. ..
	149	nop # E .. .. .. : U U L L
	150
85250231 AV	151	addq $17,16,$17 # .. .. .. E
	152	EXO( stq $1,0($16) ) # .. .. L ..
	153	EXO( stq $2,8($16) ) # .. L .. ..
1da177e4 LT	154	subq $0,16,$0 # E .. .. .. : U L L U
1da177e4 LT	155
85250231 AV	156	addq $16,16,$16 # .. .. .. E
	157	EXI( ldq $1,0($17) ) # .. .. L ..
	158	EXI( ldq $2,8($17) ) # .. L .. ..
1da177e4 LT	159	subq $4, 32, $3 # E .. .. .. : U U L L : is there enough for another trip?
1da177e4 LT	160
85250231 AV	161	EXO( stq $1,0($16) ) # .. .. .. L
85250231 AV	162	EXO( stq $2,8($16) ) # .. .. L ..
1da177e4	163	subq $0,16,$0 # .. E .. ..
85250231	164	addq $17,16,$17 # E .. .. .. : U L L U
1da177e4 LT	165
	166	nop # .. .. .. E
	167	nop # .. .. E ..
85250231	168	addq $16,16,$16 # .. E .. ..
1da177e4 LT	169	bgt $3,$unroll4 # U .. .. .. : U L U L
	170
	171	nop
	172	nop
	173	nop
	174	beq $4, $noquads
	175
	176	$onequad:
85250231	177	EXI( ldq $1,0($17) )
1da177e4	178	subq $4,8,$4
85250231	179	addq $17,8,$17
1da177e4 LT	180	nop
1da177e4 LT	181
85250231	182	EXO( stq $1,0($16) )
1da177e4	183	subq $0,8,$0
85250231	184	addq $16,8,$16
1da177e4 LT	185	bne $4,$onequad
	186
	187	$noquads:
	188	nop
	189	nop
	190	nop
	191	beq $0,$zerolength
	192
	193	/*
	194	* For small copies (or the tail of a larger copy), do a very simple byte loop.
	195	* There's no point in doing a lot of complex alignment calculations to try to
	196	* to quadword stuff for a small amount of data.
	197	* $0 - remaining number of bytes left to copy
85250231 AV	198	* $16 - current dest addr
85250231 AV	199	* $17 - current source addr
1da177e4 LT	200	*/
	201
	202	$onebyteloop:
85250231 AV	203	EXI ( ldbu $2,0($17) ) # .. .. .. L : No loads in the same quad
85250231 AV	204	addq $16,1,$16 # .. .. E .. : as the store (Section 3.8 in CWG)
1da177e4 LT	205	nop # .. E .. .. :
	206	nop # E .. .. .. : U L U L
	207
	208	$dirtyentry:
	209	/*
85250231	210	* the -1 is to compensate for the inc($16) done in a previous quadpack
1da177e4 LT	211	* which allows us zero dependencies within either quadpack in the loop
1da177e4 LT	212	*/
85250231 AV	213	EXO ( stb $2,-1($16) ) # .. .. .. L :
85250231 AV	214	addq $17,1,$17 # .. .. E .. : quadpack as the load
1da177e4 LT	215	subq $0,1,$0 # .. E .. .. : change count _after_ copy
	216	bgt $0,$onebyteloop # U .. .. .. : U L U L
	217
	218	$zerolength:
085354f9	219	$exitin:
1da177e4 LT	220	$exitout: # Destination for exception recovery(?)
	221	nop # .. .. .. E
	222	nop # .. .. E ..
	223	nop # .. E .. ..
85250231	224	ret $31,($26),1 # L0 .. .. .. : L U L U
1da177e4	225
1da177e4	226	.end __copy_user
00fc0e0d	227	EXPORT_SYMBOL(__copy_user)