[linux-2.6-block.git] / arch / arm / lib / memmove.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 *  linux/arch/arm/lib/memmove.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	(C) MontaVista Software Inc.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/unwind.h>

		.text

/*
 * Prototype: void *memmove(void *dest, const void *src, size_t n);
 *
 * Note:
 *
 * If the memory regions don't overlap, we simply branch to memcpy which is
 * normally a bit faster. Otherwise the copy is done going downwards.  This
 * is a transposition of the code from copy_template.S but with the copy
 * occurring in the opposite direction.
 */

ENTRY(__memmove)
WEAK(memmove)
	UNWIND(	.fnstart			)

		subs	ip, r0, r1
		cmphi	r2, ip
		bls	__memcpy
	UNWIND(	.fnend				)

	UNWIND(	.fnstart			)
	UNWIND(	.save	{r0, r4, fpreg, lr}	)
		stmfd	sp!, {r0, r4, UNWIND(fpreg,) lr}
	UNWIND(	.setfp	fpreg, sp		)
	UNWIND(	mov	fpreg, sp		)
		add	r1, r1, r2
		add	r0, r0, r2
		subs	r2, r2, #4
		blt	8f
		ands	ip, r0, #3
	PLD(	pld	[r1, #-4]		)
		bne	9f
		ands	ip, r1, #3
		bne	10f

1:		subs	r2, r2, #(28)
		stmfd	sp!, {r5, r6, r8, r9}
		blt	5f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
	CALGN(	bcs	2f			)
	CALGN(	adr	r4, 6f			)
	CALGN(	subs	r2, r2, ip		)  @ C is set here
	CALGN(	rsb	ip, ip, #32		)
	CALGN(	add	pc, r4, ip		)

	PLD(	pld	[r1, #-4]		)
2:	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	4f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

3:	PLD(	pld	[r1, #-128]		)
4:		ldmdb	r1!, {r3, r4, r5, r6, r8, r9, ip, lr}
		subs	r2, r2, #32
		stmdb	r0!, {r3, r4, r5, r6, r8, r9, ip, lr}
		bge	3b
	PLD(	cmn	r2, #96			)
	PLD(	bge	4b			)

5:		ands	ip, r2, #28
		rsb	ip, ip, #32
		addne	pc, pc, ip		@ C is always clear here
		b	7f
6:		W(nop)
		W(ldr)	r3, [r1, #-4]!
		W(ldr)	r4, [r1, #-4]!
		W(ldr)	r5, [r1, #-4]!
		W(ldr)	r6, [r1, #-4]!
		W(ldr)	r8, [r1, #-4]!
		W(ldr)	r9, [r1, #-4]!
		W(ldr)	lr, [r1, #-4]!

		add	pc, pc, ip
		nop
		W(nop)
		W(str)	r3, [r0, #-4]!
		W(str)	r4, [r0, #-4]!
		W(str)	r5, [r0, #-4]!
		W(str)	r6, [r0, #-4]!
		W(str)	r8, [r0, #-4]!
		W(str)	r9, [r0, #-4]!
		W(str)	lr, [r0, #-4]!

	CALGN(	bcs	2b			)

7:		ldmfd	sp!, {r5, r6, r8, r9}

8:		movs	r2, r2, lsl #31
		ldrbne	r3, [r1, #-1]!
		ldrbcs	r4, [r1, #-1]!
		ldrbcs	ip, [r1, #-1]
		strbne	r3, [r0, #-1]!
		strbcs	r4, [r0, #-1]!
		strbcs	ip, [r0, #-1]
		ldmfd	sp!, {r0, r4, UNWIND(fpreg,) pc}

9:		cmp	ip, #2
		ldrbgt	r3, [r1, #-1]!
		ldrbge	r4, [r1, #-1]!
		ldrb	lr, [r1, #-1]!
		strbgt	r3, [r0, #-1]!
		strbge	r4, [r0, #-1]!
		subs	r2, r2, ip
		strb	lr, [r0, #-1]!
		blt	8b
		ands	ip, r1, #3
		beq	1b

10:		bic	r1, r1, #3
		cmp	ip, #2
		ldr	r3, [r1, #0]
		beq	17f
		blt	18f


		.macro	backward_copy_shift push pull

		subs	r2, r2, #28
		blt	14f

	CALGN(	ands	ip, r0, #31		)
	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
	CALGN(	subcc	r2, r2, ip		)
	CALGN(	bcc	15f			)

11:		stmfd	sp!, {r5, r6, r8 - r10}

	PLD(	pld	[r1, #-4]		)
	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	13f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

12:	PLD(	pld	[r1, #-128]		)
13:		ldmdb   r1!, {r8, r9, r10, ip}
		mov     lr, r3, lspush #\push
		subs    r2, r2, #32
		ldmdb   r1!, {r3, r4, r5, r6}
		orr     lr, lr, ip, lspull #\pull
		mov     ip, ip, lspush #\push
		orr     ip, ip, r10, lspull #\pull
		mov     r10, r10, lspush #\push
		orr     r10, r10, r9, lspull #\pull
		mov     r9, r9, lspush #\push
		orr     r9, r9, r8, lspull #\pull
		mov     r8, r8, lspush #\push
		orr     r8, r8, r6, lspull #\pull
		mov     r6, r6, lspush #\push
		orr     r6, r6, r5, lspull #\pull
		mov     r5, r5, lspush #\push
		orr     r5, r5, r4, lspull #\pull
		mov     r4, r4, lspush #\push
		orr     r4, r4, r3, lspull #\pull
		stmdb   r0!, {r4 - r6, r8 - r10, ip, lr}
		bge	12b
	PLD(	cmn	r2, #96			)
	PLD(	bge	13b			)

		ldmfd	sp!, {r5, r6, r8 - r10}

14:		ands	ip, r2, #28
		beq	16f

15:		mov     lr, r3, lspush #\push
		ldr	r3, [r1, #-4]!
		subs	ip, ip, #4
		orr	lr, lr, r3, lspull #\pull
		str	lr, [r0, #-4]!
		bgt	15b
	CALGN(	cmp	r2, #0			)
	CALGN(	bge	11b			)

16:		add	r1, r1, #(\pull / 8)
		b	8b

		.endm


		backward_copy_shift	push=8	pull=24

17:		backward_copy_shift	push=16	pull=16

18:		backward_copy_shift	push=24	pull=8

	UNWIND(	.fnend				)
ENDPROC(memmove)
ENDPROC(__memmove)
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
75494230 NP	2	/*
	3	* linux/arch/arm/lib/memmove.S
	4	*
	5	* Author: Nicolas Pitre
	6	* Created: Sep 28, 2005
	7	* Copyright: (C) MontaVista Software Inc.
75494230 NP	8	*/
	9
	10	#include <linux/linkage.h>
	11	#include <asm/assembler.h>
207a6cb0	12	#include <asm/unwind.h>
75494230	13
75494230 NP	14	.text
	15
	16	/*
	17	* Prototype: void memmove(void dest, const void *src, size_t n);
	18	*
	19	* Note:
	20	*
	21	* If the memory regions don't overlap, we simply branch to memcpy which is
	22	* normally a bit faster. Otherwise the copy is done going downwards. This
	23	* is a transposition of the code from copy_template.S but with the copy
	24	* occurring in the opposite direction.
	25	*/
	26
d6d51a96	27	ENTRY(__memmove)
735e8d93	28	WEAK(memmove)
207a6cb0	29	UNWIND( .fnstart )
75494230 NP	30
	31	subs ip, r0, r1
	32	cmphi r2, ip
d6d51a96	33	bls __memcpy
207a6cb0 LY	34	UNWIND( .fnend )
	35
	36	UNWIND( .fnstart )
ccb81601 AB	37	UNWIND( .save {r0, r4, fpreg, lr} )
	38	stmfd sp!, {r0, r4, UNWIND(fpreg,) lr}
	39	UNWIND( .setfp fpreg, sp )
	40	UNWIND( mov fpreg, sp )
75494230 NP	41	add r1, r1, r2
	42	add r0, r0, r2
	43	subs r2, r2, #4
	44	blt 8f
	45	ands ip, r0, #3
	46	PLD( pld [r1, #-4] )
	47	bne 9f
	48	ands ip, r1, #3
	49	bne 10f
	50
	51	1: subs r2, r2, #(28)
ccb81601	52	stmfd sp!, {r5, r6, r8, r9}
75494230 NP	53	blt 5f
75494230 NP	54
2239aff6	55	CALGN( ands ip, r0, #31 )
e44fc388	56	CALGN( sbcsne r4, ip, r2 ) @ C is always set here
75494230 NP	57	CALGN( bcs 2f )
	58	CALGN( adr r4, 6f )
	59	CALGN( subs r2, r2, ip ) @ C is set here
4c4925c1	60	CALGN( rsb ip, ip, #32 )
75494230 NP	61	CALGN( add pc, r4, ip )
	62
	63	PLD( pld [r1, #-4] )
	64	2: PLD( subs r2, r2, #96 )
	65	PLD( pld [r1, #-32] )
	66	PLD( blt 4f )
	67	PLD( pld [r1, #-64] )
	68	PLD( pld [r1, #-96] )
	69
	70	3: PLD( pld [r1, #-128] )
ccb81601	71	4: ldmdb r1!, {r3, r4, r5, r6, r8, r9, ip, lr}
75494230	72	subs r2, r2, #32
ccb81601	73	stmdb r0!, {r3, r4, r5, r6, r8, r9, ip, lr}
75494230 NP	74	bge 3b
	75	PLD( cmn r2, #96 )
	76	PLD( bge 4b )
	77
	78	5: ands ip, r2, #28
	79	rsb ip, ip, #32
	80	addne pc, pc, ip @ C is always clear here
	81	b 7f
fd522a8d	82	6: W(nop)
8b592783 CM	83	W(ldr) r3, [r1, #-4]!
	84	W(ldr) r4, [r1, #-4]!
	85	W(ldr) r5, [r1, #-4]!
	86	W(ldr) r6, [r1, #-4]!
8b592783	87	W(ldr) r8, [r1, #-4]!
ccb81601	88	W(ldr) r9, [r1, #-4]!
8b592783	89	W(ldr) lr, [r1, #-4]!
75494230 NP	90
	91	add pc, pc, ip
	92	nop
fd522a8d	93	W(nop)
8b592783 CM	94	W(str) r3, [r0, #-4]!
	95	W(str) r4, [r0, #-4]!
	96	W(str) r5, [r0, #-4]!
	97	W(str) r6, [r0, #-4]!
8b592783	98	W(str) r8, [r0, #-4]!
ccb81601	99	W(str) r9, [r0, #-4]!
8b592783	100	W(str) lr, [r0, #-4]!
75494230 NP	101
	102	CALGN( bcs 2b )
	103
ccb81601	104	7: ldmfd sp!, {r5, r6, r8, r9}
75494230 NP	105
75494230 NP	106	8: movs r2, r2, lsl #31
e44fc388 SA	107	ldrbne r3, [r1, #-1]!
	108	ldrbcs r4, [r1, #-1]!
	109	ldrbcs ip, [r1, #-1]
	110	strbne r3, [r0, #-1]!
	111	strbcs r4, [r0, #-1]!
	112	strbcs ip, [r0, #-1]
ccb81601	113	ldmfd sp!, {r0, r4, UNWIND(fpreg,) pc}
75494230 NP	114
75494230 NP	115	9: cmp ip, #2
e44fc388 SA	116	ldrbgt r3, [r1, #-1]!
e44fc388 SA	117	ldrbge r4, [r1, #-1]!
75494230	118	ldrb lr, [r1, #-1]!
e44fc388 SA	119	strbgt r3, [r0, #-1]!
e44fc388 SA	120	strbge r4, [r0, #-1]!
75494230 NP	121	subs r2, r2, ip
	122	strb lr, [r0, #-1]!
	123	blt 8b
	124	ands ip, r1, #3
	125	beq 1b
	126
	127	10: bic r1, r1, #3
	128	cmp ip, #2
	129	ldr r3, [r1, #0]
	130	beq 17f
	131	blt 18f
	132
	133
	134	.macro backward_copy_shift push pull
	135
	136	subs r2, r2, #28
	137	blt 14f
	138
2239aff6	139	CALGN( ands ip, r0, #31 )
e44fc388	140	CALGN( sbcsne r4, ip, r2 ) @ C is always set here
75494230 NP	141	CALGN( subcc r2, r2, ip )
	142	CALGN( bcc 15f )
	143
ccb81601	144	11: stmfd sp!, {r5, r6, r8 - r10}
75494230 NP	145
	146	PLD( pld [r1, #-4] )
	147	PLD( subs r2, r2, #96 )
	148	PLD( pld [r1, #-32] )
	149	PLD( blt 13f )
	150	PLD( pld [r1, #-64] )
	151	PLD( pld [r1, #-96] )
	152
	153	12: PLD( pld [r1, #-128] )
ccb81601	154	13: ldmdb r1!, {r8, r9, r10, ip}
d98b90ea	155	mov lr, r3, lspush #\push
75494230 NP	156	subs r2, r2, #32
75494230 NP	157	ldmdb r1!, {r3, r4, r5, r6}
d98b90ea VK	158	orr lr, lr, ip, lspull #\pull
d98b90ea VK	159	mov ip, ip, lspush #\push
ccb81601 AB	160	orr ip, ip, r10, lspull #\pull
	161	mov r10, r10, lspush #\push
	162	orr r10, r10, r9, lspull #\pull
d98b90ea VK	163	mov r9, r9, lspush #\push
	164	orr r9, r9, r8, lspull #\pull
	165	mov r8, r8, lspush #\push
ccb81601	166	orr r8, r8, r6, lspull #\pull
d98b90ea VK	167	mov r6, r6, lspush #\push
	168	orr r6, r6, r5, lspull #\pull
	169	mov r5, r5, lspush #\push
	170	orr r5, r5, r4, lspull #\pull
	171	mov r4, r4, lspush #\push
	172	orr r4, r4, r3, lspull #\pull
ccb81601	173	stmdb r0!, {r4 - r6, r8 - r10, ip, lr}
75494230 NP	174	bge 12b
	175	PLD( cmn r2, #96 )
	176	PLD( bge 13b )
	177
ccb81601	178	ldmfd sp!, {r5, r6, r8 - r10}
75494230 NP	179
	180	14: ands ip, r2, #28
	181	beq 16f
	182
d98b90ea	183	15: mov lr, r3, lspush #\push
75494230 NP	184	ldr r3, [r1, #-4]!
75494230 NP	185	subs ip, ip, #4
d98b90ea	186	orr lr, lr, r3, lspull #\pull
75494230 NP	187	str lr, [r0, #-4]!
	188	bgt 15b
	189	CALGN( cmp r2, #0 )
	190	CALGN( bge 11b )
	191
	192	16: add r1, r1, #(\pull / 8)
	193	b 8b
	194
	195	.endm
	196
	197
	198	backward_copy_shift push=8 pull=24
	199
	200	17: backward_copy_shift push=16 pull=16
	201
	202	18: backward_copy_shift push=24 pull=8
	203
ccb81601	204	UNWIND( .fnend )
93ed3970	205	ENDPROC(memmove)
d6d51a96	206	ENDPROC(__memmove)