[linux-2.6-block.git] / arch / powerpc / lib / memcpy_64.S

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (C) 2002 Paul Mackerras, IBM Corp.
 */
#include <asm/processor.h>
#include <asm/ppc_asm.h>
#include <asm/export.h>
#include <asm/asm-compat.h>
#include <asm/feature-fixups.h>
#include <asm/kasan.h>

#ifndef SELFTEST_CASE
/* For big-endian, 0 == most CPUs, 1 == POWER6, 2 == Cell */
#define SELFTEST_CASE	0
#endif

	.align	7
_GLOBAL_TOC_KASAN(memcpy)
BEGIN_FTR_SECTION
#ifdef __LITTLE_ENDIAN__
	cmpdi	cr7,r5,0
#else
	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* save destination pointer for return value */
#endif
FTR_SECTION_ELSE
#ifdef CONFIG_PPC_BOOK3S_64
	b	memcpy_power7
#endif
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
#ifdef __LITTLE_ENDIAN__
	/* dumb little-endian memcpy that will get replaced at runtime */
	addi r9,r3,-1
	addi r4,r4,-1
	beqlr cr7
	mtctr r5
1:	lbzu r10,1(r4)
	stbu r10,1(r9)
	bdnz 1b
	blr
#else
	PPC_MTOCRF(0x01,r5)
	cmpldi	cr1,r5,16
	neg	r6,r3		# LS 3 bits = # bytes to 8-byte dest bdry
	andi.	r6,r6,7
	dcbt	0,r4
	blt	cr1,.Lshort_copy
/* Below we want to nop out the bne if we're on a CPU that has the
   CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
   cleared.
   At the time of writing the only CPU that has this combination of bits
   set is Power6. */
test_feature = (SELFTEST_CASE == 1)
BEGIN_FTR_SECTION
	nop
FTR_SECTION_ELSE
	bne	.Ldst_unaligned
ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
                    CPU_FTR_UNALIGNED_LD_STD)
.Ldst_aligned:
	addi	r3,r3,-16
test_feature = (SELFTEST_CASE == 0)
BEGIN_FTR_SECTION
	andi.	r0,r4,7
	bne	.Lsrc_unaligned
END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
	srdi	r7,r5,4
	ld	r9,0(r4)
	addi	r4,r4,-8
	mtctr	r7
	andi.	r5,r5,7
	bf	cr7*4+0,2f
	addi	r3,r3,8
	addi	r4,r4,8
	mr	r8,r9
	blt	cr1,3f
1:	ld	r9,8(r4)
	std	r8,8(r3)
2:	ldu	r8,16(r4)
	stdu	r9,16(r3)
	bdnz	1b
3:	std	r8,8(r3)
	beq	3f
	addi	r3,r3,16
.Ldo_tail:
	bf	cr7*4+1,1f
	lwz	r9,8(r4)
	addi	r4,r4,4
	stw	r9,0(r3)
	addi	r3,r3,4
1:	bf	cr7*4+2,2f
	lhz	r9,8(r4)
	addi	r4,r4,2
	sth	r9,0(r3)
	addi	r3,r3,2
2:	bf	cr7*4+3,3f
	lbz	r9,8(r4)
	stb	r9,0(r3)
3:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr

.Lsrc_unaligned:
	srdi	r6,r5,3
	addi	r5,r5,-16
	subf	r4,r0,r4
	srdi	r7,r5,4
	sldi	r10,r0,3
	cmpdi	cr6,r6,3
	andi.	r5,r5,7
	mtctr	r7
	subfic	r11,r10,64
	add	r5,r5,r0

	bt	cr7*4+0,0f

	ld	r9,0(r4)	# 3+2n loads, 2+2n stores
	ld	r0,8(r4)
	sld	r6,r9,r10
	ldu	r9,16(r4)
	srd	r7,r0,r11
	sld	r8,r0,r10
	or	r7,r7,r6
	blt	cr6,4f
	ld	r0,8(r4)
	# s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
	b	2f

0:	ld	r0,0(r4)	# 4+2n loads, 3+2n stores
	ldu	r9,8(r4)
	sld	r8,r0,r10
	addi	r3,r3,-8
	blt	cr6,5f
	ld	r0,8(r4)
	srd	r12,r9,r11
	sld	r6,r9,r10
	ldu	r9,16(r4)
	or	r12,r8,r12
	srd	r7,r0,r11
	sld	r8,r0,r10
	addi	r3,r3,16
	beq	cr6,3f

	# d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
1:	or	r7,r7,r6
	ld	r0,8(r4)
	std	r12,8(r3)
2:	srd	r12,r9,r11
	sld	r6,r9,r10
	ldu	r9,16(r4)
	or	r12,r8,r12
	stdu	r7,16(r3)
	srd	r7,r0,r11
	sld	r8,r0,r10
	bdnz	1b

3:	std	r12,8(r3)
	or	r7,r7,r6
4:	std	r7,16(r3)
5:	srd	r12,r9,r11
	or	r12,r8,r12
	std	r12,24(r3)
	beq	4f
	cmpwi	cr1,r5,8
	addi	r3,r3,32
	sld	r9,r9,r10
	ble	cr1,6f
	ld	r0,8(r4)
	srd	r7,r0,r11
	or	r9,r7,r9
6:
	bf	cr7*4+1,1f
	rotldi	r9,r9,32
	stw	r9,0(r3)
	addi	r3,r3,4
1:	bf	cr7*4+2,2f
	rotldi	r9,r9,16
	sth	r9,0(r3)
	addi	r3,r3,2
2:	bf	cr7*4+3,3f
	rotldi	r9,r9,8
	stb	r9,0(r3)
3:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr

.Ldst_unaligned:
	PPC_MTOCRF(0x01,r6)		# put #bytes to 8B bdry into cr7
	subf	r5,r6,r5
	li	r7,0
	cmpldi	cr1,r5,16
	bf	cr7*4+3,1f
	lbz	r0,0(r4)
	stb	r0,0(r3)
	addi	r7,r7,1
1:	bf	cr7*4+2,2f
	lhzx	r0,r7,r4
	sthx	r0,r7,r3
	addi	r7,r7,2
2:	bf	cr7*4+1,3f
	lwzx	r0,r7,r4
	stwx	r0,r7,r3
3:	PPC_MTOCRF(0x01,r5)
	add	r4,r6,r4
	add	r3,r6,r3
	b	.Ldst_aligned

.Lshort_copy:
	bf	cr7*4+0,1f
	lwz	r0,0(r4)
	lwz	r9,4(r4)
	addi	r4,r4,8
	stw	r0,0(r3)
	stw	r9,4(r3)
	addi	r3,r3,8
1:	bf	cr7*4+1,2f
	lwz	r0,0(r4)
	addi	r4,r4,4
	stw	r0,0(r3)
	addi	r3,r3,4
2:	bf	cr7*4+2,3f
	lhz	r0,0(r4)
	addi	r4,r4,2
	sth	r0,0(r3)
	addi	r3,r3,2
3:	bf	cr7*4+3,4f
	lbz	r0,0(r4)
	stb	r0,0(r3)
4:	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)	/* return dest pointer */
	blr
#endif
EXPORT_SYMBOL(memcpy)
EXPORT_SYMBOL_KASAN(memcpy)
Commit	Line	Data
2874c5fd	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
14cf11af	2	/*
14cf11af	3	* Copyright (C) 2002 Paul Mackerras, IBM Corp.
14cf11af PM	4	*/
	5	#include <asm/processor.h>
	6	#include <asm/ppc_asm.h>
9445aa1a	7	#include <asm/export.h>
ec0c464c	8	#include <asm/asm-compat.h>
2c86cd18	9	#include <asm/feature-fixups.h>
26deb043	10	#include <asm/kasan.h>
14cf11af	11
98c45f51 PM	12	#ifndef SELFTEST_CASE
	13	/* For big-endian, 0 == most CPUs, 1 == POWER6, 2 == Cell */
	14	#define SELFTEST_CASE 0
	15	#endif
	16
14cf11af	17	.align 7
26deb043	18	_GLOBAL_TOC_KASAN(memcpy)
b3f271e8	19	BEGIN_FTR_SECTION
00f554fa PB	20	#ifdef __LITTLE_ENDIAN__
	21	cmpdi cr7,r5,0
	22	#else
752a6422	23	std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* save destination pointer for return value */
00f554fa	24	#endif
b3f271e8	25	FTR_SECTION_ELSE
15a3204d	26	#ifdef CONFIG_PPC_BOOK3S_64
b3f271e8	27	b memcpy_power7
22d651dc	28	#endif
b3f271e8	29	ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
00f554fa PB	30	#ifdef __LITTLE_ENDIAN__
	31	/* dumb little-endian memcpy that will get replaced at runtime */
	32	addi r9,r3,-1
	33	addi r4,r4,-1
	34	beqlr cr7
	35	mtctr r5
	36	1: lbzu r10,1(r4)
	37	stbu r10,1(r9)
	38	bdnz 1b
	39	blr
	40	#else
86e32fdc	41	PPC_MTOCRF(0x01,r5)
14cf11af PM	42	cmpldi cr1,r5,16
	43	neg r6,r3 # LS 3 bits = # bytes to 8-byte dest bdry
	44	andi. r6,r6,7
	45	dcbt 0,r4
	46	blt cr1,.Lshort_copy
25d6e2d7 MN	47	/* Below we want to nop out the bne if we're on a CPU that has the
	48	CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
	49	cleared.
	50	At the time of writing the only CPU that has this combination of bits
	51	set is Power6. */
98c45f51	52	test_feature = (SELFTEST_CASE == 1)
25d6e2d7 MN	53	BEGIN_FTR_SECTION
	54	nop
	55	FTR_SECTION_ELSE
14cf11af	56	bne .Ldst_unaligned
25d6e2d7 MN	57	ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD \| CPU_FTR_CP_USE_DCBTZ, \
25d6e2d7 MN	58	CPU_FTR_UNALIGNED_LD_STD)
14cf11af	59	.Ldst_aligned:
14cf11af	60	addi r3,r3,-16
98c45f51	61	test_feature = (SELFTEST_CASE == 0)
25d6e2d7 MN	62	BEGIN_FTR_SECTION
25d6e2d7 MN	63	andi. r0,r4,7
14cf11af	64	bne .Lsrc_unaligned
25d6e2d7	65	END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
14cf11af PM	66	srdi r7,r5,4
	67	ld r9,0(r4)
	68	addi r4,r4,-8
	69	mtctr r7
	70	andi. r5,r5,7
	71	bf cr7*4+0,2f
	72	addi r3,r3,8
	73	addi r4,r4,8
	74	mr r8,r9
	75	blt cr1,3f
	76	1: ld r9,8(r4)
	77	std r8,8(r3)
	78	2: ldu r8,16(r4)
	79	stdu r9,16(r3)
	80	bdnz 1b
	81	3: std r8,8(r3)
d0027bf0	82	beq 3f
14cf11af	83	addi r3,r3,16
14cf11af PM	84	.Ldo_tail:
14cf11af PM	85	bf cr7*4+1,1f
e423b9ec MN	86	lwz r9,8(r4)
e423b9ec MN	87	addi r4,r4,4
14cf11af PM	88	stw r9,0(r3)
	89	addi r3,r3,4
	90	1: bf cr7*4+2,2f
e423b9ec MN	91	lhz r9,8(r4)
e423b9ec MN	92	addi r4,r4,2
14cf11af PM	93	sth r9,0(r3)
	94	addi r3,r3,2
	95	2: bf cr7*4+3,3f
e423b9ec	96	lbz r9,8(r4)
14cf11af	97	stb r9,0(r3)
752a6422	98	3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
d0027bf0	99	blr
14cf11af PM	100
	101	.Lsrc_unaligned:
	102	srdi r6,r5,3
	103	addi r5,r5,-16
	104	subf r4,r0,r4
	105	srdi r7,r5,4
	106	sldi r10,r0,3
	107	cmpdi cr6,r6,3
	108	andi. r5,r5,7
	109	mtctr r7
	110	subfic r11,r10,64
	111	add r5,r5,r0
	112
	113	bt cr7*4+0,0f
	114
	115	ld r9,0(r4) # 3+2n loads, 2+2n stores
	116	ld r0,8(r4)
	117	sld r6,r9,r10
	118	ldu r9,16(r4)
	119	srd r7,r0,r11
	120	sld r8,r0,r10
	121	or r7,r7,r6
	122	blt cr6,4f
	123	ld r0,8(r4)
	124	# s1<< in r8, d0=(s0<<\|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
	125	b 2f
	126
	127	0: ld r0,0(r4) # 4+2n loads, 3+2n stores
	128	ldu r9,8(r4)
	129	sld r8,r0,r10
	130	addi r3,r3,-8
	131	blt cr6,5f
	132	ld r0,8(r4)
	133	srd r12,r9,r11
	134	sld r6,r9,r10
	135	ldu r9,16(r4)
	136	or r12,r8,r12
	137	srd r7,r0,r11
	138	sld r8,r0,r10
	139	addi r3,r3,16
	140	beq cr6,3f
	141
	142	# d0=(s0<<\|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
	143	1: or r7,r7,r6
	144	ld r0,8(r4)
	145	std r12,8(r3)
	146	2: srd r12,r9,r11
	147	sld r6,r9,r10
	148	ldu r9,16(r4)
	149	or r12,r8,r12
	150	stdu r7,16(r3)
	151	srd r7,r0,r11
	152	sld r8,r0,r10
	153	bdnz 1b
	154
	155	3: std r12,8(r3)
	156	or r7,r7,r6
	157	4: std r7,16(r3)
	158	5: srd r12,r9,r11
	159	or r12,r8,r12
	160	std r12,24(r3)
d0027bf0	161	beq 4f
14cf11af PM	162	cmpwi cr1,r5,8
	163	addi r3,r3,32
	164	sld r9,r9,r10
e423b9ec	165	ble cr1,6f
14cf11af PM	166	ld r0,8(r4)
	167	srd r7,r0,r11
	168	or r9,r7,r9
e423b9ec MN	169	6:
	170	bf cr7*4+1,1f
	171	rotldi r9,r9,32
	172	stw r9,0(r3)
	173	addi r3,r3,4
	174	1: bf cr7*4+2,2f
	175	rotldi r9,r9,16
	176	sth r9,0(r3)
	177	addi r3,r3,2
	178	2: bf cr7*4+3,3f
	179	rotldi r9,r9,8
	180	stb r9,0(r3)
752a6422	181	3: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
e423b9ec	182	blr
14cf11af PM	183
14cf11af PM	184	.Ldst_unaligned:
86e32fdc	185	PPC_MTOCRF(0x01,r6) # put #bytes to 8B bdry into cr7
14cf11af PM	186	subf r5,r6,r5
14cf11af PM	187	li r7,0
25d6e2d7	188	cmpldi cr1,r5,16
14cf11af PM	189	bf cr7*4+3,1f
	190	lbz r0,0(r4)
	191	stb r0,0(r3)
	192	addi r7,r7,1
	193	1: bf cr7*4+2,2f
	194	lhzx r0,r7,r4
	195	sthx r0,r7,r3
	196	addi r7,r7,2
	197	2: bf cr7*4+1,3f
	198	lwzx r0,r7,r4
	199	stwx r0,r7,r3
86e32fdc	200	3: PPC_MTOCRF(0x01,r5)
14cf11af PM	201	add r4,r6,r4
	202	add r3,r6,r3
	203	b .Ldst_aligned
	204
	205	.Lshort_copy:
	206	bf cr7*4+0,1f
	207	lwz r0,0(r4)
	208	lwz r9,4(r4)
	209	addi r4,r4,8
	210	stw r0,0(r3)
	211	stw r9,4(r3)
	212	addi r3,r3,8
	213	1: bf cr7*4+1,2f
	214	lwz r0,0(r4)
	215	addi r4,r4,4
	216	stw r0,0(r3)
	217	addi r3,r3,4
	218	2: bf cr7*4+2,3f
	219	lhz r0,0(r4)
	220	addi r4,r4,2
	221	sth r0,0(r3)
	222	addi r3,r3,2
	223	3: bf cr7*4+3,4f
	224	lbz r0,0(r4)
	225	stb r0,0(r3)
752a6422	226	4: ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) /* return dest pointer */
d0027bf0	227	blr
00f554fa	228	#endif
9445aa1a	229	EXPORT_SYMBOL(memcpy)
26deb043	230	EXPORT_SYMBOL_KASAN(memcpy)