[linux-block.git] / arch / s390 / lib / mem.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * String handling functions.
 *
 * Copyright IBM Corp. 2012
 */

#include <linux/linkage.h>
#include <asm/export.h>

/*
 * void *memmove(void *dest, const void *src, size_t n)
 */
ENTRY(memmove)
	ltgr	%r4,%r4
	lgr	%r1,%r2
	bzr	%r14
	aghi	%r4,-1
	clgr	%r2,%r3
	jnh	.Lmemmove_forward
	la	%r5,1(%r4,%r3)
	clgr	%r2,%r5
	jl	.Lmemmove_reverse
.Lmemmove_forward:
	srlg	%r0,%r4,8
	ltgr	%r0,%r0
	jz	.Lmemmove_forward_remainder
.Lmemmove_forward_loop:
	mvc	0(256,%r1),0(%r3)
	la	%r1,256(%r1)
	la	%r3,256(%r3)
	brctg	%r0,.Lmemmove_forward_loop
.Lmemmove_forward_remainder:
	larl	%r5,.Lmemmove_mvc
	ex	%r4,0(%r5)
	br	%r14
.Lmemmove_reverse:
	ic	%r0,0(%r4,%r3)
	stc	%r0,0(%r4,%r1)
	brctg	%r4,.Lmemmove_reverse
	ic	%r0,0(%r4,%r3)
	stc	%r0,0(%r4,%r1)
	br	%r14
.Lmemmove_mvc:
	mvc	0(1,%r1),0(%r3)
EXPORT_SYMBOL(memmove)

/*
 * memset implementation
 *
 * This code corresponds to the C construct below. We do distinguish
 * between clearing (c == 0) and setting a memory array (c != 0) simply
 * because nearly all memset invocations in the kernel clear memory and
 * the xc instruction is preferred in such cases.
 *
 * void *memset(void *s, int c, size_t n)
 * {
 *	if (likely(c == 0))
 *		return __builtin_memset(s, 0, n);
 *	return __builtin_memset(s, c, n);
 * }
 */
ENTRY(memset)
	ltgr	%r4,%r4
	bzr	%r14
	ltgr	%r3,%r3
	jnz	.Lmemset_fill
	aghi	%r4,-1
	srlg	%r3,%r4,8
	ltgr	%r3,%r3
	lgr	%r1,%r2
	jz	.Lmemset_clear_remainder
.Lmemset_clear_loop:
	xc	0(256,%r1),0(%r1)
	la	%r1,256(%r1)
	brctg	%r3,.Lmemset_clear_loop
.Lmemset_clear_remainder:
	larl	%r3,.Lmemset_xc
	ex	%r4,0(%r3)
	br	%r14
.Lmemset_fill:
	stc	%r3,0(%r2)
	cghi	%r4,1
	lgr	%r1,%r2
	ber	%r14
	aghi	%r4,-2
	srlg	%r3,%r4,8
	ltgr	%r3,%r3
	jz	.Lmemset_fill_remainder
.Lmemset_fill_loop:
	mvc	1(256,%r1),0(%r1)
	la	%r1,256(%r1)
	brctg	%r3,.Lmemset_fill_loop
.Lmemset_fill_remainder:
	larl	%r3,.Lmemset_mvc
	ex	%r4,0(%r3)
	br	%r14
.Lmemset_xc:
	xc	0(1,%r1),0(%r1)
.Lmemset_mvc:
	mvc	1(1,%r1),0(%r1)
EXPORT_SYMBOL(memset)

/*
 * memcpy implementation
 *
 * void *memcpy(void *dest, const void *src, size_t n)
 */
ENTRY(memcpy)
	ltgr	%r4,%r4
	bzr	%r14
	aghi	%r4,-1
	srlg	%r5,%r4,8
	ltgr	%r5,%r5
	lgr	%r1,%r2
	jnz	.Lmemcpy_loop
.Lmemcpy_remainder:
	larl	%r5,.Lmemcpy_mvc
	ex	%r4,0(%r5)
	br	%r14
.Lmemcpy_loop:
	mvc	0(256,%r1),0(%r3)
	la	%r1,256(%r1)
	la	%r3,256(%r3)
	brctg	%r5,.Lmemcpy_loop
	j	.Lmemcpy_remainder
.Lmemcpy_mvc:
	mvc	0(1,%r1),0(%r3)
EXPORT_SYMBOL(memcpy)
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
535c611d HC	2	/*
	3	* String handling functions.
	4	*
	5	* Copyright IBM Corp. 2012
	6	*/
	7
	8	#include <linux/linkage.h>
711f5df7	9	#include <asm/export.h>
535c611d	10
b4623d4e HC	11	/*
	12	* void memmove(void dest, const void *src, size_t n)
	13	*/
	14	ENTRY(memmove)
	15	ltgr %r4,%r4
	16	lgr %r1,%r2
	17	bzr %r14
551f4134	18	aghi %r4,-1
b4623d4e HC	19	clgr %r2,%r3
b4623d4e HC	20	jnh .Lmemmove_forward
551f4134	21	la %r5,1(%r4,%r3)
b4623d4e HC	22	clgr %r2,%r5
	23	jl .Lmemmove_reverse
	24	.Lmemmove_forward:
b4623d4e HC	25	srlg %r0,%r4,8
b4623d4e HC	26	ltgr %r0,%r0
551f4134 HC	27	jz .Lmemmove_forward_remainder
551f4134 HC	28	.Lmemmove_forward_loop:
b4623d4e HC	29	mvc 0(256,%r1),0(%r3)
	30	la %r1,256(%r1)
	31	la %r3,256(%r3)
551f4134 HC	32	brctg %r0,.Lmemmove_forward_loop
551f4134 HC	33	.Lmemmove_forward_remainder:
b4623d4e HC	34	larl %r5,.Lmemmove_mvc
	35	ex %r4,0(%r5)
	36	br %r14
	37	.Lmemmove_reverse:
b4623d4e HC	38	ic %r0,0(%r4,%r3)
b4623d4e HC	39	stc %r0,0(%r4,%r1)
551f4134	40	brctg %r4,.Lmemmove_reverse
b4623d4e HC	41	ic %r0,0(%r4,%r3)
	42	stc %r0,0(%r4,%r1)
	43	br %r14
	44	.Lmemmove_mvc:
	45	mvc 0(1,%r1),0(%r3)
	46	EXPORT_SYMBOL(memmove)
	47
535c611d HC	48	/*
	49	* memset implementation
	50	*
	51	* This code corresponds to the C construct below. We do distinguish
	52	* between clearing (c == 0) and setting a memory array (c != 0) simply
	53	* because nearly all memset invocations in the kernel clear memory and
	54	* the xc instruction is preferred in such cases.
	55	*
	56	* void memset(void s, int c, size_t n)
	57	* {
	58	* if (likely(c == 0))
	59	* return __builtin_memset(s, 0, n);
	60	* return __builtin_memset(s, c, n);
	61	* }
	62	*/
	63	ENTRY(memset)
	64	ltgr %r4,%r4
	65	bzr %r14
	66	ltgr %r3,%r3
	67	jnz .Lmemset_fill
	68	aghi %r4,-1
	69	srlg %r3,%r4,8
	70	ltgr %r3,%r3
	71	lgr %r1,%r2
551f4134	72	jz .Lmemset_clear_remainder
535c611d HC	73	.Lmemset_clear_loop:
	74	xc 0(256,%r1),0(%r1)
	75	la %r1,256(%r1)
	76	brctg %r3,.Lmemset_clear_loop
551f4134	77	.Lmemset_clear_remainder:
535c611d HC	78	larl %r3,.Lmemset_xc
	79	ex %r4,0(%r3)
	80	br %r14
	81	.Lmemset_fill:
	82	stc %r3,0(%r2)
	83	cghi %r4,1
	84	lgr %r1,%r2
	85	ber %r14
	86	aghi %r4,-2
	87	srlg %r3,%r4,8
	88	ltgr %r3,%r3
551f4134	89	jz .Lmemset_fill_remainder
535c611d HC	90	.Lmemset_fill_loop:
	91	mvc 1(256,%r1),0(%r1)
	92	la %r1,256(%r1)
	93	brctg %r3,.Lmemset_fill_loop
551f4134	94	.Lmemset_fill_remainder:
535c611d HC	95	larl %r3,.Lmemset_mvc
	96	ex %r4,0(%r3)
	97	br %r14
	98	.Lmemset_xc:
	99	xc 0(1,%r1),0(%r1)
	100	.Lmemset_mvc:
	101	mvc 1(1,%r1),0(%r1)
711f5df7	102	EXPORT_SYMBOL(memset)
535c611d HC	103
	104	/*
	105	* memcpy implementation
	106	*
	107	* void memcpy(void dest, const void *src, size_t n)
	108	*/
	109	ENTRY(memcpy)
	110	ltgr %r4,%r4
	111	bzr %r14
	112	aghi %r4,-1
	113	srlg %r5,%r4,8
	114	ltgr %r5,%r5
	115	lgr %r1,%r2
	116	jnz .Lmemcpy_loop
551f4134	117	.Lmemcpy_remainder:
535c611d HC	118	larl %r5,.Lmemcpy_mvc
	119	ex %r4,0(%r5)
	120	br %r14
	121	.Lmemcpy_loop:
	122	mvc 0(256,%r1),0(%r3)
	123	la %r1,256(%r1)
	124	la %r3,256(%r3)
	125	brctg %r5,.Lmemcpy_loop
551f4134	126	j .Lmemcpy_remainder
535c611d HC	127	.Lmemcpy_mvc:
535c611d HC	128	mvc 0(1,%r1),0(%r3)
711f5df7	129	EXPORT_SYMBOL(memcpy)