[linux-block.git] / arch / loongarch / lib / clear_user.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */

#include <asm/alternative-asm.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/asm-extable.h>
#include <asm/cpu.h>
#include <asm/export.h>
#include <asm/regdef.h>

.irp to, 0, 1, 2, 3, 4, 5, 6, 7
.L_fixup_handle_\to\():
	sub.d	a0, a2, a0
	addi.d	a0, a0, (\to) * (-8)
	jr	ra
.endr

.irp to, 0, 2, 4
.L_fixup_handle_s\to\():
	addi.d	a0, a1, -\to
	jr	ra
.endr

SYM_FUNC_START(__clear_user)
	/*
	 * Some CPUs support hardware unaligned access
	 */
	ALTERNATIVE	"b __clear_user_generic",	\
			"b __clear_user_fast", CPU_FEATURE_UAL
SYM_FUNC_END(__clear_user)

EXPORT_SYMBOL(__clear_user)

/*
 * unsigned long __clear_user_generic(void *addr, size_t size)
 *
 * a0: addr
 * a1: size
 */
SYM_FUNC_START(__clear_user_generic)
	beqz	a1, 2f

1:	st.b	zero, a0, 0
	addi.d	a0, a0, 1
	addi.d	a1, a1, -1
	bgtz	a1, 1b

2:	move	a0, a1
	jr	ra

	_asm_extable 1b, .L_fixup_handle_s0
SYM_FUNC_END(__clear_user_generic)

/*
 * unsigned long __clear_user_fast(void *addr, unsigned long size)
 *
 * a0: addr
 * a1: size
 */
SYM_FUNC_START(__clear_user_fast)
	sltui	t0, a1, 9
	bnez	t0, .Lsmall

	add.d	a2, a0, a1
0:	st.d	zero, a0, 0

	/* align up address */
	addi.d	a0, a0, 8
	bstrins.d	a0, zero, 2, 0

	addi.d	a3, a2, -64
	bgeu	a0, a3, .Llt64

	/* set 64 bytes at a time */
.Lloop64:
1:	st.d	zero, a0, 0
2:	st.d	zero, a0, 8
3:	st.d	zero, a0, 16
4:	st.d	zero, a0, 24
5:	st.d	zero, a0, 32
6:	st.d	zero, a0, 40
7:	st.d	zero, a0, 48
8:	st.d	zero, a0, 56
	addi.d	a0, a0, 64
	bltu	a0, a3, .Lloop64

	/* set the remaining bytes */
.Llt64:
	addi.d	a3, a2, -32
	bgeu	a0, a3, .Llt32
9:	st.d	zero, a0, 0
10:	st.d	zero, a0, 8
11:	st.d	zero, a0, 16
12:	st.d	zero, a0, 24
	addi.d	a0, a0, 32

.Llt32:
	addi.d	a3, a2, -16
	bgeu	a0, a3, .Llt16
13:	st.d	zero, a0, 0
14:	st.d	zero, a0, 8
	addi.d	a0, a0, 16

.Llt16:
	addi.d	a3, a2, -8
	bgeu	a0, a3, .Llt8
15:	st.d	zero, a0, 0
	addi.d	a0, a0, 8

.Llt8:
16:	st.d	zero, a2, -8

	/* return */
	move	a0, zero
	jr	ra

	.align	4
.Lsmall:
	pcaddi	t0, 4
	slli.d	a2, a1, 4
	add.d	t0, t0, a2
	jr	t0

	.align	4
	move	a0, zero
	jr	ra

	.align	4
17:	st.b	zero, a0, 0
	move	a0, zero
	jr	ra

	.align	4
18:	st.h	zero, a0, 0
	move	a0, zero
	jr	ra

	.align	4
19:	st.h	zero, a0, 0
20:	st.b	zero, a0, 2
	move	a0, zero
	jr	ra

	.align	4
21:	st.w	zero, a0, 0
	move	a0, zero
	jr	ra

	.align	4
22:	st.w	zero, a0, 0
23:	st.b	zero, a0, 4
	move	a0, zero
	jr	ra

	.align	4
24:	st.w	zero, a0, 0
25:	st.h	zero, a0, 4
	move	a0, zero
	jr	ra

	.align	4
26:	st.w	zero, a0, 0
27:	st.w	zero, a0, 3
	move	a0, zero
	jr	ra

	.align	4
28:	st.d	zero, a0, 0
	move	a0, zero
	jr	ra

	/* fixup and ex_table */
	_asm_extable 0b, .L_fixup_handle_0
	_asm_extable 1b, .L_fixup_handle_0
	_asm_extable 2b, .L_fixup_handle_1
	_asm_extable 3b, .L_fixup_handle_2
	_asm_extable 4b, .L_fixup_handle_3
	_asm_extable 5b, .L_fixup_handle_4
	_asm_extable 6b, .L_fixup_handle_5
	_asm_extable 7b, .L_fixup_handle_6
	_asm_extable 8b, .L_fixup_handle_7
	_asm_extable 9b, .L_fixup_handle_0
	_asm_extable 10b, .L_fixup_handle_1
	_asm_extable 11b, .L_fixup_handle_2
	_asm_extable 12b, .L_fixup_handle_3
	_asm_extable 13b, .L_fixup_handle_0
	_asm_extable 14b, .L_fixup_handle_1
	_asm_extable 15b, .L_fixup_handle_0
	_asm_extable 16b, .L_fixup_handle_0
	_asm_extable 17b, .L_fixup_handle_s0
	_asm_extable 18b, .L_fixup_handle_s0
	_asm_extable 19b, .L_fixup_handle_s0
	_asm_extable 20b, .L_fixup_handle_s2
	_asm_extable 21b, .L_fixup_handle_s0
	_asm_extable 22b, .L_fixup_handle_s0
	_asm_extable 23b, .L_fixup_handle_s4
	_asm_extable 24b, .L_fixup_handle_s0
	_asm_extable 25b, .L_fixup_handle_s4
	_asm_extable 26b, .L_fixup_handle_s0
	_asm_extable 27b, .L_fixup_handle_s4
	_asm_extable 28b, .L_fixup_handle_s0
SYM_FUNC_END(__clear_user_fast)
Commit	Line	Data
559671e0 HC	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
	4	*/
	5
a275a82d	6	#include <asm/alternative-asm.h>
559671e0 HC	7	#include <asm/asm.h>
559671e0 HC	8	#include <asm/asmmacro.h>
508f28c6	9	#include <asm/asm-extable.h>
a275a82d	10	#include <asm/cpu.h>
559671e0 HC	11	#include <asm/export.h>
	12	#include <asm/regdef.h>
	13
a275a82d	14	.irp to, 0, 1, 2, 3, 4, 5, 6, 7
912bcfaf	15	.L_fixup_handle_\to\():
8941e93c WR	16	sub.d a0, a2, a0
	17	addi.d a0, a0, (\to) * (-8)
	18	jr ra
	19	.endr
	20
	21	.irp to, 0, 2, 4
	22	.L_fixup_handle_s\to\():
	23	addi.d a0, a1, -\to
559671e0	24	jr ra
912bcfaf	25	.endr
559671e0	26
a275a82d HC	27	SYM_FUNC_START(__clear_user)
	28	/*
	29	* Some CPUs support hardware unaligned access
	30	*/
	31	ALTERNATIVE "b __clear_user_generic", \
	32	"b __clear_user_fast", CPU_FEATURE_UAL
	33	SYM_FUNC_END(__clear_user)
	34
	35	EXPORT_SYMBOL(__clear_user)
	36
559671e0	37	/*
a275a82d	38	* unsigned long __clear_user_generic(void *addr, size_t size)
559671e0 HC	39	*
	40	* a0: addr
	41	* a1: size
	42	*/
a275a82d	43	SYM_FUNC_START(__clear_user_generic)
559671e0 HC	44	beqz a1, 2f
	45
	46	1: st.b zero, a0, 0
	47	addi.d a0, a0, 1
	48	addi.d a1, a1, -1
1fdb9a92	49	bgtz a1, 1b
559671e0 HC	50
	51	2: move a0, a1
	52	jr ra
	53
8941e93c	54	_asm_extable 1b, .L_fixup_handle_s0
a275a82d	55	SYM_FUNC_END(__clear_user_generic)
559671e0	56
a275a82d HC	57	/*
	58	* unsigned long __clear_user_fast(void *addr, unsigned long size)
	59	*
	60	* a0: addr
	61	* a1: size
	62	*/
	63	SYM_FUNC_START(__clear_user_fast)
8941e93c WR	64	sltui t0, a1, 9
8941e93c WR	65	bnez t0, .Lsmall
a275a82d	66
8941e93c WR	67	add.d a2, a0, a1
	68	0: st.d zero, a0, 0
	69
	70	/* align up address */
	71	addi.d a0, a0, 8
	72	bstrins.d a0, zero, 2, 0
	73
	74	addi.d a3, a2, -64
	75	bgeu a0, a3, .Llt64
a275a82d HC	76
a275a82d HC	77	/* set 64 bytes at a time */
8941e93c	78	.Lloop64:
a275a82d HC	79	1: st.d zero, a0, 0
	80	2: st.d zero, a0, 8
	81	3: st.d zero, a0, 16
	82	4: st.d zero, a0, 24
	83	5: st.d zero, a0, 32
	84	6: st.d zero, a0, 40
	85	7: st.d zero, a0, 48
	86	8: st.d zero, a0, 56
a275a82d	87	addi.d a0, a0, 64
8941e93c	88	bltu a0, a3, .Lloop64
a275a82d HC	89
a275a82d HC	90	/* set the remaining bytes */
8941e93c WR	91	.Llt64:
	92	addi.d a3, a2, -32
	93	bgeu a0, a3, .Llt32
	94	9: st.d zero, a0, 0
	95	10: st.d zero, a0, 8
	96	11: st.d zero, a0, 16
	97	12: st.d zero, a0, 24
	98	addi.d a0, a0, 32
	99
	100	.Llt32:
	101	addi.d a3, a2, -16
	102	bgeu a0, a3, .Llt16
	103	13: st.d zero, a0, 0
	104	14: st.d zero, a0, 8
	105	addi.d a0, a0, 16
	106
	107	.Llt16:
	108	addi.d a3, a2, -8
	109	bgeu a0, a3, .Llt8
	110	15: st.d zero, a0, 0
e66d511f	111	addi.d a0, a0, 8
8941e93c WR	112
	113	.Llt8:
	114	16: st.d zero, a2, -8
a275a82d HC	115
a275a82d HC	116	/* return */
8941e93c WR	117	move a0, zero
	118	jr ra
	119
	120	.align 4
	121	.Lsmall:
	122	pcaddi t0, 4
	123	slli.d a2, a1, 4
	124	add.d t0, t0, a2
	125	jr t0
	126
	127	.align 4
	128	move a0, zero
	129	jr ra
	130
	131	.align 4
	132	17: st.b zero, a0, 0
	133	move a0, zero
	134	jr ra
	135
	136	.align 4
	137	18: st.h zero, a0, 0
	138	move a0, zero
	139	jr ra
	140
	141	.align 4
	142	19: st.h zero, a0, 0
	143	20: st.b zero, a0, 2
	144	move a0, zero
	145	jr ra
	146
	147	.align 4
	148	21: st.w zero, a0, 0
	149	move a0, zero
	150	jr ra
	151
	152	.align 4
	153	22: st.w zero, a0, 0
	154	23: st.b zero, a0, 4
	155	move a0, zero
	156	jr ra
	157
	158	.align 4
	159	24: st.w zero, a0, 0
	160	25: st.h zero, a0, 4
	161	move a0, zero
	162	jr ra
	163
	164	.align 4
	165	26: st.w zero, a0, 0
	166	27: st.w zero, a0, 3
	167	move a0, zero
	168	jr ra
	169
	170	.align 4
	171	28: st.d zero, a0, 0
	172	move a0, zero
a275a82d HC	173	jr ra
	174
	175	/* fixup and ex_table */
8941e93c	176	_asm_extable 0b, .L_fixup_handle_0
a275a82d HC	177	_asm_extable 1b, .L_fixup_handle_0
	178	_asm_extable 2b, .L_fixup_handle_1
	179	_asm_extable 3b, .L_fixup_handle_2
	180	_asm_extable 4b, .L_fixup_handle_3
	181	_asm_extable 5b, .L_fixup_handle_4
	182	_asm_extable 6b, .L_fixup_handle_5
	183	_asm_extable 7b, .L_fixup_handle_6
	184	_asm_extable 8b, .L_fixup_handle_7
	185	_asm_extable 9b, .L_fixup_handle_0
8941e93c WR	186	_asm_extable 10b, .L_fixup_handle_1
	187	_asm_extable 11b, .L_fixup_handle_2
	188	_asm_extable 12b, .L_fixup_handle_3
	189	_asm_extable 13b, .L_fixup_handle_0
	190	_asm_extable 14b, .L_fixup_handle_1
	191	_asm_extable 15b, .L_fixup_handle_0
e66d511f	192	_asm_extable 16b, .L_fixup_handle_0
8941e93c WR	193	_asm_extable 17b, .L_fixup_handle_s0
	194	_asm_extable 18b, .L_fixup_handle_s0
	195	_asm_extable 19b, .L_fixup_handle_s0
	196	_asm_extable 20b, .L_fixup_handle_s2
	197	_asm_extable 21b, .L_fixup_handle_s0
	198	_asm_extable 22b, .L_fixup_handle_s0
	199	_asm_extable 23b, .L_fixup_handle_s4
	200	_asm_extable 24b, .L_fixup_handle_s0
	201	_asm_extable 25b, .L_fixup_handle_s4
	202	_asm_extable 26b, .L_fixup_handle_s0
	203	_asm_extable 27b, .L_fixup_handle_s4
	204	_asm_extable 28b, .L_fixup_handle_s0
a275a82d	205	SYM_FUNC_END(__clear_user_fast)