[linux-2.6-block.git] / arch / arm / lib / bitops.h

/* SPDX-License-Identifier: GPL-2.0 */
#include <asm/assembler.h>
#include <asm/unwind.h>

#if __LINUX_ARM_ARCH__ >= 6
	.macro	bitop, name, instr
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	mov	r2, #1
	and	r3, r0, #31		@ Get bit offset
	mov	r0, r0, lsr #5
	add	r1, r1, r0, lsl #2	@ Get word offset
#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
	.arch_extension	mp
	ALT_SMP(W(pldw)	[r1])
	ALT_UP(W(nop))
#endif
	mov	r3, r2, lsl r3
1:	ldrex	r2, [r1]
	\instr	r2, r2, r3
	strex	r0, r2, [r1]
	cmp	r0, #0
	bne	1b
	bx	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm

	.macro	__testop, name, instr, store, barrier
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	mov	r2, #1
	and	r3, r0, #31		@ Get bit offset
	mov	r0, r0, lsr #5
	add	r1, r1, r0, lsl #2	@ Get word offset
	mov	r3, r2, lsl r3		@ create mask
	\barrier
#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
	.arch_extension	mp
	ALT_SMP(W(pldw)	[r1])
	ALT_UP(W(nop))
#endif
1:	ldrex	r2, [r1]
	ands	r0, r2, r3		@ save old value of bit
	\instr	r2, r2, r3		@ toggle bit
	strex	ip, r2, [r1]
	cmp	ip, #0
	bne	1b
	\barrier
	cmp	r0, #0
	movne	r0, #1
2:	bx	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm

	.macro	testop, name, instr, store
	__testop \name, \instr, \store, smp_dmb
	.endm

	.macro	sync_testop, name, instr, store
	__testop \name, \instr, \store, __smp_dmb
	.endm
#else
	.macro	bitop, name, instr
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	and	r2, r0, #31
	mov	r0, r0, lsr #5
	mov	r3, #1
	mov	r3, r3, lsl r2
	save_and_disable_irqs ip
	ldr	r2, [r1, r0, lsl #2]
	\instr	r2, r2, r3
	str	r2, [r1, r0, lsl #2]
	restore_irqs ip
	ret	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm

/**
 * testop - implement a test_and_xxx_bit operation.
 * @instr: operational instruction
 * @store: store instruction
 *
 * Note: we can trivially conditionalise the store instruction
 * to avoid dirtying the data cache.
 */
	.macro	testop, name, instr, store
ENTRY(	\name		)
UNWIND(	.fnstart	)
	ands	ip, r1, #3
	strbne	r1, [ip]		@ assert word-aligned
	and	r3, r0, #31
	mov	r0, r0, lsr #5
	save_and_disable_irqs ip
	ldr	r2, [r1, r0, lsl #2]!
	mov	r0, #1
	tst	r2, r0, lsl r3
	\instr	r2, r2, r0, lsl r3
	\store	r2, [r1]
	moveq	r0, #0
	restore_irqs ip
	ret	lr
UNWIND(	.fnend		)
ENDPROC(\name		)
	.endm
#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
6ebbf2ce	2	#include <asm/assembler.h>
c36ef4b1 WD	3	#include <asm/unwind.h>
c36ef4b1 WD	4
6323f0cc	5	#if __LINUX_ARM_ARCH__ >= 6
c36ef4b1 WD	6	.macro bitop, name, instr
	7	ENTRY( \name )
	8	UNWIND( .fnstart )
a16ede35	9	ands ip, r1, #3
c001899a	10	strbne r1, [ip] @ assert word-aligned
54ea06f6	11	mov r2, #1
6323f0cc RK	12	and r3, r0, #31 @ Get bit offset
	13	mov r0, r0, lsr #5
	14	add r1, r1, r0, lsl #2 @ Get word offset
b7ec6994	15	#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
d779c07d WD	16	.arch_extension mp
	17	ALT_SMP(W(pldw) [r1])
	18	ALT_UP(W(nop))
	19	#endif
54ea06f6	20	mov r3, r2, lsl r3
6323f0cc	21	1: ldrex r2, [r1]
54ea06f6	22	\instr r2, r2, r3
6323f0cc	23	strex r0, r2, [r1]
e7ec0293	24	cmp r0, #0
54ea06f6	25	bne 1b
3ba6e69a	26	bx lr
c36ef4b1 WD	27	UNWIND( .fnend )
c36ef4b1 WD	28	ENDPROC(\name )
54ea06f6 RK	29	.endm
54ea06f6 RK	30
dda5f312	31	.macro __testop, name, instr, store, barrier
c36ef4b1 WD	32	ENTRY( \name )
c36ef4b1 WD	33	UNWIND( .fnstart )
a16ede35	34	ands ip, r1, #3
c001899a	35	strbne r1, [ip] @ assert word-aligned
54ea06f6	36	mov r2, #1
6323f0cc RK	37	and r3, r0, #31 @ Get bit offset
	38	mov r0, r0, lsr #5
	39	add r1, r1, r0, lsl #2 @ Get word offset
54ea06f6	40	mov r3, r2, lsl r3 @ create mask
dda5f312	41	\barrier
c32ffce0 WD	42	#if __LINUX_ARM_ARCH__ >= 7 && defined(CONFIG_SMP)
	43	.arch_extension mp
	44	ALT_SMP(W(pldw) [r1])
	45	ALT_UP(W(nop))
	46	#endif
6323f0cc	47	1: ldrex r2, [r1]
54ea06f6	48	ands r0, r2, r3 @ save old value of bit
6323f0cc RK	49	\instr r2, r2, r3 @ toggle bit
6323f0cc RK	50	strex ip, r2, [r1]
614d73ed	51	cmp ip, #0
54ea06f6	52	bne 1b
dda5f312	53	\barrier
54ea06f6 RK	54	cmp r0, #0
54ea06f6 RK	55	movne r0, #1
3ba6e69a	56	2: bx lr
c36ef4b1 WD	57	UNWIND( .fnend )
c36ef4b1 WD	58	ENDPROC(\name )
54ea06f6	59	.endm
dda5f312 MR	60
	61	.macro testop, name, instr, store
	62	__testop \name, \instr, \store, smp_dmb
	63	.endm
	64
	65	.macro sync_testop, name, instr, store
	66	__testop \name, \instr, \store, __smp_dmb
	67	.endm
54ea06f6	68	#else
c36ef4b1 WD	69	.macro bitop, name, instr
	70	ENTRY( \name )
	71	UNWIND( .fnstart )
a16ede35	72	ands ip, r1, #3
c001899a	73	strbne r1, [ip] @ assert word-aligned
6323f0cc RK	74	and r2, r0, #31
6323f0cc RK	75	mov r0, r0, lsr #5
7a55fd0b RK	76	mov r3, #1
7a55fd0b RK	77	mov r3, r3, lsl r2
59d1ff3b	78	save_and_disable_irqs ip
6323f0cc	79	ldr r2, [r1, r0, lsl #2]
7a55fd0b	80	\instr r2, r2, r3
6323f0cc	81	str r2, [r1, r0, lsl #2]
7a55fd0b	82	restore_irqs ip
6ebbf2ce	83	ret lr
c36ef4b1 WD	84	UNWIND( .fnend )
c36ef4b1 WD	85	ENDPROC(\name )
7a55fd0b RK	86	.endm
	87
	88	/**
	89	* testop - implement a test_and_xxx_bit operation.
	90	* @instr: operational instruction
	91	* @store: store instruction
	92	*
	93	* Note: we can trivially conditionalise the store instruction
6cbdc8c5	94	* to avoid dirtying the data cache.
7a55fd0b	95	*/
c36ef4b1 WD	96	.macro testop, name, instr, store
	97	ENTRY( \name )
	98	UNWIND( .fnstart )
a16ede35	99	ands ip, r1, #3
c001899a	100	strbne r1, [ip] @ assert word-aligned
6323f0cc RK	101	and r3, r0, #31
6323f0cc RK	102	mov r0, r0, lsr #5
59d1ff3b	103	save_and_disable_irqs ip
6323f0cc RK	104	ldr r2, [r1, r0, lsl #2]!
6323f0cc RK	105	mov r0, #1
7a55fd0b RK	106	tst r2, r0, lsl r3
	107	\instr r2, r2, r0, lsl r3
	108	\store r2, [r1]
7a55fd0b	109	moveq r0, #0
0d928b0b	110	restore_irqs ip
6ebbf2ce	111	ret lr
c36ef4b1 WD	112	UNWIND( .fnend )
c36ef4b1 WD	113	ENDPROC(\name )
7a55fd0b	114	.endm
54ea06f6	115	#endif