[linux-2.6-block.git] / arch / arm / common / mcpm_head.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
 *
 * Created by:  Nicolas Pitre, March 2012
 * Copyright:   (C) 2012-2013  Linaro Limited
 *
 * Refer to Documentation/arm/cluster-pm-race-avoidance.rst
 * for details of the synchronisation algorithms used here.
 */

#include <linux/linkage.h>
#include <asm/mcpm.h>
#include <asm/assembler.h>

#include "vlock.h"

.if MCPM_SYNC_CLUSTER_CPUS
.error "cpus must be the first member of struct mcpm_sync_struct"
.endif

	.macro	pr_dbg	string
#if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
	b	1901f
1902:	.asciz	"CPU"
1903:	.asciz	" cluster"
1904:	.asciz	": \string"
	.align
1901:	adr	r0, 1902b
	bl	printascii
	mov	r0, r9
	bl	printhex2
	adr	r0, 1903b
	bl	printascii
	mov	r0, r10
	bl	printhex2
	adr	r0, 1904b
	bl	printascii
#endif
	.endm

	.arm
	.align

ENTRY(mcpm_entry_point)

 ARM_BE8(setend        be)
 THUMB(	badr	r12, 1f		)
 THUMB(	bx	r12		)
 THUMB(	.thumb			)
1:
	mrc	p15, 0, r0, c0, c0, 5		@ MPIDR
	ubfx	r9, r0, #0, #8			@ r9 = cpu
	ubfx	r10, r0, #8, #8			@ r10 = cluster
	mov	r3, #MAX_CPUS_PER_CLUSTER
	mla	r4, r3, r10, r9			@ r4 = canonical CPU index
	cmp	r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
	blo	2f

	/* We didn't expect this CPU.  Try to cheaply make it quiet. */
1:	wfi
	wfe
	b	1b

2:	pr_dbg	"kernel mcpm_entry_point\n"

	/*
	 * MMU is off so we need to get to various variables in a
	 * position independent way.
	 */
	adr	r5, 3f
	ldmia	r5, {r0, r6, r7, r8, r11}
	add	r0, r5, r0			@ r0 = mcpm_entry_early_pokes
	add	r6, r5, r6			@ r6 = mcpm_entry_vectors
	ldr	r7, [r5, r7]			@ r7 = mcpm_power_up_setup_phys
	add	r8, r5, r8			@ r8 = mcpm_sync
	add	r11, r5, r11			@ r11 = first_man_locks

	@ Perform an early poke, if any
	add	r0, r0, r4, lsl #3
	ldmia	r0, {r0, r1}
	teq	r0, #0
	strne	r1, [r0]

	mov	r0, #MCPM_SYNC_CLUSTER_SIZE
	mla	r8, r0, r10, r8			@ r8 = sync cluster base

	@ Signal that this CPU is coming UP:
	mov	r0, #CPU_COMING_UP
	mov	r5, #MCPM_SYNC_CPU_SIZE
	mla	r5, r9, r5, r8			@ r5 = sync cpu address
	strb	r0, [r5]

	@ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
	@ state, because there is at least one active CPU (this CPU).

	mov	r0, #VLOCK_SIZE
	mla	r11, r0, r10, r11		@ r11 = cluster first man lock
	mov	r0, r11
	mov	r1, r9				@ cpu
	bl	vlock_trylock			@ implies DMB

	cmp	r0, #0				@ failed to get the lock?
	bne	mcpm_setup_wait		@ wait for cluster setup if so

	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP			@ cluster already up?
	bne	mcpm_setup			@ if not, set up the cluster

	@ Otherwise, release the first man lock and skip setup:
	mov	r0, r11
	bl	vlock_unlock
	b	mcpm_setup_complete

mcpm_setup:
	@ Control dependency implies strb not observable before previous ldrb.

	@ Signal that the cluster is being brought up:
	mov	r0, #INBOUND_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dmb

	@ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
	@ point onwards will observe INBOUND_COMING_UP and abort.

	@ Wait for any previously-pending cluster teardown operations to abort
	@ or complete:
mcpm_teardown_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_GOING_DOWN
	bne	first_man_setup
	wfe
	b	mcpm_teardown_wait

first_man_setup:
	dmb

	@ If the outbound gave up before teardown started, skip cluster setup:

	cmp	r0, #CLUSTER_UP
	beq	mcpm_setup_leave

	@ power_up_setup is now responsible for setting up the cluster:

	cmp	r7, #0
	mov	r0, #1		@ second (cluster) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	mov	r0, #CLUSTER_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	dmb

mcpm_setup_leave:
	@ Leave the cluster setup critical section:

	mov	r0, #INBOUND_NOT_COMING_UP
	strb	r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	dsb	st
	sev

	mov	r0, r11
	bl	vlock_unlock	@ implies DMB
	b	mcpm_setup_complete

	@ In the contended case, non-first men wait here for cluster setup
	@ to complete:
mcpm_setup_wait:
	ldrb	r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	cmp	r0, #CLUSTER_UP
	wfene
	bne	mcpm_setup_wait
	dmb

mcpm_setup_complete:
	@ If a platform-specific CPU setup hook is needed, it is
	@ called from here.

	cmp	r7, #0
	mov	r0, #0		@ first (CPU) affinity level
	blxne	r7		@ Call power_up_setup if defined
	dmb

	@ Mark the CPU as up:

	mov	r0, #CPU_UP
	strb	r0, [r5]

	@ Observability order of CPU_UP and opening of the gate does not matter.

mcpm_entry_gated:
	ldr	r5, [r6, r4, lsl #2]		@ r5 = CPU entry vector
	cmp	r5, #0
	wfeeq
	beq	mcpm_entry_gated
	dmb

	pr_dbg	"released\n"
	bx	r5

	.align	2

3:	.word	mcpm_entry_early_pokes - .
	.word	mcpm_entry_vectors - 3b
	.word	mcpm_power_up_setup_phys - 3b
	.word	mcpm_sync - 3b
	.word	first_man_locks - 3b

ENDPROC(mcpm_entry_point)

	.bss

	.align	CACHE_WRITEBACK_ORDER
	.type	first_man_locks, #object
first_man_locks:
	.space	VLOCK_SIZE * MAX_NR_CLUSTERS
	.align	CACHE_WRITEBACK_ORDER

	.type	mcpm_entry_vectors, #object
ENTRY(mcpm_entry_vectors)
	.space	4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_entry_early_pokes, #object
ENTRY(mcpm_entry_early_pokes)
	.space	8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER

	.type	mcpm_power_up_setup_phys, #object
ENTRY(mcpm_power_up_setup_phys)
	.space  4		@ set by mcpm_sync_init()
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
e8db288e NP	2	/*
	3	* arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
	4	*
	5	* Created by: Nicolas Pitre, March 2012
	6	* Copyright: (C) 2012-2013 Linaro Limited
	7	*
dc7a12bd	8	* Refer to Documentation/arm/cluster-pm-race-avoidance.rst
7fe31d28	9	* for details of the synchronisation algorithms used here.
e8db288e NP	10	*/
	11
	12	#include <linux/linkage.h>
	13	#include <asm/mcpm.h>
519ceb9f	14	#include <asm/assembler.h>
e8db288e	15
1ae98561 DM	16	#include "vlock.h"
1ae98561 DM	17
7fe31d28 DM	18	.if MCPM_SYNC_CLUSTER_CPUS
	19	.error "cpus must be the first member of struct mcpm_sync_struct"
	20	.endif
	21
e8db288e NP	22	.macro pr_dbg string
	23	#if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
	24	b 1901f
	25	1902: .asciz "CPU"
	26	1903: .asciz " cluster"
	27	1904: .asciz ": \string"
	28	.align
	29	1901: adr r0, 1902b
	30	bl printascii
	31	mov r0, r9
0305ffd7	32	bl printhex2
e8db288e NP	33	adr r0, 1903b
	34	bl printascii
	35	mov r0, r10
0305ffd7	36	bl printhex2
e8db288e NP	37	adr r0, 1904b
	38	bl printascii
	39	#endif
	40	.endm
	41
	42	.arm
	43	.align
	44
	45	ENTRY(mcpm_entry_point)
	46
519ceb9f	47	ARM_BE8(setend be)
14327c66	48	THUMB( badr r12, 1f )
e8db288e NP	49	THUMB( bx r12 )
	50	THUMB( .thumb )
	51	1:
	52	mrc p15, 0, r0, c0, c0, 5 @ MPIDR
	53	ubfx r9, r0, #0, #8 @ r9 = cpu
	54	ubfx r10, r0, #8, #8 @ r10 = cluster
	55	mov r3, #MAX_CPUS_PER_CLUSTER
	56	mla r4, r3, r10, r9 @ r4 = canonical CPU index
	57	cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
	58	blo 2f
	59
	60	/* We didn't expect this CPU. Try to cheaply make it quiet. */
	61	1: wfi
	62	wfe
	63	b 1b
	64
	65	2: pr_dbg "kernel mcpm_entry_point\n"
	66
	67	/*
7fe31d28	68	* MMU is off so we need to get to various variables in a
e8db288e NP	69	* position independent way.
	70	*/
	71	adr r5, 3f
de885d14 NP	72	ldmia r5, {r0, r6, r7, r8, r11}
de885d14 NP	73	add r0, r5, r0 @ r0 = mcpm_entry_early_pokes
e8db288e	74	add r6, r5, r6 @ r6 = mcpm_entry_vectors
7fe31d28 DM	75	ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys
7fe31d28 DM	76	add r8, r5, r8 @ r8 = mcpm_sync
1ae98561	77	add r11, r5, r11 @ r11 = first_man_locks
7fe31d28	78
de885d14 NP	79	@ Perform an early poke, if any
	80	add r0, r0, r4, lsl #3
	81	ldmia r0, {r0, r1}
	82	teq r0, #0
	83	strne r1, [r0]
	84
7fe31d28 DM	85	mov r0, #MCPM_SYNC_CLUSTER_SIZE
	86	mla r8, r0, r10, r8 @ r8 = sync cluster base
	87
	88	@ Signal that this CPU is coming UP:
	89	mov r0, #CPU_COMING_UP
	90	mov r5, #MCPM_SYNC_CPU_SIZE
	91	mla r5, r9, r5, r8 @ r5 = sync cpu address
	92	strb r0, [r5]
	93
	94	@ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
	95	@ state, because there is at least one active CPU (this CPU).
	96
1ae98561 DM	97	mov r0, #VLOCK_SIZE
	98	mla r11, r0, r10, r11 @ r11 = cluster first man lock
	99	mov r0, r11
	100	mov r1, r9 @ cpu
	101	bl vlock_trylock @ implies DMB
	102
	103	cmp r0, #0 @ failed to get the lock?
	104	bne mcpm_setup_wait @ wait for cluster setup if so
	105
7fe31d28 DM	106	ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	107	cmp r0, #CLUSTER_UP @ cluster already up?
	108	bne mcpm_setup @ if not, set up the cluster
	109
1ae98561 DM	110	@ Otherwise, release the first man lock and skip setup:
	111	mov r0, r11
	112	bl vlock_unlock
7fe31d28 DM	113	b mcpm_setup_complete
	114
	115	mcpm_setup:
	116	@ Control dependency implies strb not observable before previous ldrb.
	117
	118	@ Signal that the cluster is being brought up:
	119	mov r0, #INBOUND_COMING_UP
	120	strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
	121	dmb
	122
	123	@ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
	124	@ point onwards will observe INBOUND_COMING_UP and abort.
	125
	126	@ Wait for any previously-pending cluster teardown operations to abort
	127	@ or complete:
	128	mcpm_teardown_wait:
	129	ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	130	cmp r0, #CLUSTER_GOING_DOWN
	131	bne first_man_setup
	132	wfe
	133	b mcpm_teardown_wait
	134
	135	first_man_setup:
	136	dmb
	137
	138	@ If the outbound gave up before teardown started, skip cluster setup:
	139
	140	cmp r0, #CLUSTER_UP
	141	beq mcpm_setup_leave
	142
	143	@ power_up_setup is now responsible for setting up the cluster:
	144
	145	cmp r7, #0
	146	mov r0, #1 @ second (cluster) affinity level
	147	blxne r7 @ Call power_up_setup if defined
	148	dmb
	149
	150	mov r0, #CLUSTER_UP
	151	strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	152	dmb
	153
	154	mcpm_setup_leave:
	155	@ Leave the cluster setup critical section:
	156
	157	mov r0, #INBOUND_NOT_COMING_UP
	158	strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
40a5c0b4	159	dsb st
7fe31d28 DM	160	sev
7fe31d28 DM	161
1ae98561 DM	162	mov r0, r11
	163	bl vlock_unlock @ implies DMB
	164	b mcpm_setup_complete
	165
	166	@ In the contended case, non-first men wait here for cluster setup
	167	@ to complete:
	168	mcpm_setup_wait:
	169	ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
	170	cmp r0, #CLUSTER_UP
	171	wfene
	172	bne mcpm_setup_wait
	173	dmb
	174
7fe31d28 DM	175	mcpm_setup_complete:
	176	@ If a platform-specific CPU setup hook is needed, it is
	177	@ called from here.
	178
	179	cmp r7, #0
	180	mov r0, #0 @ first (CPU) affinity level
	181	blxne r7 @ Call power_up_setup if defined
	182	dmb
	183
	184	@ Mark the CPU as up:
	185
	186	mov r0, #CPU_UP
	187	strb r0, [r5]
	188
	189	@ Observability order of CPU_UP and opening of the gate does not matter.
e8db288e NP	190
	191	mcpm_entry_gated:
	192	ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector
	193	cmp r5, #0
	194	wfeeq
	195	beq mcpm_entry_gated
7fe31d28 DM	196	dmb
7fe31d28 DM	197
e8db288e NP	198	pr_dbg "released\n"
	199	bx r5
	200
	201	.align 2
	202
de885d14 NP	203	3: .word mcpm_entry_early_pokes - .
de885d14 NP	204	.word mcpm_entry_vectors - 3b
7fe31d28 DM	205	.word mcpm_power_up_setup_phys - 3b
7fe31d28 DM	206	.word mcpm_sync - 3b
1ae98561	207	.word first_man_locks - 3b
e8db288e NP	208
	209	ENDPROC(mcpm_entry_point)
	210
	211	.bss
1ae98561 DM	212
	213	.align CACHE_WRITEBACK_ORDER
	214	.type first_man_locks, #object
	215	first_man_locks:
	216	.space VLOCK_SIZE * MAX_NR_CLUSTERS
	217	.align CACHE_WRITEBACK_ORDER
e8db288e NP	218
	219	.type mcpm_entry_vectors, #object
	220	ENTRY(mcpm_entry_vectors)
	221	.space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
7fe31d28	222
de885d14 NP	223	.type mcpm_entry_early_pokes, #object
	224	ENTRY(mcpm_entry_early_pokes)
	225	.space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
	226
7fe31d28 DM	227	.type mcpm_power_up_setup_phys, #object
	228	ENTRY(mcpm_power_up_setup_phys)
	229	.space 4 @ set by mcpm_sync_init()