[linux-2.6-block.git] / arch / arm64 / kernel / sleep.S

/* SPDX-License-Identifier: GPL-2.0 */
#include <linux/errno.h>
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>

	.text
/*
 * Implementation of MPIDR_EL1 hash algorithm through shifting
 * and OR'ing.
 *
 * @dst: register containing hash result
 * @rs0: register containing affinity level 0 bit shift
 * @rs1: register containing affinity level 1 bit shift
 * @rs2: register containing affinity level 2 bit shift
 * @rs3: register containing affinity level 3 bit shift
 * @mpidr: register containing MPIDR_EL1 value
 * @mask: register containing MPIDR mask
 *
 * Pseudo C-code:
 *
 *u32 dst;
 *
 *compute_mpidr_hash(u32 rs0, u32 rs1, u32 rs2, u32 rs3, u64 mpidr, u64 mask) {
 *	u32 aff0, aff1, aff2, aff3;
 *	u64 mpidr_masked = mpidr & mask;
 *	aff0 = mpidr_masked & 0xff;
 *	aff1 = mpidr_masked & 0xff00;
 *	aff2 = mpidr_masked & 0xff0000;
 *	aff2 = mpidr_masked & 0xff00000000;
 *	dst = (aff0 >> rs0 | aff1 >> rs1 | aff2 >> rs2 | aff3 >> rs3);
 *}
 * Input registers: rs0, rs1, rs2, rs3, mpidr, mask
 * Output register: dst
 * Note: input and output registers must be disjoint register sets
         (eg: a macro instance with mpidr = x1 and dst = x1 is invalid)
 */
	.macro compute_mpidr_hash dst, rs0, rs1, rs2, rs3, mpidr, mask
	and	\mpidr, \mpidr, \mask		// mask out MPIDR bits
	and	\dst, \mpidr, #0xff		// mask=aff0
	lsr	\dst ,\dst, \rs0		// dst=aff0>>rs0
	and	\mask, \mpidr, #0xff00		// mask = aff1
	lsr	\mask ,\mask, \rs1
	orr	\dst, \dst, \mask		// dst|=(aff1>>rs1)
	and	\mask, \mpidr, #0xff0000	// mask = aff2
	lsr	\mask ,\mask, \rs2
	orr	\dst, \dst, \mask		// dst|=(aff2>>rs2)
	and	\mask, \mpidr, #0xff00000000	// mask = aff3
	lsr	\mask ,\mask, \rs3
	orr	\dst, \dst, \mask		// dst|=(aff3>>rs3)
	.endm
/*
 * Save CPU state in the provided sleep_stack_data area, and publish its
 * location for cpu_resume()'s use in sleep_save_stash.
 *
 * cpu_resume() will restore this saved state, and return. Because the
 * link-register is saved and restored, it will appear to return from this
 * function. So that the caller can tell the suspend/resume paths apart,
 * __cpu_suspend_enter() will always return a non-zero value, whereas the
 * path through cpu_resume() will return 0.
 *
 *  x0 = struct sleep_stack_data area
 */
ENTRY(__cpu_suspend_enter)
	stp	x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS]
	stp	x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16]
	stp	x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32]
	stp	x23, x24, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+48]
	stp	x25, x26, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+64]
	stp	x27, x28, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+80]

	/* save the sp in cpu_suspend_ctx */
	mov	x2, sp
	str	x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]

	/* find the mpidr_hash */
	ldr_l	x1, sleep_save_stash
	mrs	x7, mpidr_el1
	adr_l	x9, mpidr_hash
	ldr	x10, [x9, #MPIDR_HASH_MASK]
	/*
	 * Following code relies on the struct mpidr_hash
	 * members size.
	 */
	ldp	w3, w4, [x9, #MPIDR_HASH_SHIFTS]
	ldp	w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
	add	x1, x1, x8, lsl #3

	str	x0, [x1]
	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
	stp	x29, lr, [sp, #-16]!
	bl	cpu_do_suspend
	ldp	x29, lr, [sp], #16
	mov	x0, #1
	ret
ENDPROC(__cpu_suspend_enter)

	.pushsection ".idmap.text", "awx"
ENTRY(cpu_resume)
	bl	el2_setup		// if in EL2 drop to EL1 cleanly
	bl	__cpu_setup
	/* enable the MMU early - so we can access sleep_save_stash by va */
	bl	__enable_mmu
	ldr	x8, =_cpu_resume
	br	x8
ENDPROC(cpu_resume)
	.ltorg
	.popsection

ENTRY(_cpu_resume)
	mrs	x1, mpidr_el1
	adr_l	x8, mpidr_hash		// x8 = struct mpidr_hash virt address

	/* retrieve mpidr_hash members to compute the hash */
	ldr	x2, [x8, #MPIDR_HASH_MASK]
	ldp	w3, w4, [x8, #MPIDR_HASH_SHIFTS]
	ldp	w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
	compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2

	/* x7 contains hash index, let's use it to grab context pointer */
	ldr_l	x0, sleep_save_stash
	ldr	x0, [x0, x7, lsl #3]
	add	x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS
	add	x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
	/* load sp from context */
	ldr	x2, [x0, #CPU_CTX_SP]
	mov	sp, x2
	/*
	 * cpu_do_resume expects x0 to contain context address pointer
	 */
	bl	cpu_do_resume

#ifdef CONFIG_KASAN
	mov	x0, sp
	bl	kasan_unpoison_task_stack_below
#endif

	ldp	x19, x20, [x29, #16]
	ldp	x21, x22, [x29, #32]
	ldp	x23, x24, [x29, #48]
	ldp	x25, x26, [x29, #64]
	ldp	x27, x28, [x29, #80]
	ldp	x29, lr, [x29]
	mov	x0, #0
	ret
ENDPROC(_cpu_resume)
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
95322526 LP	2	#include <linux/errno.h>
	3	#include <linux/linkage.h>
	4	#include <asm/asm-offsets.h>
	5	#include <asm/assembler.h>
	6
	7	.text
	8	/*
	9	* Implementation of MPIDR_EL1 hash algorithm through shifting
	10	* and OR'ing.
	11	*
	12	* @dst: register containing hash result
	13	* @rs0: register containing affinity level 0 bit shift
	14	* @rs1: register containing affinity level 1 bit shift
	15	* @rs2: register containing affinity level 2 bit shift
	16	* @rs3: register containing affinity level 3 bit shift
	17	* @mpidr: register containing MPIDR_EL1 value
	18	* @mask: register containing MPIDR mask
	19	*
	20	* Pseudo C-code:
	21	*
	22	*u32 dst;
	23	*
	24	*compute_mpidr_hash(u32 rs0, u32 rs1, u32 rs2, u32 rs3, u64 mpidr, u64 mask) {
	25	* u32 aff0, aff1, aff2, aff3;
	26	* u64 mpidr_masked = mpidr & mask;
	27	* aff0 = mpidr_masked & 0xff;
	28	* aff1 = mpidr_masked & 0xff00;
	29	* aff2 = mpidr_masked & 0xff0000;
	30	* aff2 = mpidr_masked & 0xff00000000;
	31	* dst = (aff0 >> rs0 \| aff1 >> rs1 \| aff2 >> rs2 \| aff3 >> rs3);
	32	*}
	33	* Input registers: rs0, rs1, rs2, rs3, mpidr, mask
	34	* Output register: dst
	35	* Note: input and output registers must be disjoint register sets
	36	(eg: a macro instance with mpidr = x1 and dst = x1 is invalid)
	37	*/
	38	.macro compute_mpidr_hash dst, rs0, rs1, rs2, rs3, mpidr, mask
	39	and \mpidr, \mpidr, \mask // mask out MPIDR bits
	40	and \dst, \mpidr, #0xff // mask=aff0
	41	lsr \dst ,\dst, \rs0 // dst=aff0>>rs0
	42	and \mask, \mpidr, #0xff00 // mask = aff1
	43	lsr \mask ,\mask, \rs1
	44	orr \dst, \dst, \mask // dst\|=(aff1>>rs1)
	45	and \mask, \mpidr, #0xff0000 // mask = aff2
	46	lsr \mask ,\mask, \rs2
	47	orr \dst, \dst, \mask // dst\|=(aff2>>rs2)
	48	and \mask, \mpidr, #0xff00000000 // mask = aff3
	49	lsr \mask ,\mask, \rs3
	50	orr \dst, \dst, \mask // dst\|=(aff3>>rs3)
	51	.endm
	52	/*
adc9b2df JM	53	* Save CPU state in the provided sleep_stack_data area, and publish its
adc9b2df JM	54	* location for cpu_resume()'s use in sleep_save_stash.
95322526	55	*
adc9b2df JM	56	* cpu_resume() will restore this saved state, and return. Because the
	57	* link-register is saved and restored, it will appear to return from this
	58	* function. So that the caller can tell the suspend/resume paths apart,
	59	* __cpu_suspend_enter() will always return a non-zero value, whereas the
	60	* path through cpu_resume() will return 0.
	61	*
	62	* x0 = struct sleep_stack_data area
95322526	63	*/
714f5992	64	ENTRY(__cpu_suspend_enter)
adc9b2df JM	65	stp x29, lr, [x0, #SLEEP_STACK_DATA_CALLEE_REGS]
	66	stp x19, x20, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+16]
	67	stp x21, x22, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+32]
	68	stp x23, x24, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+48]
	69	stp x25, x26, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+64]
	70	stp x27, x28, [x0,#SLEEP_STACK_DATA_CALLEE_REGS+80]
	71
	72	/* save the sp in cpu_suspend_ctx */
95322526	73	mov x2, sp
adc9b2df JM	74	str x2, [x0, #SLEEP_STACK_DATA_SYSTEM_REGS + CPU_CTX_SP]
	75
	76	/* find the mpidr_hash */
b5fe2429	77	ldr_l x1, sleep_save_stash
95322526	78	mrs x7, mpidr_el1
b5fe2429	79	adr_l x9, mpidr_hash
95322526 LP	80	ldr x10, [x9, #MPIDR_HASH_MASK]
	81	/*
	82	* Following code relies on the struct mpidr_hash
	83	* members size.
	84	*/
	85	ldp w3, w4, [x9, #MPIDR_HASH_SHIFTS]
	86	ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
	87	compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
714f5992	88	add x1, x1, x8, lsl #3
adc9b2df	89
cabe1c81 JM	90	str x0, [x1]
cabe1c81 JM	91	add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
adc9b2df	92	stp x29, lr, [sp, #-16]!
cabe1c81	93	bl cpu_do_suspend
adc9b2df JM	94	ldp x29, lr, [sp], #16
adc9b2df JM	95	mov x0, #1
95322526	96	ret
714f5992	97	ENDPROC(__cpu_suspend_enter)
95322526	98
439e70e2	99	.pushsection ".idmap.text", "awx"
95322526 LP	100	ENTRY(cpu_resume)
95322526 LP	101	bl el2_setup // if in EL2 drop to EL1 cleanly
b5fe2429	102	bl __cpu_setup
cabe1c81	103	/* enable the MMU early - so we can access sleep_save_stash by va */
9dcf7914	104	bl __enable_mmu
bc9f3d77 AB	105	ldr x8, =_cpu_resume
bc9f3d77 AB	106	br x8
9dcf7914	107	ENDPROC(cpu_resume)
bc9f3d77 AB	108	.ltorg
	109	.popsection
	110
dc002475	111	ENTRY(_cpu_resume)
95322526	112	mrs x1, mpidr_el1
b5fe2429 AB	113	adr_l x8, mpidr_hash // x8 = struct mpidr_hash virt address
	114
	115	/* retrieve mpidr_hash members to compute the hash */
95322526 LP	116	ldr x2, [x8, #MPIDR_HASH_MASK]
	117	ldp w3, w4, [x8, #MPIDR_HASH_SHIFTS]
	118	ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
	119	compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
b5fe2429 AB	120
b5fe2429 AB	121	/* x7 contains hash index, let's use it to grab context pointer */
cabe1c81	122	ldr_l x0, sleep_save_stash
95322526	123	ldr x0, [x0, x7, lsl #3]
adc9b2df JM	124	add x29, x0, #SLEEP_STACK_DATA_CALLEE_REGS
adc9b2df JM	125	add x0, x0, #SLEEP_STACK_DATA_SYSTEM_REGS
95322526 LP	126	/* load sp from context */
95322526 LP	127	ldr x2, [x0, #CPU_CTX_SP]
95322526 LP	128	mov sp, x2
95322526 LP	129	/*
cabe1c81	130	* cpu_do_resume expects x0 to contain context address pointer
95322526	131	*/
cabe1c81 JM	132	bl cpu_do_resume
	133
	134	#ifdef CONFIG_KASAN
	135	mov x0, sp
9f7d416c	136	bl kasan_unpoison_task_stack_below
cabe1c81 JM	137	#endif
cabe1c81 JM	138
adc9b2df JM	139	ldp x19, x20, [x29, #16]
	140	ldp x21, x22, [x29, #32]
	141	ldp x23, x24, [x29, #48]
	142	ldp x25, x26, [x29, #64]
	143	ldp x27, x28, [x29, #80]
	144	ldp x29, lr, [x29]
cabe1c81 JM	145	mov x0, #0
	146	ret
	147	ENDPROC(_cpu_resume)