[linux-2.6-block.git] / arch / arm64 / include / asm / fpsimdmacros.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * FP/SIMD state saving and restoring macros
 *
 * Copyright (C) 2012 ARM Ltd.
 * Author: Catalin Marinas <catalin.marinas@arm.com>
 */

.macro fpsimd_save state, tmpnr
	stp	q0, q1, [\state, #16 * 0]
	stp	q2, q3, [\state, #16 * 2]
	stp	q4, q5, [\state, #16 * 4]
	stp	q6, q7, [\state, #16 * 6]
	stp	q8, q9, [\state, #16 * 8]
	stp	q10, q11, [\state, #16 * 10]
	stp	q12, q13, [\state, #16 * 12]
	stp	q14, q15, [\state, #16 * 14]
	stp	q16, q17, [\state, #16 * 16]
	stp	q18, q19, [\state, #16 * 18]
	stp	q20, q21, [\state, #16 * 20]
	stp	q22, q23, [\state, #16 * 22]
	stp	q24, q25, [\state, #16 * 24]
	stp	q26, q27, [\state, #16 * 26]
	stp	q28, q29, [\state, #16 * 28]
	stp	q30, q31, [\state, #16 * 30]!
	mrs	x\tmpnr, fpsr
	str	w\tmpnr, [\state, #16 * 2]
	mrs	x\tmpnr, fpcr
	str	w\tmpnr, [\state, #16 * 2 + 4]
.endm

.macro fpsimd_restore_fpcr state, tmp
	/*
	 * Writes to fpcr may be self-synchronising, so avoid restoring
	 * the register if it hasn't changed.
	 */
	mrs	\tmp, fpcr
	cmp	\tmp, \state
	b.eq	9999f
	msr	fpcr, \state
9999:
.endm

/* Clobbers \state */
.macro fpsimd_restore state, tmpnr
	ldp	q0, q1, [\state, #16 * 0]
	ldp	q2, q3, [\state, #16 * 2]
	ldp	q4, q5, [\state, #16 * 4]
	ldp	q6, q7, [\state, #16 * 6]
	ldp	q8, q9, [\state, #16 * 8]
	ldp	q10, q11, [\state, #16 * 10]
	ldp	q12, q13, [\state, #16 * 12]
	ldp	q14, q15, [\state, #16 * 14]
	ldp	q16, q17, [\state, #16 * 16]
	ldp	q18, q19, [\state, #16 * 18]
	ldp	q20, q21, [\state, #16 * 20]
	ldp	q22, q23, [\state, #16 * 22]
	ldp	q24, q25, [\state, #16 * 24]
	ldp	q26, q27, [\state, #16 * 26]
	ldp	q28, q29, [\state, #16 * 28]
	ldp	q30, q31, [\state, #16 * 30]!
	ldr	w\tmpnr, [\state, #16 * 2]
	msr	fpsr, x\tmpnr
	ldr	w\tmpnr, [\state, #16 * 2 + 4]
	fpsimd_restore_fpcr x\tmpnr, \state
.endm

/* Sanity-check macros to help avoid encoding garbage instructions */

.macro _check_general_reg nr
	.if (\nr) < 0 || (\nr) > 30
		.error "Bad register number \nr."
	.endif
.endm

.macro _sve_check_zreg znr
	.if (\znr) < 0 || (\znr) > 31
		.error "Bad Scalable Vector Extension vector register number \znr."
	.endif
.endm

.macro _sve_check_preg pnr
	.if (\pnr) < 0 || (\pnr) > 15
		.error "Bad Scalable Vector Extension predicate register number \pnr."
	.endif
.endm

.macro _check_num n, min, max
	.if (\n) < (\min) || (\n) > (\max)
		.error "Number \n out of range [\min,\max]"
	.endif
.endm

/* SVE instruction encodings for non-SVE-capable assemblers */

/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_v nz, nxbase, offset=0
	_sve_check_zreg \nz
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0xe5804000			\
		| (\nz)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_v nz, nxbase, offset=0
	_sve_check_zreg \nz
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0x85804000			\
		| (\nz)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_p np, nxbase, offset=0
	_sve_check_preg \np
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0xe5800000			\
		| (\np)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_p np, nxbase, offset=0
	_sve_check_preg \np
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0x85800000			\
		| (\np)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* RDVL X\nx, #\imm */
.macro _sve_rdvl nx, imm
	_check_general_reg \nx
	_check_num (\imm), -0x20, 0x1f
	.inst	0x04bf5000			\
		| (\nx)				\
		| (((\imm) & 0x3f) << 5)
.endm

/* RDFFR (unpredicated): RDFFR P\np.B */
.macro _sve_rdffr np
	_sve_check_preg \np
	.inst	0x2519f000			\
		| (\np)
.endm

/* WRFFR P\np.B */
.macro _sve_wrffr np
	_sve_check_preg \np
	.inst	0x25289000			\
		| ((\np) << 5)
.endm

.macro __for from:req, to:req
	.if (\from) == (\to)
		_for__body \from
	.else
		__for \from, (\from) + ((\to) - (\from)) / 2
		__for (\from) + ((\to) - (\from)) / 2 + 1, \to
	.endif
.endm

.macro _for var:req, from:req, to:req, insn:vararg
	.macro _for__body \var:req
		\insn
	.endm

	__for \from, \to

	.purgem _for__body
.endm

.macro sve_save nxbase, xpfpsr, nxtmp
 _for n, 0, 31,	_sve_str_v	\n, \nxbase, \n - 34
 _for n, 0, 15,	_sve_str_p	\n, \nxbase, \n - 16
		_sve_rdffr	0
		_sve_str_p	0, \nxbase
		_sve_ldr_p	0, \nxbase, -16

		mrs		x\nxtmp, fpsr
		str		w\nxtmp, [\xpfpsr]
		mrs		x\nxtmp, fpcr
		str		w\nxtmp, [\xpfpsr, #4]
.endm

.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
		mrs_s		x\nxtmp, SYS_ZCR_EL1
		bic		\xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK
		orr		\xtmp2, \xtmp2, \xvqminus1
		cmp		\xtmp2, x\nxtmp
		b.eq		921f
		msr_s		SYS_ZCR_EL1, \xtmp2	// self-synchronising
921:
 _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34
		_sve_ldr_p	0, \nxbase
		_sve_wrffr	0
 _for n, 0, 15,	_sve_ldr_p	\n, \nxbase, \n - 16

		ldr		w\nxtmp, [\xpfpsr]
		msr		fpsr, x\nxtmp
		ldr		w\nxtmp, [\xpfpsr, #4]
		msr		fpcr, x\nxtmp
.endm
Commit	Line	Data
caab277b	1	/* SPDX-License-Identifier: GPL-2.0-only */
cfc5180e MZ	2	/*
	3	* FP/SIMD state saving and restoring macros
	4	*
	5	* Copyright (C) 2012 ARM Ltd.
	6	* Author: Catalin Marinas <catalin.marinas@arm.com>
cfc5180e MZ	7	*/
	8
	9	.macro fpsimd_save state, tmpnr
	10	stp q0, q1, [\state, #16 * 0]
	11	stp q2, q3, [\state, #16 * 2]
	12	stp q4, q5, [\state, #16 * 4]
	13	stp q6, q7, [\state, #16 * 6]
	14	stp q8, q9, [\state, #16 * 8]
	15	stp q10, q11, [\state, #16 * 10]
	16	stp q12, q13, [\state, #16 * 12]
	17	stp q14, q15, [\state, #16 * 14]
	18	stp q16, q17, [\state, #16 * 16]
	19	stp q18, q19, [\state, #16 * 18]
	20	stp q20, q21, [\state, #16 * 20]
	21	stp q22, q23, [\state, #16 * 22]
	22	stp q24, q25, [\state, #16 * 24]
	23	stp q26, q27, [\state, #16 * 26]
	24	stp q28, q29, [\state, #16 * 28]
	25	stp q30, q31, [\state, #16 * 30]!
	26	mrs x\tmpnr, fpsr
	27	str w\tmpnr, [\state, #16 * 2]
	28	mrs x\tmpnr, fpcr
	29	str w\tmpnr, [\state, #16 * 2 + 4]
	30	.endm
	31
5959e257 WD	32	.macro fpsimd_restore_fpcr state, tmp
	33	/*
	34	* Writes to fpcr may be self-synchronising, so avoid restoring
	35	* the register if it hasn't changed.
	36	*/
	37	mrs \tmp, fpcr
	38	cmp \tmp, \state
	39	b.eq 9999f
	40	msr fpcr, \state
	41	9999:
	42	.endm
	43
	44	/* Clobbers \state */
cfc5180e MZ	45	.macro fpsimd_restore state, tmpnr
	46	ldp q0, q1, [\state, #16 * 0]
	47	ldp q2, q3, [\state, #16 * 2]
	48	ldp q4, q5, [\state, #16 * 4]
	49	ldp q6, q7, [\state, #16 * 6]
	50	ldp q8, q9, [\state, #16 * 8]
	51	ldp q10, q11, [\state, #16 * 10]
	52	ldp q12, q13, [\state, #16 * 12]
	53	ldp q14, q15, [\state, #16 * 14]
	54	ldp q16, q17, [\state, #16 * 16]
	55	ldp q18, q19, [\state, #16 * 18]
	56	ldp q20, q21, [\state, #16 * 20]
	57	ldp q22, q23, [\state, #16 * 22]
	58	ldp q24, q25, [\state, #16 * 24]
	59	ldp q26, q27, [\state, #16 * 26]
	60	ldp q28, q29, [\state, #16 * 28]
	61	ldp q30, q31, [\state, #16 * 30]!
	62	ldr w\tmpnr, [\state, #16 * 2]
	63	msr fpsr, x\tmpnr
	64	ldr w\tmpnr, [\state, #16 * 2 + 4]
5959e257	65	fpsimd_restore_fpcr x\tmpnr, \state
cfc5180e	66	.endm
1fc5dce7 DM	67
	68	/* Sanity-check macros to help avoid encoding garbage instructions */
	69
	70	.macro _check_general_reg nr
	71	.if (\nr) < 0 \|\| (\nr) > 30
	72	.error "Bad register number \nr."
	73	.endif
	74	.endm
	75
	76	.macro _sve_check_zreg znr
	77	.if (\znr) < 0 \|\| (\znr) > 31
	78	.error "Bad Scalable Vector Extension vector register number \znr."
	79	.endif
	80	.endm
	81
	82	.macro _sve_check_preg pnr
	83	.if (\pnr) < 0 \|\| (\pnr) > 15
	84	.error "Bad Scalable Vector Extension predicate register number \pnr."
	85	.endif
	86	.endm
	87
	88	.macro _check_num n, min, max
	89	.if (\n) < (\min) \|\| (\n) > (\max)
	90	.error "Number \n out of range [\min,\max]"
	91	.endif
	92	.endm
	93
	94	/* SVE instruction encodings for non-SVE-capable assemblers */
	95
	96	/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
	97	.macro _sve_str_v nz, nxbase, offset=0
	98	_sve_check_zreg \nz
	99	_check_general_reg \nxbase
	100	_check_num (\offset), -0x100, 0xff
	101	.inst 0xe5804000 \
	102	\| (\nz) \
	103	\| ((\nxbase) << 5) \
	104	\| (((\offset) & 7) << 10) \
	105	\| (((\offset) & 0x1f8) << 13)
	106	.endm
	107
	108	/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
	109	.macro _sve_ldr_v nz, nxbase, offset=0
	110	_sve_check_zreg \nz
	111	_check_general_reg \nxbase
	112	_check_num (\offset), -0x100, 0xff
	113	.inst 0x85804000 \
	114	\| (\nz) \
	115	\| ((\nxbase) << 5) \
	116	\| (((\offset) & 7) << 10) \
	117	\| (((\offset) & 0x1f8) << 13)
	118	.endm
	119
	120	/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
	121	.macro _sve_str_p np, nxbase, offset=0
	122	_sve_check_preg \np
	123	_check_general_reg \nxbase
	124	_check_num (\offset), -0x100, 0xff
	125	.inst 0xe5800000 \
	126	\| (\np) \
	127	\| ((\nxbase) << 5) \
	128	\| (((\offset) & 7) << 10) \
	129	\| (((\offset) & 0x1f8) << 13)
	130	.endm
131
132	/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
133	.macro _sve_ldr_p np, nxbase, offset=0
134	_sve_check_preg \np
135	_check_general_reg \nxbase
136	_check_num (\offset), -0x100, 0xff
137	.inst 0x85800000 \
138	\| (\np) \
139	\| ((\nxbase) << 5) \
140	\| (((\offset) & 7) << 10) \
141	\| (((\offset) & 0x1f8) << 13)
142	.endm
143
144	/* RDVL X\nx, #\imm */
145	.macro _sve_rdvl nx, imm
146	_check_general_reg \nx
147	_check_num (\imm), -0x20, 0x1f
148	.inst 0x04bf5000 \
149	\| (\nx) \
150	\| (((\imm) & 0x3f) << 5)
151	.endm
152
153	/* RDFFR (unpredicated): RDFFR P\np.B */
154	.macro _sve_rdffr np
155	_sve_check_preg \np
156	.inst 0x2519f000 \
157	\| (\np)
158	.endm
159
160	/* WRFFR P\np.B */
161	.macro _sve_wrffr np
162	_sve_check_preg \np
163	.inst 0x25289000 \
164	\| ((\np) << 5)
165	.endm
166
167	.macro __for from:req, to:req
168	.if (\from) == (\to)
169	_for__body \from
170	.else
171	__for \from, (\from) + ((\to) - (\from)) / 2
172	__for (\from) + ((\to) - (\from)) / 2 + 1, \to
173	.endif
174	.endm
175
176	.macro _for var:req, from:req, to:req, insn:vararg
177	.macro _for__body \var:req
178	\insn
179	.endm
180
181	__for \from, \to
182
183	.purgem _for__body
184	.endm
185
186	.macro sve_save nxbase, xpfpsr, nxtmp
187	_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
188	_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
189	_sve_rdffr 0
190	_sve_str_p 0, \nxbase
191	_sve_ldr_p 0, \nxbase, -16
192
193	mrs x\nxtmp, fpsr
194	str w\nxtmp, [\xpfpsr]
195	mrs x\nxtmp, fpcr
196	str w\nxtmp, [\xpfpsr, #4]
197	.endm
198
159fd7b8	199	.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
1fc5dce7	200	mrs_s x\nxtmp, SYS_ZCR_EL1
159fd7b8 DM	201	bic \xtmp2, x\nxtmp, ZCR_ELx_LEN_MASK
	202	orr \xtmp2, \xtmp2, \xvqminus1
	203	cmp \xtmp2, x\nxtmp
	204	b.eq 921f
	205	msr_s SYS_ZCR_EL1, \xtmp2 // self-synchronising
	206	921:
1fc5dce7 DM	207	_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
	208	_sve_ldr_p 0, \nxbase
	209	_sve_wrffr 0
	210	_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
	211
	212	ldr w\nxtmp, [\xpfpsr]
	213	msr fpsr, x\nxtmp
	214	ldr w\nxtmp, [\xpfpsr, #4]
	215	msr fpcr, x\nxtmp
	216	.endm