[linux-2.6-block.git] / arch / arm64 / include / asm / fpsimdmacros.h

/*
 * FP/SIMD state saving and restoring macros
 *
 * Copyright (C) 2012 ARM Ltd.
 * Author: Catalin Marinas <catalin.marinas@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

.macro fpsimd_save state, tmpnr
	stp	q0, q1, [\state, #16 * 0]
	stp	q2, q3, [\state, #16 * 2]
	stp	q4, q5, [\state, #16 * 4]
	stp	q6, q7, [\state, #16 * 6]
	stp	q8, q9, [\state, #16 * 8]
	stp	q10, q11, [\state, #16 * 10]
	stp	q12, q13, [\state, #16 * 12]
	stp	q14, q15, [\state, #16 * 14]
	stp	q16, q17, [\state, #16 * 16]
	stp	q18, q19, [\state, #16 * 18]
	stp	q20, q21, [\state, #16 * 20]
	stp	q22, q23, [\state, #16 * 22]
	stp	q24, q25, [\state, #16 * 24]
	stp	q26, q27, [\state, #16 * 26]
	stp	q28, q29, [\state, #16 * 28]
	stp	q30, q31, [\state, #16 * 30]!
	mrs	x\tmpnr, fpsr
	str	w\tmpnr, [\state, #16 * 2]
	mrs	x\tmpnr, fpcr
	str	w\tmpnr, [\state, #16 * 2 + 4]
.endm

.macro fpsimd_restore_fpcr state, tmp
	/*
	 * Writes to fpcr may be self-synchronising, so avoid restoring
	 * the register if it hasn't changed.
	 */
	mrs	\tmp, fpcr
	cmp	\tmp, \state
	b.eq	9999f
	msr	fpcr, \state
9999:
.endm

/* Clobbers \state */
.macro fpsimd_restore state, tmpnr
	ldp	q0, q1, [\state, #16 * 0]
	ldp	q2, q3, [\state, #16 * 2]
	ldp	q4, q5, [\state, #16 * 4]
	ldp	q6, q7, [\state, #16 * 6]
	ldp	q8, q9, [\state, #16 * 8]
	ldp	q10, q11, [\state, #16 * 10]
	ldp	q12, q13, [\state, #16 * 12]
	ldp	q14, q15, [\state, #16 * 14]
	ldp	q16, q17, [\state, #16 * 16]
	ldp	q18, q19, [\state, #16 * 18]
	ldp	q20, q21, [\state, #16 * 20]
	ldp	q22, q23, [\state, #16 * 22]
	ldp	q24, q25, [\state, #16 * 24]
	ldp	q26, q27, [\state, #16 * 26]
	ldp	q28, q29, [\state, #16 * 28]
	ldp	q30, q31, [\state, #16 * 30]!
	ldr	w\tmpnr, [\state, #16 * 2]
	msr	fpsr, x\tmpnr
	ldr	w\tmpnr, [\state, #16 * 2 + 4]
	fpsimd_restore_fpcr x\tmpnr, \state
.endm

/* Sanity-check macros to help avoid encoding garbage instructions */

.macro _check_general_reg nr
	.if (\nr) < 0 || (\nr) > 30
		.error "Bad register number \nr."
	.endif
.endm

.macro _sve_check_zreg znr
	.if (\znr) < 0 || (\znr) > 31
		.error "Bad Scalable Vector Extension vector register number \znr."
	.endif
.endm

.macro _sve_check_preg pnr
	.if (\pnr) < 0 || (\pnr) > 15
		.error "Bad Scalable Vector Extension predicate register number \pnr."
	.endif
.endm

.macro _check_num n, min, max
	.if (\n) < (\min) || (\n) > (\max)
		.error "Number \n out of range [\min,\max]"
	.endif
.endm

/* SVE instruction encodings for non-SVE-capable assemblers */

/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_v nz, nxbase, offset=0
	_sve_check_zreg \nz
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0xe5804000			\
		| (\nz)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_v nz, nxbase, offset=0
	_sve_check_zreg \nz
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0x85804000			\
		| (\nz)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_str_p np, nxbase, offset=0
	_sve_check_preg \np
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0xe5800000			\
		| (\np)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
.macro _sve_ldr_p np, nxbase, offset=0
	_sve_check_preg \np
	_check_general_reg \nxbase
	_check_num (\offset), -0x100, 0xff
	.inst	0x85800000			\
		| (\np)				\
		| ((\nxbase) << 5)		\
		| (((\offset) & 7) << 10)	\
		| (((\offset) & 0x1f8) << 13)
.endm

/* RDVL X\nx, #\imm */
.macro _sve_rdvl nx, imm
	_check_general_reg \nx
	_check_num (\imm), -0x20, 0x1f
	.inst	0x04bf5000			\
		| (\nx)				\
		| (((\imm) & 0x3f) << 5)
.endm

/* RDFFR (unpredicated): RDFFR P\np.B */
.macro _sve_rdffr np
	_sve_check_preg \np
	.inst	0x2519f000			\
		| (\np)
.endm

/* WRFFR P\np.B */
.macro _sve_wrffr np
	_sve_check_preg \np
	.inst	0x25289000			\
		| ((\np) << 5)
.endm

.macro __for from:req, to:req
	.if (\from) == (\to)
		_for__body \from
	.else
		__for \from, (\from) + ((\to) - (\from)) / 2
		__for (\from) + ((\to) - (\from)) / 2 + 1, \to
	.endif
.endm

.macro _for var:req, from:req, to:req, insn:vararg
	.macro _for__body \var:req
		\insn
	.endm

	__for \from, \to

	.purgem _for__body
.endm

.macro sve_save nxbase, xpfpsr, nxtmp
 _for n, 0, 31,	_sve_str_v	\n, \nxbase, \n - 34
 _for n, 0, 15,	_sve_str_p	\n, \nxbase, \n - 16
		_sve_rdffr	0
		_sve_str_p	0, \nxbase
		_sve_ldr_p	0, \nxbase, -16

		mrs		x\nxtmp, fpsr
		str		w\nxtmp, [\xpfpsr]
		mrs		x\nxtmp, fpcr
		str		w\nxtmp, [\xpfpsr, #4]
.endm

.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp
		mrs_s		x\nxtmp, SYS_ZCR_EL1
		bic		x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK
		orr		x\nxtmp, x\nxtmp, \xvqminus1
		msr_s		SYS_ZCR_EL1, x\nxtmp	// self-synchronising

 _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34
		_sve_ldr_p	0, \nxbase
		_sve_wrffr	0
 _for n, 0, 15,	_sve_ldr_p	\n, \nxbase, \n - 16

		ldr		w\nxtmp, [\xpfpsr]
		msr		fpsr, x\nxtmp
		ldr		w\nxtmp, [\xpfpsr, #4]
		msr		fpcr, x\nxtmp
.endm
Commit	Line	Data
cfc5180e MZ	1	/*
	2	* FP/SIMD state saving and restoring macros
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	* Author: Catalin Marinas <catalin.marinas@arm.com>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	* GNU General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	18	*/
	19
	20	.macro fpsimd_save state, tmpnr
	21	stp q0, q1, [\state, #16 * 0]
	22	stp q2, q3, [\state, #16 * 2]
	23	stp q4, q5, [\state, #16 * 4]
	24	stp q6, q7, [\state, #16 * 6]
	25	stp q8, q9, [\state, #16 * 8]
	26	stp q10, q11, [\state, #16 * 10]
	27	stp q12, q13, [\state, #16 * 12]
	28	stp q14, q15, [\state, #16 * 14]
	29	stp q16, q17, [\state, #16 * 16]
	30	stp q18, q19, [\state, #16 * 18]
	31	stp q20, q21, [\state, #16 * 20]
	32	stp q22, q23, [\state, #16 * 22]
	33	stp q24, q25, [\state, #16 * 24]
	34	stp q26, q27, [\state, #16 * 26]
	35	stp q28, q29, [\state, #16 * 28]
	36	stp q30, q31, [\state, #16 * 30]!
	37	mrs x\tmpnr, fpsr
	38	str w\tmpnr, [\state, #16 * 2]
	39	mrs x\tmpnr, fpcr
	40	str w\tmpnr, [\state, #16 * 2 + 4]
	41	.endm
	42
5959e257 WD	43	.macro fpsimd_restore_fpcr state, tmp
	44	/*
	45	* Writes to fpcr may be self-synchronising, so avoid restoring
	46	* the register if it hasn't changed.
	47	*/
	48	mrs \tmp, fpcr
	49	cmp \tmp, \state
	50	b.eq 9999f
	51	msr fpcr, \state
	52	9999:
	53	.endm
	54
	55	/* Clobbers \state */
cfc5180e MZ	56	.macro fpsimd_restore state, tmpnr
	57	ldp q0, q1, [\state, #16 * 0]
	58	ldp q2, q3, [\state, #16 * 2]
	59	ldp q4, q5, [\state, #16 * 4]
	60	ldp q6, q7, [\state, #16 * 6]
	61	ldp q8, q9, [\state, #16 * 8]
	62	ldp q10, q11, [\state, #16 * 10]
	63	ldp q12, q13, [\state, #16 * 12]
	64	ldp q14, q15, [\state, #16 * 14]
	65	ldp q16, q17, [\state, #16 * 16]
	66	ldp q18, q19, [\state, #16 * 18]
	67	ldp q20, q21, [\state, #16 * 20]
	68	ldp q22, q23, [\state, #16 * 22]
	69	ldp q24, q25, [\state, #16 * 24]
	70	ldp q26, q27, [\state, #16 * 26]
	71	ldp q28, q29, [\state, #16 * 28]
	72	ldp q30, q31, [\state, #16 * 30]!
	73	ldr w\tmpnr, [\state, #16 * 2]
	74	msr fpsr, x\tmpnr
	75	ldr w\tmpnr, [\state, #16 * 2 + 4]
5959e257	76	fpsimd_restore_fpcr x\tmpnr, \state
cfc5180e	77	.endm
1fc5dce7 DM	78
	79	/* Sanity-check macros to help avoid encoding garbage instructions */
	80
	81	.macro _check_general_reg nr
	82	.if (\nr) < 0 \|\| (\nr) > 30
	83	.error "Bad register number \nr."
	84	.endif
	85	.endm
	86
	87	.macro _sve_check_zreg znr
	88	.if (\znr) < 0 \|\| (\znr) > 31
	89	.error "Bad Scalable Vector Extension vector register number \znr."
	90	.endif
	91	.endm
	92
	93	.macro _sve_check_preg pnr
	94	.if (\pnr) < 0 \|\| (\pnr) > 15
	95	.error "Bad Scalable Vector Extension predicate register number \pnr."
	96	.endif
	97	.endm
	98
	99	.macro _check_num n, min, max
	100	.if (\n) < (\min) \|\| (\n) > (\max)
	101	.error "Number \n out of range [\min,\max]"
	102	.endif
	103	.endm
	104
	105	/* SVE instruction encodings for non-SVE-capable assemblers */
	106
	107	/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
	108	.macro _sve_str_v nz, nxbase, offset=0
	109	_sve_check_zreg \nz
	110	_check_general_reg \nxbase
	111	_check_num (\offset), -0x100, 0xff
	112	.inst 0xe5804000 \
	113	\| (\nz) \
	114	\| ((\nxbase) << 5) \
	115	\| (((\offset) & 7) << 10) \
	116	\| (((\offset) & 0x1f8) << 13)
	117	.endm
	118
	119	/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
	120	.macro _sve_ldr_v nz, nxbase, offset=0
	121	_sve_check_zreg \nz
	122	_check_general_reg \nxbase
	123	_check_num (\offset), -0x100, 0xff
	124	.inst 0x85804000 \
	125	\| (\nz) \
	126	\| ((\nxbase) << 5) \
	127	\| (((\offset) & 7) << 10) \
	128	\| (((\offset) & 0x1f8) << 13)
	129	.endm
	130
	131	/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
	132	.macro _sve_str_p np, nxbase, offset=0
	133	_sve_check_preg \np
	134	_check_general_reg \nxbase
	135	_check_num (\offset), -0x100, 0xff
	136	.inst 0xe5800000 \
	137	\| (\np) \
	138	\| ((\nxbase) << 5) \
	139	\| (((\offset) & 7) << 10) \
	140	\| (((\offset) & 0x1f8) << 13)
	141	.endm
142
143	/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
144	.macro _sve_ldr_p np, nxbase, offset=0
145	_sve_check_preg \np
146	_check_general_reg \nxbase
147	_check_num (\offset), -0x100, 0xff
148	.inst 0x85800000 \
149	\| (\np) \
150	\| ((\nxbase) << 5) \
151	\| (((\offset) & 7) << 10) \
152	\| (((\offset) & 0x1f8) << 13)
153	.endm
154
155	/* RDVL X\nx, #\imm */
156	.macro _sve_rdvl nx, imm
157	_check_general_reg \nx
158	_check_num (\imm), -0x20, 0x1f
159	.inst 0x04bf5000 \
160	\| (\nx) \
161	\| (((\imm) & 0x3f) << 5)
162	.endm
163
164	/* RDFFR (unpredicated): RDFFR P\np.B */
165	.macro _sve_rdffr np
166	_sve_check_preg \np
167	.inst 0x2519f000 \
168	\| (\np)
169	.endm
170
171	/* WRFFR P\np.B */
172	.macro _sve_wrffr np
173	_sve_check_preg \np
174	.inst 0x25289000 \
175	\| ((\np) << 5)
176	.endm
177
178	.macro __for from:req, to:req
179	.if (\from) == (\to)
180	_for__body \from
181	.else
182	__for \from, (\from) + ((\to) - (\from)) / 2
183	__for (\from) + ((\to) - (\from)) / 2 + 1, \to
184	.endif
185	.endm
186
187	.macro _for var:req, from:req, to:req, insn:vararg
188	.macro _for__body \var:req
189	\insn
190	.endm
191
192	__for \from, \to
193
194	.purgem _for__body
195	.endm
196
197	.macro sve_save nxbase, xpfpsr, nxtmp
198	_for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
199	_for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
200	_sve_rdffr 0
201	_sve_str_p 0, \nxbase
202	_sve_ldr_p 0, \nxbase, -16
203
204	mrs x\nxtmp, fpsr
205	str w\nxtmp, [\xpfpsr]
206	mrs x\nxtmp, fpcr
207	str w\nxtmp, [\xpfpsr, #4]
208	.endm
209
210	.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp
211	mrs_s x\nxtmp, SYS_ZCR_EL1
212	bic x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK
213	orr x\nxtmp, x\nxtmp, \xvqminus1
214	msr_s SYS_ZCR_EL1, x\nxtmp // self-synchronising
215
216	_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
217	_sve_ldr_p 0, \nxbase
218	_sve_wrffr 0
219	_for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
220
221	ldr w\nxtmp, [\xpfpsr]
222	msr fpsr, x\nxtmp
223	ldr w\nxtmp, [\xpfpsr, #4]
224	msr fpcr, x\nxtmp
225	.endm