[linux-2.6-block.git] / arch / arm64 / crypto / aes-cipher-core.S

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Scalar AES core transform
 *
 * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/cache.h>

	.text

	rk		.req	x0
	out		.req	x1
	in		.req	x2
	rounds		.req	x3
	tt		.req	x2

	.macro		__pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
	.ifc		\op\shift, b0
	ubfiz		\reg0, \in0, #2, #8
	ubfiz		\reg1, \in1e, #2, #8
	.else
	ubfx		\reg0, \in0, #\shift, #8
	ubfx		\reg1, \in1e, #\shift, #8
	.endif

	/*
	 * AArch64 cannot do byte size indexed loads from a table containing
	 * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
	 * valid instruction. So perform the shift explicitly first for the
	 * high bytes (the low byte is shifted implicitly by using ubfiz rather
	 * than ubfx above)
	 */
	.ifnc		\op, b
	ldr		\reg0, [tt, \reg0, uxtw #2]
	ldr		\reg1, [tt, \reg1, uxtw #2]
	.else
	.if		\shift > 0
	lsl		\reg0, \reg0, #2
	lsl		\reg1, \reg1, #2
	.endif
	ldrb		\reg0, [tt, \reg0, uxtw]
	ldrb		\reg1, [tt, \reg1, uxtw]
	.endif
	.endm

	.macro		__pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
	ubfx		\reg0, \in0, #\shift, #8
	ubfx		\reg1, \in1d, #\shift, #8
	ldr\op		\reg0, [tt, \reg0, uxtw #\sz]
	ldr\op		\reg1, [tt, \reg1, uxtw #\sz]
	.endm

	.macro		__hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
	ldp		\out0, \out1, [rk], #8

	__pair\enc	\sz, \op, w12, w13, \in0, \in1, \in3, 0
	__pair\enc	\sz, \op, w14, w15, \in1, \in2, \in0, 8
	__pair\enc	\sz, \op, w16, w17, \in2, \in3, \in1, 16
	__pair\enc	\sz, \op, \t0, \t1, \in3, \in0, \in2, 24

	eor		\out0, \out0, w12
	eor		\out1, \out1, w13
	eor		\out0, \out0, w14, ror #24
	eor		\out1, \out1, w15, ror #24
	eor		\out0, \out0, w16, ror #16
	eor		\out1, \out1, w17, ror #16
	eor		\out0, \out0, \t0, ror #8
	eor		\out1, \out1, \t1, ror #8
	.endm

	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
	.endm

	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
	.endm

	.macro		do_crypt, round, ttab, ltab, bsz
	ldp		w4, w5, [in]
	ldp		w6, w7, [in, #8]
	ldp		w8, w9, [rk], #16
	ldp		w10, w11, [rk, #-8]

CPU_BE(	rev		w4, w4		)
CPU_BE(	rev		w5, w5		)
CPU_BE(	rev		w6, w6		)
CPU_BE(	rev		w7, w7		)

	eor		w4, w4, w8
	eor		w5, w5, w9
	eor		w6, w6, w10
	eor		w7, w7, w11

	adr_l		tt, \ttab

	tbnz		rounds, #1, 1f

0:	\round		w8, w9, w10, w11, w4, w5, w6, w7
	\round		w4, w5, w6, w7, w8, w9, w10, w11

1:	subs		rounds, rounds, #4
	\round		w8, w9, w10, w11, w4, w5, w6, w7
	b.ls		3f
2:	\round		w4, w5, w6, w7, w8, w9, w10, w11
	b		0b
3:	adr_l		tt, \ltab
	\round		w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b

CPU_BE(	rev		w4, w4		)
CPU_BE(	rev		w5, w5		)
CPU_BE(	rev		w6, w6		)
CPU_BE(	rev		w7, w7		)

	stp		w4, w5, [out]
	stp		w6, w7, [out, #8]
	ret
	.endm

SYM_FUNC_START(__aes_arm64_encrypt)
	do_crypt	fround, crypto_ft_tab, crypto_ft_tab + 1, 2
SYM_FUNC_END(__aes_arm64_encrypt)

	.align		5
SYM_FUNC_START(__aes_arm64_decrypt)
	do_crypt	iround, crypto_it_tab, crypto_aes_inv_sbox, 0
SYM_FUNC_END(__aes_arm64_decrypt)
Commit	Line	Data
d2912cb1	1	/* SPDX-License-Identifier: GPL-2.0-only */
bed593c0 AB	2	/*
	3	* Scalar AES core transform
	4	*
	5	* Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
bed593c0 AB	6	*/
	7
	8	#include <linux/linkage.h>
	9	#include <asm/assembler.h>
7c83d689	10	#include <asm/cache.h>
bed593c0 AB	11
	12	.text
	13
	14	rk .req x0
	15	out .req x1
	16	in .req x2
	17	rounds .req x3
7c83d689	18	tt .req x2
bed593c0	19
7c83d689 AB	20	.macro __pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
	21	.ifc \op\shift, b0
	22	ubfiz \reg0, \in0, #2, #8
	23	ubfiz \reg1, \in1e, #2, #8
	24	.else
c458c4ad	25	ubfx \reg0, \in0, #\shift, #8
c458c4ad	26	ubfx \reg1, \in1e, #\shift, #8
bed593c0	27	.endif
7c83d689 AB	28
	29	/*
	30	* AArch64 cannot do byte size indexed loads from a table containing
	31	* 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
	32	* valid instruction. So perform the shift explicitly first for the
	33	* high bytes (the low byte is shifted implicitly by using ubfiz rather
	34	* than ubfx above)
	35	*/
	36	.ifnc \op, b
c458c4ad AB	37	ldr \reg0, [tt, \reg0, uxtw #2]
c458c4ad AB	38	ldr \reg1, [tt, \reg1, uxtw #2]
7c83d689 AB	39	.else
	40	.if \shift > 0
	41	lsl \reg0, \reg0, #2
	42	lsl \reg1, \reg1, #2
	43	.endif
	44	ldrb \reg0, [tt, \reg0, uxtw]
	45	ldrb \reg1, [tt, \reg1, uxtw]
	46	.endif
c458c4ad	47	.endm
bed593c0	48
7c83d689 AB	49	.macro __pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
	50	ubfx \reg0, \in0, #\shift, #8
	51	ubfx \reg1, \in1d, #\shift, #8
	52	ldr\op \reg0, [tt, \reg0, uxtw #\sz]
	53	ldr\op \reg1, [tt, \reg1, uxtw #\sz]
	54	.endm
	55
	56	.macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
c458c4ad	57	ldp \out0, \out1, [rk], #8
bed593c0	58
7c83d689 AB	59	__pair\enc \sz, \op, w12, w13, \in0, \in1, \in3, 0
	60	__pair\enc \sz, \op, w14, w15, \in1, \in2, \in0, 8
	61	__pair\enc \sz, \op, w16, w17, \in2, \in3, \in1, 16
	62	__pair\enc \sz, \op, \t0, \t1, \in3, \in0, \in2, 24
	63
	64	eor \out0, \out0, w12
	65	eor \out1, \out1, w13
	66	eor \out0, \out0, w14, ror #24
	67	eor \out1, \out1, w15, ror #24
	68	eor \out0, \out0, w16, ror #16
	69	eor \out1, \out1, w17, ror #16
c458c4ad	70	eor \out0, \out0, \t0, ror #8
bed593c0 AB	71	eor \out1, \out1, \t1, ror #8
	72	.endm
	73
7c83d689 AB	74	.macro fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
	75	__hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
	76	__hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
bed593c0 AB	77	.endm
bed593c0 AB	78
7c83d689 AB	79	.macro iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
	80	__hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
	81	__hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
bed593c0 AB	82	.endm
bed593c0 AB	83
7c83d689 AB	84	.macro do_crypt, round, ttab, ltab, bsz
	85	ldp w4, w5, [in]
	86	ldp w6, w7, [in, #8]
	87	ldp w8, w9, [rk], #16
	88	ldp w10, w11, [rk, #-8]
bed593c0	89
7c83d689	90	CPU_BE( rev w4, w4 )
bed593c0 AB	91	CPU_BE( rev w5, w5 )
	92	CPU_BE( rev w6, w6 )
	93	CPU_BE( rev w7, w7 )
bed593c0	94
7c83d689	95	eor w4, w4, w8
bed593c0 AB	96	eor w5, w5, w9
	97	eor w6, w6, w10
	98	eor w7, w7, w11
bed593c0	99
262ea4f6	100	adr_l tt, \ttab
bed593c0 AB	101
	102	tbnz rounds, #1, 1f
	103
7c83d689 AB	104	0: \round w8, w9, w10, w11, w4, w5, w6, w7
7c83d689 AB	105	\round w4, w5, w6, w7, w8, w9, w10, w11
bed593c0 AB	106
bed593c0 AB	107	1: subs rounds, rounds, #4
7c83d689 AB	108	\round w8, w9, w10, w11, w4, w5, w6, w7
	109	b.ls 3f
	110	2: \round w4, w5, w6, w7, w8, w9, w10, w11
	111	b 0b
	112	3: adr_l tt, \ltab
	113	\round w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
	114
	115	CPU_BE( rev w4, w4 )
bed593c0 AB	116	CPU_BE( rev w5, w5 )
	117	CPU_BE( rev w6, w6 )
	118	CPU_BE( rev w7, w7 )
bed593c0	119
7c83d689 AB	120	stp w4, w5, [out]
7c83d689 AB	121	stp w6, w7, [out, #8]
bed593c0	122	ret
bed593c0 AB	123	.endm
bed593c0 AB	124
0e89640b	125	SYM_FUNC_START(__aes_arm64_encrypt)
96a74f7b	126	do_crypt fround, crypto_ft_tab, crypto_ft_tab + 1, 2
0e89640b	127	SYM_FUNC_END(__aes_arm64_encrypt)
96a74f7b AB	128
96a74f7b AB	129	.align 5
0e89640b	130	SYM_FUNC_START(__aes_arm64_decrypt)
642a88fb	131	do_crypt iround, crypto_it_tab, crypto_aes_inv_sbox, 0
0e89640b	132	SYM_FUNC_END(__aes_arm64_decrypt)