[linux-2.6-block.git] / arch / x86 / crypto / twofish-avx-x86_64-asm_64.S

/*
 * Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/linkage.h>
#include "glue_helper-asm-avx.S"

.file "twofish-avx-x86_64-asm_64.S"

.data
.align 16

.Lbswap128_mask:
	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
.Lxts_gf128mul_and_shl1_mask:
	.byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0

.text

/* structure of crypto context */
#define s0	0
#define s1	1024
#define s2	2048
#define s3	3072
#define w	4096
#define k	4128

/**********************************************************************
  8-way AVX twofish
 **********************************************************************/
#define CTX %rdi

#define RA1 %xmm0
#define RB1 %xmm1
#define RC1 %xmm2
#define RD1 %xmm3

#define RA2 %xmm4
#define RB2 %xmm5
#define RC2 %xmm6
#define RD2 %xmm7

#define RX0 %xmm8
#define RY0 %xmm9

#define RX1 %xmm10
#define RY1 %xmm11

#define RK1 %xmm12
#define RK2 %xmm13

#define RT %xmm14
#define RR %xmm15

#define RID1  %rbp
#define RID1d %ebp
#define RID2  %rsi
#define RID2d %esi

#define RGI1   %rdx
#define RGI1bl %dl
#define RGI1bh %dh
#define RGI2   %rcx
#define RGI2bl %cl
#define RGI2bh %ch

#define RGI3   %rax
#define RGI3bl %al
#define RGI3bh %ah
#define RGI4   %rbx
#define RGI4bl %bl
#define RGI4bh %bh

#define RGS1  %r8
#define RGS1d %r8d
#define RGS2  %r9
#define RGS2d %r9d
#define RGS3  %r10
#define RGS3d %r10d


#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
	movzbl		src ## bl,        RID1d;     \
	movzbl		src ## bh,        RID2d;     \
	shrq $16,	src;                         \
	movl		t0(CTX, RID1, 4), dst ## d;  \
	movl		t1(CTX, RID2, 4), RID2d;     \
	movzbl		src ## bl,        RID1d;     \
	xorl		RID2d,            dst ## d;  \
	movzbl		src ## bh,        RID2d;     \
	interleave_op(il_reg);			     \
	xorl		t2(CTX, RID1, 4), dst ## d;  \
	xorl		t3(CTX, RID2, 4), dst ## d;

#define dummy(d) /* do nothing */

#define shr_next(reg) \
	shrq $16,	reg;

#define G(gi1, gi2, x, t0, t1, t2, t3) \
	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1);  \
	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2);  \
	\
	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none);      \
	shlq $32,	RGS2;                                        \
	orq		RGS1, RGS2;                                  \
	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none);      \
	shlq $32,	RGS1;                                        \
	orq		RGS1, RGS3;

#define round_head_2(a, b, x1, y1, x2, y2) \
	vmovq		b ## 1, RGI3;           \
	vpextrq $1,	b ## 1, RGI4;           \
	\
	G(RGI1, RGI2, x1, s0, s1, s2, s3);      \
	vmovq		a ## 2, RGI1;           \
	vpextrq $1,	a ## 2, RGI2;           \
	vmovq		RGS2, x1;               \
	vpinsrq $1,	RGS3, x1, x1;           \
	\
	G(RGI3, RGI4, y1, s1, s2, s3, s0);      \
	vmovq		b ## 2, RGI3;           \
	vpextrq $1,	b ## 2, RGI4;           \
	vmovq		RGS2, y1;               \
	vpinsrq $1,	RGS3, y1, y1;           \
	\
	G(RGI1, RGI2, x2, s0, s1, s2, s3);      \
	vmovq		RGS2, x2;               \
	vpinsrq $1,	RGS3, x2, x2;           \
	\
	G(RGI3, RGI4, y2, s1, s2, s3, s0);      \
	vmovq		RGS2, y2;               \
	vpinsrq $1,	RGS3, y2, y2;

#define encround_tail(a, b, c, d, x, y, prerotate) \
	vpaddd			x, y,   x; \
	vpaddd			x, RK1, RT;\
	prerotate(b);			   \
	vpxor			RT, c,  c; \
	vpaddd			y, x,   y; \
	vpaddd			y, RK2, y; \
	vpsrld $1,		c, RT;     \
	vpslld $(32 - 1),	c, c;      \
	vpor			c, RT,  c; \
	vpxor			d, y,   d; \

#define decround_tail(a, b, c, d, x, y, prerotate) \
	vpaddd			x, y,   x; \
	vpaddd			x, RK1, RT;\
	prerotate(a);			   \
	vpxor			RT, c,  c; \
	vpaddd			y, x,   y; \
	vpaddd			y, RK2, y; \
	vpxor			d, y,   d; \
	vpsrld $1,		d, y;      \
	vpslld $(32 - 1),	d, d;      \
	vpor			d, y,   d; \

#define rotate_1l(x) \
	vpslld $1,		x, RR;     \
	vpsrld $(32 - 1),	x, x;      \
	vpor			x, RR,  x;

#define preload_rgi(c) \
	vmovq			c, RGI1; \
	vpextrq $1,		c, RGI2;

#define encrypt_round(n, a, b, c, d, preload, prerotate) \
	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
	encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
	preload(c ## 1);                                         \
	encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);

#define decrypt_round(n, a, b, c, d, preload, prerotate) \
	vbroadcastss (k+4*(2*(n)))(CTX),   RK1;                  \
	vbroadcastss (k+4*(2*(n)+1))(CTX), RK2;                  \
	round_head_2(a, b, RX0, RY0, RX1, RY1);                  \
	decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
	preload(c ## 1);                                         \
	decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);

#define encrypt_cycle(n) \
	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
	encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);

#define encrypt_cycle_last(n) \
	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
	encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);

#define decrypt_cycle(n) \
	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
	decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);

#define decrypt_cycle_last(n) \
	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
	decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);

#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
	vpunpckldq		x1, x0, t0; \
	vpunpckhdq		x1, x0, t2; \
	vpunpckldq		x3, x2, t1; \
	vpunpckhdq		x3, x2, x3; \
	\
	vpunpcklqdq		t1, t0, x0; \
	vpunpckhqdq		t1, t0, x1; \
	vpunpcklqdq		x3, t2, x2; \
	vpunpckhqdq		x3, t2, x3;

#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
	vpxor		x0, wkey, x0; \
	vpxor		x1, wkey, x1; \
	vpxor		x2, wkey, x2; \
	vpxor		x3, wkey, x3; \
	\
	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)

#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
	\
	vpxor		x0, wkey, x0; \
	vpxor		x1, wkey, x1; \
	vpxor		x2, wkey, x2; \
	vpxor		x3, wkey, x3;

.align 8
__twofish_enc_blk8:
	/* input:
	 *	%rdi: ctx, CTX
	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
	 * output:
	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
	 */

	vmovdqu w(CTX), RK1;

	pushq %rbp;
	pushq %rbx;
	pushq %rcx;

	inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
	preload_rgi(RA1);
	rotate_1l(RD1);
	inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
	rotate_1l(RD2);

	encrypt_cycle(0);
	encrypt_cycle(1);
	encrypt_cycle(2);
	encrypt_cycle(3);
	encrypt_cycle(4);
	encrypt_cycle(5);
	encrypt_cycle(6);
	encrypt_cycle_last(7);

	vmovdqu (w+4*4)(CTX), RK1;

	popq %rcx;
	popq %rbx;
	popq %rbp;

	outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
	outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);

	ret;
ENDPROC(__twofish_enc_blk8)

.align 8
__twofish_dec_blk8:
	/* input:
	 *	%rdi: ctx, CTX
	 *	RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
	 * output:
	 *	RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
	 */

	vmovdqu (w+4*4)(CTX), RK1;

	pushq %rbp;
	pushq %rbx;

	inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
	preload_rgi(RC1);
	rotate_1l(RA1);
	inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
	rotate_1l(RA2);

	decrypt_cycle(7);
	decrypt_cycle(6);
	decrypt_cycle(5);
	decrypt_cycle(4);
	decrypt_cycle(3);
	decrypt_cycle(2);
	decrypt_cycle(1);
	decrypt_cycle_last(0);

	vmovdqu (w)(CTX), RK1;

	popq %rbx;
	popq %rbp;

	outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
	outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);

	ret;
ENDPROC(__twofish_dec_blk8)

ENTRY(twofish_ecb_enc_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 */

	movq %rsi, %r11;

	load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);

	call __twofish_enc_blk8;

	store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);

	ret;
ENDPROC(twofish_ecb_enc_8way)

ENTRY(twofish_ecb_dec_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 */

	movq %rsi, %r11;

	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);

	call __twofish_dec_blk8;

	store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);

	ret;
ENDPROC(twofish_ecb_dec_8way)

ENTRY(twofish_cbc_dec_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 */

	pushq %r12;

	movq %rsi, %r11;
	movq %rdx, %r12;

	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);

	call __twofish_dec_blk8;

	store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);

	popq %r12;

	ret;
ENDPROC(twofish_cbc_dec_8way)

ENTRY(twofish_ctr_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 *	%rcx: iv (little endian, 128bit)
	 */

	pushq %r12;

	movq %rsi, %r11;
	movq %rdx, %r12;

	load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
		      RD2, RX0, RX1, RY0);

	call __twofish_enc_blk8;

	store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);

	popq %r12;

	ret;
ENDPROC(twofish_ctr_8way)

ENTRY(twofish_xts_enc_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
	 */

	movq %rsi, %r11;

	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
	load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);

	call __twofish_enc_blk8;

	/* dst <= regs xor IVs(in dst) */
	store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);

	ret;
ENDPROC(twofish_xts_enc_8way)

ENTRY(twofish_xts_dec_8way)
	/* input:
	 *	%rdi: ctx, CTX
	 *	%rsi: dst
	 *	%rdx: src
	 *	%rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
	 */

	movq %rsi, %r11;

	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
	load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2,
		      RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);

	call __twofish_dec_blk8;

	/* dst <= regs xor IVs(in dst) */
	store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);

	ret;
ENDPROC(twofish_xts_dec_8way)
Commit	Line	Data
107778b5 JG	1	/*
	2	* Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
	3	*
	4	* Copyright (C) 2012 Johannes Goetzfried
	5	* <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
	6	*
18be4527	7	* Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
f94a73f8	8	*
107778b5 JG	9	* This program is free software; you can redistribute it and/or modify
	10	* it under the terms of the GNU General Public License as published by
	11	* the Free Software Foundation; either version 2 of the License, or
	12	* (at your option) any later version.
	13	*
	14	* This program is distributed in the hope that it will be useful,
	15	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	* GNU General Public License for more details.
	18	*
	19	* You should have received a copy of the GNU General Public License
	20	* along with this program; if not, write to the Free Software
	21	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
	22	* USA
	23	*
	24	*/
	25
d3f5188d	26	#include <linux/linkage.h>
8435a3c3 JK	27	#include "glue_helper-asm-avx.S"
8435a3c3 JK	28
107778b5	29	.file "twofish-avx-x86_64-asm_64.S"
8435a3c3 JK	30
	31	.data
	32	.align 16
	33
	34	.Lbswap128_mask:
	35	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
18be4527 JK	36	.Lxts_gf128mul_and_shl1_mask:
18be4527 JK	37	.byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0
8435a3c3	38
107778b5 JG	39	.text
	40
	41	/* structure of crypto context */
	42	#define s0 0
	43	#define s1 1024
	44	#define s2 2048
	45	#define s3 3072
	46	#define w 4096
	47	#define k 4128
	48
	49	/**********************************************************************
	50	8-way AVX twofish
	51	**********************************************************************/
	52	#define CTX %rdi
	53
	54	#define RA1 %xmm0
	55	#define RB1 %xmm1
	56	#define RC1 %xmm2
	57	#define RD1 %xmm3
	58
	59	#define RA2 %xmm4
	60	#define RB2 %xmm5
	61	#define RC2 %xmm6
	62	#define RD2 %xmm7
	63
f94a73f8 JK	64	#define RX0 %xmm8
	65	#define RY0 %xmm9
	66
	67	#define RX1 %xmm10
	68	#define RY1 %xmm11
107778b5	69
f94a73f8 JK	70	#define RK1 %xmm12
f94a73f8 JK	71	#define RK2 %xmm13
107778b5	72
f94a73f8 JK	73	#define RT %xmm14
	74	#define RR %xmm15
	75
	76	#define RID1 %rbp
	77	#define RID1d %ebp
	78	#define RID2 %rsi
	79	#define RID2d %esi
107778b5 JG	80
	81	#define RGI1 %rdx
	82	#define RGI1bl %dl
	83	#define RGI1bh %dh
	84	#define RGI2 %rcx
	85	#define RGI2bl %cl
	86	#define RGI2bh %ch
	87
f94a73f8 JK	88	#define RGI3 %rax
	89	#define RGI3bl %al
	90	#define RGI3bh %ah
	91	#define RGI4 %rbx
	92	#define RGI4bl %bl
	93	#define RGI4bh %bh
	94
107778b5 JG	95	#define RGS1 %r8
	96	#define RGS1d %r8d
	97	#define RGS2 %r9
	98	#define RGS2d %r9d
	99	#define RGS3 %r10
	100	#define RGS3d %r10d
	101
	102
f94a73f8 JK	103	#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
	104	movzbl src ## bl, RID1d; \
	105	movzbl src ## bh, RID2d; \
107778b5	106	shrq $16, src; \
f94a73f8 JK	107	movl t0(CTX, RID1, 4), dst ## d; \
	108	movl t1(CTX, RID2, 4), RID2d; \
	109	movzbl src ## bl, RID1d; \
	110	xorl RID2d, dst ## d; \
	111	movzbl src ## bh, RID2d; \
	112	interleave_op(il_reg); \
107778b5 JG	113	xorl t2(CTX, RID1, 4), dst ## d; \
	114	xorl t3(CTX, RID2, 4), dst ## d;
	115
f94a73f8 JK	116	#define dummy(d) /* do nothing */
	117
	118	#define shr_next(reg) \
	119	shrq $16, reg;
	120
	121	#define G(gi1, gi2, x, t0, t1, t2, t3) \
	122	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1); \
	123	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2); \
	124	\
	125	lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none); \
	126	shlq $32, RGS2; \
	127	orq RGS1, RGS2; \
	128	lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none); \
	129	shlq $32, RGS1; \
	130	orq RGS1, RGS3;
	131
	132	#define round_head_2(a, b, x1, y1, x2, y2) \
	133	vmovq b ## 1, RGI3; \
	134	vpextrq $1, b ## 1, RGI4; \
107778b5	135	\
f94a73f8 JK	136	G(RGI1, RGI2, x1, s0, s1, s2, s3); \
	137	vmovq a ## 2, RGI1; \
	138	vpextrq $1, a ## 2, RGI2; \
	139	vmovq RGS2, x1; \
	140	vpinsrq $1, RGS3, x1, x1; \
107778b5	141	\
f94a73f8 JK	142	G(RGI3, RGI4, y1, s1, s2, s3, s0); \
	143	vmovq b ## 2, RGI3; \
	144	vpextrq $1, b ## 2, RGI4; \
	145	vmovq RGS2, y1; \
	146	vpinsrq $1, RGS3, y1, y1; \
107778b5	147	\
f94a73f8 JK	148	G(RGI1, RGI2, x2, s0, s1, s2, s3); \
	149	vmovq RGS2, x2; \
	150	vpinsrq $1, RGS3, x2, x2; \
	151	\
	152	G(RGI3, RGI4, y2, s1, s2, s3, s0); \
	153	vmovq RGS2, y2; \
	154	vpinsrq $1, RGS3, y2, y2;
107778b5	155
f94a73f8	156	#define encround_tail(a, b, c, d, x, y, prerotate) \
107778b5	157	vpaddd x, y, x; \
f94a73f8 JK	158	vpaddd x, RK1, RT;\
	159	prerotate(b); \
	160	vpxor RT, c, c; \
107778b5	161	vpaddd y, x, y; \
107778b5	162	vpaddd y, RK2, y; \
f94a73f8	163	vpsrld $1, c, RT; \
107778b5	164	vpslld $(32 - 1), c, c; \
f94a73f8 JK	165	vpor c, RT, c; \
	166	vpxor d, y, d; \
	167
	168	#define decround_tail(a, b, c, d, x, y, prerotate) \
107778b5	169	vpaddd x, y, x; \
f94a73f8 JK	170	vpaddd x, RK1, RT;\
	171	prerotate(a); \
	172	vpxor RT, c, c; \
107778b5 JG	173	vpaddd y, x, y; \
	174	vpaddd y, RK2, y; \
	175	vpxor d, y, d; \
	176	vpsrld $1, d, y; \
	177	vpslld $(32 - 1), d, d; \
	178	vpor d, y, d; \
f94a73f8 JK	179
	180	#define rotate_1l(x) \
	181	vpslld $1, x, RR; \
	182	vpsrld $(32 - 1), x, x; \
	183	vpor x, RR, x;
	184
	185	#define preload_rgi(c) \
	186	vmovq c, RGI1; \
	187	vpextrq $1, c, RGI2;
	188
	189	#define encrypt_round(n, a, b, c, d, preload, prerotate) \
	190	vbroadcastss (k+4(2(n)))(CTX), RK1; \
	191	vbroadcastss (k+4(2(n)+1))(CTX), RK2; \
	192	round_head_2(a, b, RX0, RY0, RX1, RY1); \
	193	encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
	194	preload(c ## 1); \
	195	encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
	196
	197	#define decrypt_round(n, a, b, c, d, preload, prerotate) \
	198	vbroadcastss (k+4(2(n)))(CTX), RK1; \
	199	vbroadcastss (k+4(2(n)+1))(CTX), RK2; \
	200	round_head_2(a, b, RX0, RY0, RX1, RY1); \
	201	decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
	202	preload(c ## 1); \
	203	decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
107778b5 JG	204
107778b5 JG	205	#define encrypt_cycle(n) \
f94a73f8 JK	206	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
	207	encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);
	208
	209	#define encrypt_cycle_last(n) \
	210	encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
	211	encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);
107778b5 JG	212
107778b5 JG	213	#define decrypt_cycle(n) \
f94a73f8 JK	214	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
f94a73f8 JK	215	decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);
107778b5	216
f94a73f8 JK	217	#define decrypt_cycle_last(n) \
	218	decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
	219	decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);
107778b5 JG	220
	221	#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
	222	vpunpckldq x1, x0, t0; \
	223	vpunpckhdq x1, x0, t2; \
	224	vpunpckldq x3, x2, t1; \
	225	vpunpckhdq x3, x2, x3; \
	226	\
	227	vpunpcklqdq t1, t0, x0; \
	228	vpunpckhqdq t1, t0, x1; \
	229	vpunpcklqdq x3, t2, x2; \
	230	vpunpckhqdq x3, t2, x3;
	231
8435a3c3 JK	232	#define inpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
	233	vpxor x0, wkey, x0; \
	234	vpxor x1, wkey, x1; \
	235	vpxor x2, wkey, x2; \
	236	vpxor x3, wkey, x3; \
107778b5 JG	237	\
	238	transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
	239
8435a3c3	240	#define outunpack_blocks(x0, x1, x2, x3, wkey, t0, t1, t2) \
107778b5 JG	241	transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
107778b5 JG	242	\
8435a3c3 JK	243	vpxor x0, wkey, x0; \
	244	vpxor x1, wkey, x1; \
	245	vpxor x2, wkey, x2; \
	246	vpxor x3, wkey, x3;
107778b5 JG	247
107778b5 JG	248	.align 8
8435a3c3	249	__twofish_enc_blk8:
107778b5 JG	250	/* input:
107778b5 JG	251	* %rdi: ctx, CTX
8435a3c3 JK	252	* RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: blocks
	253	* output:
	254	* RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
107778b5 JG	255	*/
107778b5 JG	256
8435a3c3 JK	257	vmovdqu w(CTX), RK1;
8435a3c3 JK	258
f94a73f8	259	pushq %rbp;
107778b5 JG	260	pushq %rbx;
	261	pushq %rcx;
	262
8435a3c3	263	inpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
f94a73f8 JK	264	preload_rgi(RA1);
f94a73f8 JK	265	rotate_1l(RD1);
8435a3c3	266	inpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
f94a73f8	267	rotate_1l(RD2);
107778b5	268
107778b5 JG	269	encrypt_cycle(0);
	270	encrypt_cycle(1);
	271	encrypt_cycle(2);
	272	encrypt_cycle(3);
	273	encrypt_cycle(4);
	274	encrypt_cycle(5);
	275	encrypt_cycle(6);
f94a73f8	276	encrypt_cycle_last(7);
107778b5 JG	277
	278	vmovdqu (w+4*4)(CTX), RK1;
	279
	280	popq %rcx;
	281	popq %rbx;
f94a73f8	282	popq %rbp;
107778b5	283
8435a3c3 JK	284	outunpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
8435a3c3 JK	285	outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
107778b5 JG	286
107778b5 JG	287	ret;
d3f5188d	288	ENDPROC(__twofish_enc_blk8)
107778b5 JG	289
107778b5 JG	290	.align 8
8435a3c3	291	__twofish_dec_blk8:
107778b5 JG	292	/* input:
107778b5 JG	293	* %rdi: ctx, CTX
8435a3c3 JK	294	* RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2: encrypted blocks
	295	* output:
	296	* RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2: decrypted blocks
107778b5 JG	297	*/
107778b5 JG	298
8435a3c3 JK	299	vmovdqu (w+4*4)(CTX), RK1;
8435a3c3 JK	300
f94a73f8	301	pushq %rbp;
107778b5 JG	302	pushq %rbx;
107778b5 JG	303
8435a3c3	304	inpack_blocks(RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
f94a73f8 JK	305	preload_rgi(RC1);
f94a73f8 JK	306	rotate_1l(RA1);
8435a3c3	307	inpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
f94a73f8	308	rotate_1l(RA2);
107778b5	309
107778b5 JG	310	decrypt_cycle(7);
	311	decrypt_cycle(6);
	312	decrypt_cycle(5);
	313	decrypt_cycle(4);
	314	decrypt_cycle(3);
	315	decrypt_cycle(2);
	316	decrypt_cycle(1);
f94a73f8	317	decrypt_cycle_last(0);
107778b5 JG	318
	319	vmovdqu (w)(CTX), RK1;
	320
	321	popq %rbx;
f94a73f8	322	popq %rbp;
107778b5	323
8435a3c3 JK	324	outunpack_blocks(RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
	325	outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
	326
	327	ret;
d3f5188d	328	ENDPROC(__twofish_dec_blk8)
8435a3c3	329
d3f5188d	330	ENTRY(twofish_ecb_enc_8way)
8435a3c3 JK	331	/* input:
	332	* %rdi: ctx, CTX
	333	* %rsi: dst
	334	* %rdx: src
	335	*/
	336
	337	movq %rsi, %r11;
	338
	339	load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
	340
	341	call __twofish_enc_blk8;
	342
	343	store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
	344
	345	ret;
d3f5188d	346	ENDPROC(twofish_ecb_enc_8way)
8435a3c3	347
d3f5188d	348	ENTRY(twofish_ecb_dec_8way)
8435a3c3 JK	349	/* input:
	350	* %rdi: ctx, CTX
	351	* %rsi: dst
	352	* %rdx: src
	353	*/
	354
	355	movq %rsi, %r11;
	356
	357	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
	358
	359	call __twofish_dec_blk8;
	360
	361	store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
	362
	363	ret;
d3f5188d	364	ENDPROC(twofish_ecb_dec_8way)
8435a3c3	365
d3f5188d	366	ENTRY(twofish_cbc_dec_8way)
8435a3c3 JK	367	/* input:
	368	* %rdi: ctx, CTX
	369	* %rsi: dst
	370	* %rdx: src
	371	*/
	372
	373	pushq %r12;
	374
	375	movq %rsi, %r11;
	376	movq %rdx, %r12;
	377
	378	load_8way(%rdx, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
	379
	380	call __twofish_dec_blk8;
	381
	382	store_cbc_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
	383
	384	popq %r12;
	385
	386	ret;
d3f5188d	387	ENDPROC(twofish_cbc_dec_8way)
8435a3c3	388
d3f5188d	389	ENTRY(twofish_ctr_8way)
8435a3c3 JK	390	/* input:
	391	* %rdi: ctx, CTX
	392	* %rsi: dst
	393	* %rdx: src
	394	* %rcx: iv (little endian, 128bit)
	395	*/
	396
	397	pushq %r12;
	398
	399	movq %rsi, %r11;
	400	movq %rdx, %r12;
	401
	402	load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
	403	RD2, RX0, RX1, RY0);
	404
	405	call __twofish_enc_blk8;
	406
	407	store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
	408
	409	popq %r12;
107778b5 JG	410
107778b5 JG	411	ret;
d3f5188d	412	ENDPROC(twofish_ctr_8way)
18be4527 JK	413
	414	ENTRY(twofish_xts_enc_8way)
	415	/* input:
	416	* %rdi: ctx, CTX
	417	* %rsi: dst
	418	* %rdx: src
	419	* %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
	420	*/
	421
	422	movq %rsi, %r11;
	423
	424	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
	425	load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
	426	RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
	427
	428	call __twofish_enc_blk8;
	429
	430	/* dst <= regs xor IVs(in dst) */
	431	store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
	432
	433	ret;
	434	ENDPROC(twofish_xts_enc_8way)
	435
	436	ENTRY(twofish_xts_dec_8way)
	437	/* input:
	438	* %rdi: ctx, CTX
	439	* %rsi: dst
	440	* %rdx: src
	441	* %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
	442	*/
	443
	444	movq %rsi, %r11;
	445
	446	/* regs <= src, dst <= IVs, regs <= regs xor IVs */
	447	load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2,
	448	RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask);
	449
	450	call __twofish_dec_blk8;
	451
	452	/* dst <= regs xor IVs(in dst) */
	453	store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
	454
	455	ret;
	456	ENDPROC(twofish_xts_dec_8way)