[linux-2.6-block.git] / arch / x86 / crypto / sha1_ni_asm.S

/*
 * Intel SHA Extensions optimized implementation of a SHA-1 update function
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * Copyright(c) 2015 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * Contact Information:
 * 	Sean Gulley <sean.m.gulley@intel.com>
 * 	Tim Chen <tim.c.chen@linux.intel.com>
 *
 * BSD LICENSE
 *
 * Copyright(c) 2015 Intel Corporation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 	* Redistributions of source code must retain the above copyright
 * 	  notice, this list of conditions and the following disclaimer.
 * 	* Redistributions in binary form must reproduce the above copyright
 * 	  notice, this list of conditions and the following disclaimer in
 * 	  the documentation and/or other materials provided with the
 * 	  distribution.
 * 	* Neither the name of Intel Corporation nor the names of its
 * 	  contributors may be used to endorse or promote products derived
 * 	  from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

#include <linux/linkage.h>

#define DIGEST_PTR	%rdi	/* 1st arg */
#define DATA_PTR	%rsi	/* 2nd arg */
#define NUM_BLKS	%rdx	/* 3rd arg */

#define RSPSAVE		%rax

/* gcc conversion */
#define FRAME_SIZE	32	/* space for 2x16 bytes */

#define ABCD		%xmm0
#define E0		%xmm1	/* Need two E's b/c they ping pong */
#define E1		%xmm2
#define MSG0		%xmm3
#define MSG1		%xmm4
#define MSG2		%xmm5
#define MSG3		%xmm6
#define SHUF_MASK	%xmm7


/*
 * Intel SHA Extensions optimized implementation of a SHA-1 update function
 *
 * The function takes a pointer to the current hash values, a pointer to the
 * input data, and a number of 64 byte blocks to process.  Once all blocks have
 * been processed, the digest pointer is  updated with the resulting hash value.
 * The function only processes complete blocks, there is no functionality to
 * store partial blocks. All message padding and hash value initialization must
 * be done outside the update function.
 *
 * The indented lines in the loop are instructions related to rounds processing.
 * The non-indented lines are instructions related to the message schedule.
 *
 * void sha1_ni_transform(uint32_t *digest, const void *data,
		uint32_t numBlocks)
 * digest : pointer to digest
 * data: pointer to input data
 * numBlocks: Number of blocks to process
 */
.text
.align 32
ENTRY(sha1_ni_transform)
	mov		%rsp, RSPSAVE
	sub		$FRAME_SIZE, %rsp
	and		$~0xF, %rsp

	shl		$6, NUM_BLKS		/* convert to bytes */
	jz		.Ldone_hash
	add		DATA_PTR, NUM_BLKS	/* pointer to end of data */

	/* load initial hash values */
	pinsrd		$3, 1*16(DIGEST_PTR), E0
	movdqu		0*16(DIGEST_PTR), ABCD
	pand		UPPER_WORD_MASK(%rip), E0
	pshufd		$0x1B, ABCD, ABCD

	movdqa		PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK

.Lloop0:
	/* Save hash values for addition after rounds */
	movdqa		E0, (0*16)(%rsp)
	movdqa		ABCD, (1*16)(%rsp)

	/* Rounds 0-3 */
	movdqu		0*16(DATA_PTR), MSG0
	pshufb		SHUF_MASK, MSG0
		paddd		MSG0, E0
		movdqa		ABCD, E1
		sha1rnds4	$0, E0, ABCD

	/* Rounds 4-7 */
	movdqu		1*16(DATA_PTR), MSG1
	pshufb		SHUF_MASK, MSG1
		sha1nexte	MSG1, E1
		movdqa		ABCD, E0
		sha1rnds4	$0, E1, ABCD
	sha1msg1	MSG1, MSG0

	/* Rounds 8-11 */
	movdqu		2*16(DATA_PTR), MSG2
	pshufb		SHUF_MASK, MSG2
		sha1nexte	MSG2, E0
		movdqa		ABCD, E1
		sha1rnds4	$0, E0, ABCD
	sha1msg1	MSG2, MSG1
	pxor		MSG2, MSG0

	/* Rounds 12-15 */
	movdqu		3*16(DATA_PTR), MSG3
	pshufb		SHUF_MASK, MSG3
		sha1nexte	MSG3, E1
		movdqa		ABCD, E0
	sha1msg2	MSG3, MSG0
		sha1rnds4	$0, E1, ABCD
	sha1msg1	MSG3, MSG2
	pxor		MSG3, MSG1

	/* Rounds 16-19 */
		sha1nexte	MSG0, E0
		movdqa		ABCD, E1
	sha1msg2	MSG0, MSG1
		sha1rnds4	$0, E0, ABCD
	sha1msg1	MSG0, MSG3
	pxor		MSG0, MSG2

	/* Rounds 20-23 */
		sha1nexte	MSG1, E1
		movdqa		ABCD, E0
	sha1msg2	MSG1, MSG2
		sha1rnds4	$1, E1, ABCD
	sha1msg1	MSG1, MSG0
	pxor		MSG1, MSG3

	/* Rounds 24-27 */
		sha1nexte	MSG2, E0
		movdqa		ABCD, E1
	sha1msg2	MSG2, MSG3
		sha1rnds4	$1, E0, ABCD
	sha1msg1	MSG2, MSG1
	pxor		MSG2, MSG0

	/* Rounds 28-31 */
		sha1nexte	MSG3, E1
		movdqa		ABCD, E0
	sha1msg2	MSG3, MSG0
		sha1rnds4	$1, E1, ABCD
	sha1msg1	MSG3, MSG2
	pxor		MSG3, MSG1

	/* Rounds 32-35 */
		sha1nexte	MSG0, E0
		movdqa		ABCD, E1
	sha1msg2	MSG0, MSG1
		sha1rnds4	$1, E0, ABCD
	sha1msg1	MSG0, MSG3
	pxor		MSG0, MSG2

	/* Rounds 36-39 */
		sha1nexte	MSG1, E1
		movdqa		ABCD, E0
	sha1msg2	MSG1, MSG2
		sha1rnds4	$1, E1, ABCD
	sha1msg1	MSG1, MSG0
	pxor		MSG1, MSG3

	/* Rounds 40-43 */
		sha1nexte	MSG2, E0
		movdqa		ABCD, E1
	sha1msg2	MSG2, MSG3
		sha1rnds4	$2, E0, ABCD
	sha1msg1	MSG2, MSG1
	pxor		MSG2, MSG0

	/* Rounds 44-47 */
		sha1nexte	MSG3, E1
		movdqa		ABCD, E0
	sha1msg2	MSG3, MSG0
		sha1rnds4	$2, E1, ABCD
	sha1msg1	MSG3, MSG2
	pxor		MSG3, MSG1

	/* Rounds 48-51 */
		sha1nexte	MSG0, E0
		movdqa		ABCD, E1
	sha1msg2	MSG0, MSG1
		sha1rnds4	$2, E0, ABCD
	sha1msg1	MSG0, MSG3
	pxor		MSG0, MSG2

	/* Rounds 52-55 */
		sha1nexte	MSG1, E1
		movdqa		ABCD, E0
	sha1msg2	MSG1, MSG2
		sha1rnds4	$2, E1, ABCD
	sha1msg1	MSG1, MSG0
	pxor		MSG1, MSG3

	/* Rounds 56-59 */
		sha1nexte	MSG2, E0
		movdqa		ABCD, E1
	sha1msg2	MSG2, MSG3
		sha1rnds4	$2, E0, ABCD
	sha1msg1	MSG2, MSG1
	pxor		MSG2, MSG0

	/* Rounds 60-63 */
		sha1nexte	MSG3, E1
		movdqa		ABCD, E0
	sha1msg2	MSG3, MSG0
		sha1rnds4	$3, E1, ABCD
	sha1msg1	MSG3, MSG2
	pxor		MSG3, MSG1

	/* Rounds 64-67 */
		sha1nexte	MSG0, E0
		movdqa		ABCD, E1
	sha1msg2	MSG0, MSG1
		sha1rnds4	$3, E0, ABCD
	sha1msg1	MSG0, MSG3
	pxor		MSG0, MSG2

	/* Rounds 68-71 */
		sha1nexte	MSG1, E1
		movdqa		ABCD, E0
	sha1msg2	MSG1, MSG2
		sha1rnds4	$3, E1, ABCD
	pxor		MSG1, MSG3

	/* Rounds 72-75 */
		sha1nexte	MSG2, E0
		movdqa		ABCD, E1
	sha1msg2	MSG2, MSG3
		sha1rnds4	$3, E0, ABCD

	/* Rounds 76-79 */
		sha1nexte	MSG3, E1
		movdqa		ABCD, E0
		sha1rnds4	$3, E1, ABCD

	/* Add current hash values with previously saved */
	sha1nexte	(0*16)(%rsp), E0
	paddd		(1*16)(%rsp), ABCD

	/* Increment data pointer and loop if more to process */
	add		$64, DATA_PTR
	cmp		NUM_BLKS, DATA_PTR
	jne		.Lloop0

	/* Write hash values back in the correct order */
	pshufd		$0x1B, ABCD, ABCD
	movdqu		ABCD, 0*16(DIGEST_PTR)
	pextrd		$3, E0, 1*16(DIGEST_PTR)

.Ldone_hash:
	mov		RSPSAVE, %rsp

	ret
ENDPROC(sha1_ni_transform)

.section	.rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
.align 16
PSHUFFLE_BYTE_FLIP_MASK:
	.octa 0x000102030405060708090a0b0c0d0e0f

.section	.rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
.align 16
UPPER_WORD_MASK:
	.octa 0xFFFFFFFF000000000000000000000000
Commit	Line	Data
c356a7e9	1	/*
	2	* Intel SHA Extensions optimized implementation of a SHA-1 update function
	3	*
	4	* This file is provided under a dual BSD/GPLv2 license. When using or
	5	* redistributing this file, you may do so under either license.
	6	*
	7	* GPL LICENSE SUMMARY
	8	*
	9	* Copyright(c) 2015 Intel Corporation.
	10	*
	11	* This program is free software; you can redistribute it and/or modify
	12	* it under the terms of version 2 of the GNU General Public License as
	13	* published by the Free Software Foundation.
	14	*
	15	* This program is distributed in the hope that it will be useful, but
	16	* WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* General Public License for more details.
	19	*
	20	* Contact Information:
	21	* Sean Gulley <sean.m.gulley@intel.com>
	22	* Tim Chen <tim.c.chen@linux.intel.com>
	23	*
	24	* BSD LICENSE
	25	*
	26	* Copyright(c) 2015 Intel Corporation.
	27	*
	28	* Redistribution and use in source and binary forms, with or without
	29	* modification, are permitted provided that the following conditions
	30	* are met:
	31	*
	32	* * Redistributions of source code must retain the above copyright
	33	* notice, this list of conditions and the following disclaimer.
	34	* * Redistributions in binary form must reproduce the above copyright
	35	* notice, this list of conditions and the following disclaimer in
	36	* the documentation and/or other materials provided with the
	37	* distribution.
	38	* * Neither the name of Intel Corporation nor the names of its
	39	* contributors may be used to endorse or promote products derived
	40	* from this software without specific prior written permission.
	41	*
	42	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	43	* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	44	* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	45	* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	46	* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	47	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	48	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	49	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	50	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	51	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	52	* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	53	*
	54	*/
	55
	56	#include <linux/linkage.h>
	57
	58	#define DIGEST_PTR %rdi /* 1st arg */
	59	#define DATA_PTR %rsi /* 2nd arg */
	60	#define NUM_BLKS %rdx /* 3rd arg */
	61
	62	#define RSPSAVE %rax
	63
	64	/* gcc conversion */
65	#define FRAME_SIZE 32 /* space for 2x16 bytes */
66
67	#define ABCD %xmm0
68	#define E0 %xmm1 /* Need two E's b/c they ping pong */
69	#define E1 %xmm2
70	#define MSG0 %xmm3
71	#define MSG1 %xmm4
72	#define MSG2 %xmm5
73	#define MSG3 %xmm6
74	#define SHUF_MASK %xmm7
75
76
77	/*
78	* Intel SHA Extensions optimized implementation of a SHA-1 update function
79	*
80	* The function takes a pointer to the current hash values, a pointer to the
81	* input data, and a number of 64 byte blocks to process. Once all blocks have
82	* been processed, the digest pointer is updated with the resulting hash value.
83	* The function only processes complete blocks, there is no functionality to
84	* store partial blocks. All message padding and hash value initialization must
85	* be done outside the update function.
86	*
87	* The indented lines in the loop are instructions related to rounds processing.
88	* The non-indented lines are instructions related to the message schedule.
89	*
90	* void sha1_ni_transform(uint32_t digest, const void data,
91	uint32_t numBlocks)
92	* digest : pointer to digest
93	* data: pointer to input data
94	* numBlocks: Number of blocks to process
95	*/
96	.text
97	.align 32
98	ENTRY(sha1_ni_transform)
99	mov %rsp, RSPSAVE
100	sub $FRAME_SIZE, %rsp
101	and $~0xF, %rsp
102
103	shl $6, NUM_BLKS /* convert to bytes */
104	jz .Ldone_hash
105	add DATA_PTR, NUM_BLKS /* pointer to end of data */
106
107	/* load initial hash values */
108	pinsrd $3, 1*16(DIGEST_PTR), E0
109	movdqu 0*16(DIGEST_PTR), ABCD
110	pand UPPER_WORD_MASK(%rip), E0
111	pshufd $0x1B, ABCD, ABCD
112
113	movdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
114
115	.Lloop0:
116	/* Save hash values for addition after rounds */
117	movdqa E0, (0*16)(%rsp)
118	movdqa ABCD, (1*16)(%rsp)
119
120	/* Rounds 0-3 */
121	movdqu 0*16(DATA_PTR), MSG0
122	pshufb SHUF_MASK, MSG0
123	paddd MSG0, E0
124	movdqa ABCD, E1
125	sha1rnds4 $0, E0, ABCD
126
127	/* Rounds 4-7 */
128	movdqu 1*16(DATA_PTR), MSG1
129	pshufb SHUF_MASK, MSG1
130	sha1nexte MSG1, E1
131	movdqa ABCD, E0
132	sha1rnds4 $0, E1, ABCD
133	sha1msg1 MSG1, MSG0
134
135	/* Rounds 8-11 */
136	movdqu 2*16(DATA_PTR), MSG2
137	pshufb SHUF_MASK, MSG2
138	sha1nexte MSG2, E0
139	movdqa ABCD, E1
140	sha1rnds4 $0, E0, ABCD
141	sha1msg1 MSG2, MSG1
142	pxor MSG2, MSG0
143
144	/* Rounds 12-15 */
145	movdqu 3*16(DATA_PTR), MSG3
146	pshufb SHUF_MASK, MSG3
147	sha1nexte MSG3, E1
148	movdqa ABCD, E0
149	sha1msg2 MSG3, MSG0
150	sha1rnds4 $0, E1, ABCD
151	sha1msg1 MSG3, MSG2
152	pxor MSG3, MSG1
153
154	/* Rounds 16-19 */
155	sha1nexte MSG0, E0
156	movdqa ABCD, E1
157	sha1msg2 MSG0, MSG1
158	sha1rnds4 $0, E0, ABCD
159	sha1msg1 MSG0, MSG3
160	pxor MSG0, MSG2
161
162	/* Rounds 20-23 */
163	sha1nexte MSG1, E1
164	movdqa ABCD, E0
165	sha1msg2 MSG1, MSG2
166	sha1rnds4 $1, E1, ABCD
167	sha1msg1 MSG1, MSG0
168	pxor MSG1, MSG3
169
170	/* Rounds 24-27 */
171	sha1nexte MSG2, E0
172	movdqa ABCD, E1
173	sha1msg2 MSG2, MSG3
174	sha1rnds4 $1, E0, ABCD
175	sha1msg1 MSG2, MSG1
176	pxor MSG2, MSG0
177
178	/* Rounds 28-31 */
179	sha1nexte MSG3, E1
180	movdqa ABCD, E0
181	sha1msg2 MSG3, MSG0
182	sha1rnds4 $1, E1, ABCD
183	sha1msg1 MSG3, MSG2
184	pxor MSG3, MSG1
185
186	/* Rounds 32-35 */
187	sha1nexte MSG0, E0
188	movdqa ABCD, E1
189	sha1msg2 MSG0, MSG1
190	sha1rnds4 $1, E0, ABCD
191	sha1msg1 MSG0, MSG3
192	pxor MSG0, MSG2
193
194	/* Rounds 36-39 */
195	sha1nexte MSG1, E1
196	movdqa ABCD, E0
197	sha1msg2 MSG1, MSG2
198	sha1rnds4 $1, E1, ABCD
199	sha1msg1 MSG1, MSG0
200	pxor MSG1, MSG3
201
202	/* Rounds 40-43 */
203	sha1nexte MSG2, E0
204	movdqa ABCD, E1
205	sha1msg2 MSG2, MSG3
206	sha1rnds4 $2, E0, ABCD
207	sha1msg1 MSG2, MSG1
208	pxor MSG2, MSG0
209
210	/* Rounds 44-47 */
211	sha1nexte MSG3, E1
212	movdqa ABCD, E0
213	sha1msg2 MSG3, MSG0
214	sha1rnds4 $2, E1, ABCD
215	sha1msg1 MSG3, MSG2
216	pxor MSG3, MSG1
217
218	/* Rounds 48-51 */
219	sha1nexte MSG0, E0
220	movdqa ABCD, E1
221	sha1msg2 MSG0, MSG1
222	sha1rnds4 $2, E0, ABCD
223	sha1msg1 MSG0, MSG3
224	pxor MSG0, MSG2
225
226	/* Rounds 52-55 */
227	sha1nexte MSG1, E1
228	movdqa ABCD, E0
229	sha1msg2 MSG1, MSG2
230	sha1rnds4 $2, E1, ABCD
231	sha1msg1 MSG1, MSG0
232	pxor MSG1, MSG3
233
234	/* Rounds 56-59 */
235	sha1nexte MSG2, E0
236	movdqa ABCD, E1
237	sha1msg2 MSG2, MSG3
238	sha1rnds4 $2, E0, ABCD
239	sha1msg1 MSG2, MSG1
240	pxor MSG2, MSG0
241
242	/* Rounds 60-63 */
243	sha1nexte MSG3, E1
244	movdqa ABCD, E0
245	sha1msg2 MSG3, MSG0
246	sha1rnds4 $3, E1, ABCD
247	sha1msg1 MSG3, MSG2
248	pxor MSG3, MSG1
249
250	/* Rounds 64-67 */
251	sha1nexte MSG0, E0
252	movdqa ABCD, E1
253	sha1msg2 MSG0, MSG1
254	sha1rnds4 $3, E0, ABCD
255	sha1msg1 MSG0, MSG3
256	pxor MSG0, MSG2
257
258	/* Rounds 68-71 */
259	sha1nexte MSG1, E1
260	movdqa ABCD, E0
261	sha1msg2 MSG1, MSG2
262	sha1rnds4 $3, E1, ABCD
263	pxor MSG1, MSG3
264
265	/* Rounds 72-75 */
266	sha1nexte MSG2, E0
267	movdqa ABCD, E1
268	sha1msg2 MSG2, MSG3
269	sha1rnds4 $3, E0, ABCD
270
271	/* Rounds 76-79 */
272	sha1nexte MSG3, E1
273	movdqa ABCD, E0
274	sha1rnds4 $3, E1, ABCD
275
276	/* Add current hash values with previously saved */
277	sha1nexte (0*16)(%rsp), E0
278	paddd (1*16)(%rsp), ABCD
279
280	/* Increment data pointer and loop if more to process */
281	add $64, DATA_PTR
282	cmp NUM_BLKS, DATA_PTR
283	jne .Lloop0
284
285	/* Write hash values back in the correct order */
286	pshufd $0x1B, ABCD, ABCD
287	movdqu ABCD, 0*16(DIGEST_PTR)
288	pextrd $3, E0, 1*16(DIGEST_PTR)
289
290	.Ldone_hash:
291	mov RSPSAVE, %rsp
292
293	ret
294	ENDPROC(sha1_ni_transform)
295
e183914a DV	296	.section .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
e183914a DV	297	.align 16
c356a7e9	298	PSHUFFLE_BYTE_FLIP_MASK:
c356a7e9	299	.octa 0x000102030405060708090a0b0c0d0e0f
e183914a DV	300
	301	.section .rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
	302	.align 16
c356a7e9	303	UPPER_WORD_MASK:
c356a7e9	304	.octa 0xFFFFFFFF000000000000000000000000