[linux-block.git] / arch / x86 / crypto / sha1_ssse3_glue.c

/*
 * Cryptographic API.
 *
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
 * Supplemental SSE3 instructions.
 *
 * This file is based on sha1_generic.c
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 * Copyright (c) Mathias Krause <minipli@googlemail.com>
 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
#include <asm/fpu/api.h>

typedef void (sha1_transform_fn)(u32 *digest, const char *data,
				unsigned int rounds);

static int sha1_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len, sha1_transform_fn *sha1_xform)
{
	struct sha1_state *sctx = shash_desc_ctx(desc);

	if (!irq_fpu_usable() ||
	    (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
		return crypto_sha1_update(desc, data, len);

	/* make sure casting to sha1_block_fn() is safe */
	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);

	kernel_fpu_begin();
	sha1_base_do_update(desc, data, len,
			    (sha1_block_fn *)sha1_xform);
	kernel_fpu_end();

	return 0;
}

static int sha1_finup(struct shash_desc *desc, const u8 *data,
		      unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
{
	if (!irq_fpu_usable())
		return crypto_sha1_finup(desc, data, len, out);

	kernel_fpu_begin();
	if (len)
		sha1_base_do_update(desc, data, len,
				    (sha1_block_fn *)sha1_xform);
	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
	kernel_fpu_end();

	return sha1_base_finish(desc, out);
}

asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
				     unsigned int rounds);

static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
			(sha1_transform_fn *) sha1_transform_ssse3);
}

static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
			(sha1_transform_fn *) sha1_transform_ssse3);
}

/* Add padding and return the message digest. */
static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
{
	return sha1_ssse3_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ssse3_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_ssse3_update,
	.final		=	sha1_ssse3_final,
	.finup		=	sha1_ssse3_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-ssse3",
		.cra_priority	=	150,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_ssse3(void)
{
	if (boot_cpu_has(X86_FEATURE_SSSE3))
		return crypto_register_shash(&sha1_ssse3_alg);
	return 0;
}

static void unregister_sha1_ssse3(void)
{
	if (boot_cpu_has(X86_FEATURE_SSSE3))
		crypto_unregister_shash(&sha1_ssse3_alg);
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
				   unsigned int rounds);

static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
			(sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
			(sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_final(struct shash_desc *desc, u8 *out)
{
	return sha1_avx_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_avx_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_avx_update,
	.final		=	sha1_avx_final,
	.finup		=	sha1_avx_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-avx",
		.cra_priority	=	160,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static bool avx_usable(void)
{
	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
		if (boot_cpu_has(X86_FEATURE_AVX))
			pr_info("AVX detected but unusable.\n");
		return false;
	}

	return true;
}

static int register_sha1_avx(void)
{
	if (avx_usable())
		return crypto_register_shash(&sha1_avx_alg);
	return 0;
}

static void unregister_sha1_avx(void)
{
	if (avx_usable())
		crypto_unregister_shash(&sha1_avx_alg);
}

#else  /* CONFIG_AS_AVX */
static inline int register_sha1_avx(void) { return 0; }
static inline void unregister_sha1_avx(void) { }
#endif /* CONFIG_AS_AVX */


#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
#define SHA1_AVX2_BLOCK_OPTSIZE	4	/* optimal 4*64 bytes of SHA1 blocks */

asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
				    unsigned int rounds);

static bool avx2_usable(void)
{
	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
		&& boot_cpu_has(X86_FEATURE_BMI1)
		&& boot_cpu_has(X86_FEATURE_BMI2))
		return true;

	return false;
}

static void sha1_apply_transform_avx2(u32 *digest, const char *data,
				unsigned int rounds)
{
	/* Select the optimal transform based on data block size */
	if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
		sha1_transform_avx2(digest, data, rounds);
	else
		sha1_transform_avx(digest, data, rounds);
}

static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
		(sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
		(sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
{
	return sha1_avx2_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_avx2_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_avx2_update,
	.final		=	sha1_avx2_final,
	.finup		=	sha1_avx2_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-avx2",
		.cra_priority	=	170,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_avx2(void)
{
	if (avx2_usable())
		return crypto_register_shash(&sha1_avx2_alg);
	return 0;
}

static void unregister_sha1_avx2(void)
{
	if (avx2_usable())
		crypto_unregister_shash(&sha1_avx2_alg);
}

#else
static inline int register_sha1_avx2(void) { return 0; }
static inline void unregister_sha1_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA1_NI
asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
				   unsigned int rounds);

static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
			     unsigned int len)
{
	return sha1_update(desc, data, len,
		(sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
			      unsigned int len, u8 *out)
{
	return sha1_finup(desc, data, len, out,
		(sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_final(struct shash_desc *desc, u8 *out)
{
	return sha1_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ni_alg = {
	.digestsize	=	SHA1_DIGEST_SIZE,
	.init		=	sha1_base_init,
	.update		=	sha1_ni_update,
	.final		=	sha1_ni_final,
	.finup		=	sha1_ni_finup,
	.descsize	=	sizeof(struct sha1_state),
	.base		=	{
		.cra_name	=	"sha1",
		.cra_driver_name =	"sha1-ni",
		.cra_priority	=	250,
		.cra_blocksize	=	SHA1_BLOCK_SIZE,
		.cra_module	=	THIS_MODULE,
	}
};

static int register_sha1_ni(void)
{
	if (boot_cpu_has(X86_FEATURE_SHA_NI))
		return crypto_register_shash(&sha1_ni_alg);
	return 0;
}

static void unregister_sha1_ni(void)
{
	if (boot_cpu_has(X86_FEATURE_SHA_NI))
		crypto_unregister_shash(&sha1_ni_alg);
}

#else
static inline int register_sha1_ni(void) { return 0; }
static inline void unregister_sha1_ni(void) { }
#endif

static int __init sha1_ssse3_mod_init(void)
{
	if (register_sha1_ssse3())
		goto fail;

	if (register_sha1_avx()) {
		unregister_sha1_ssse3();
		goto fail;
	}

	if (register_sha1_avx2()) {
		unregister_sha1_avx();
		unregister_sha1_ssse3();
		goto fail;
	}

	if (register_sha1_ni()) {
		unregister_sha1_avx2();
		unregister_sha1_avx();
		unregister_sha1_ssse3();
		goto fail;
	}

	return 0;
fail:
	return -ENODEV;
}

static void __exit sha1_ssse3_mod_fini(void)
{
	unregister_sha1_ni();
	unregister_sha1_avx2();
	unregister_sha1_avx();
	unregister_sha1_ssse3();
}

module_init(sha1_ssse3_mod_init);
module_exit(sha1_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha1");
MODULE_ALIAS_CRYPTO("sha1-ssse3");
MODULE_ALIAS_CRYPTO("sha1-avx");
MODULE_ALIAS_CRYPTO("sha1-avx2");
#ifdef CONFIG_AS_SHA1_NI
MODULE_ALIAS_CRYPTO("sha1-ni");
#endif
Commit	Line	Data
66be8951 MK	1	/*
	2	* Cryptographic API.
	3	*
	4	* Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
	5	* Supplemental SSE3 instructions.
	6	*
	7	* This file is based on sha1_generic.c
	8	*
	9	* Copyright (c) Alan Smithee.
	10	* Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
	11	* Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
	12	* Copyright (c) Mathias Krause <minipli@googlemail.com>
7c1da8d0	13	* Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
66be8951 MK	14	*
	15	* This program is free software; you can redistribute it and/or modify it
	16	* under the terms of the GNU General Public License as published by the Free
	17	* Software Foundation; either version 2 of the License, or (at your option)
	18	* any later version.
	19	*
	20	*/
	21
	22	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	23
	24	#include <crypto/internal/hash.h>
	25	#include <linux/init.h>
	26	#include <linux/module.h>
	27	#include <linux/mm.h>
	28	#include <linux/cryptohash.h>
	29	#include <linux/types.h>
	30	#include <crypto/sha.h>
824b4376	31	#include <crypto/sha1_base.h>
df6b35f4	32	#include <asm/fpu/api.h>
66be8951	33
85c66ecd	34	typedef void (sha1_transform_fn)(u32 digest, const char data,
85c66ecd	35	unsigned int rounds);
66be8951	36
85c66ecd	37	static int sha1_update(struct shash_desc desc, const u8 data,
85c66ecd	38	unsigned int len, sha1_transform_fn *sha1_xform)
66be8951 MK	39	{
66be8951 MK	40	struct sha1_state *sctx = shash_desc_ctx(desc);
66be8951	41
824b4376 AB	42	if (!irq_fpu_usable() \|\|
	43	(sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
	44	return crypto_sha1_update(desc, data, len);
66be8951	45
824b4376 AB	46	/* make sure casting to sha1_block_fn() is safe */
824b4376 AB	47	BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
66be8951	48
824b4376 AB	49	kernel_fpu_begin();
824b4376 AB	50	sha1_base_do_update(desc, data, len,
85c66ecd	51	(sha1_block_fn *)sha1_xform);
824b4376	52	kernel_fpu_end();
66be8951 MK	53
	54	return 0;
	55	}
	56
85c66ecd	57	static int sha1_finup(struct shash_desc desc, const u8 data,
85c66ecd	58	unsigned int len, u8 out, sha1_transform_fn sha1_xform)
66be8951	59	{
824b4376 AB	60	if (!irq_fpu_usable())
824b4376 AB	61	return crypto_sha1_finup(desc, data, len, out);
66be8951	62
824b4376 AB	63	kernel_fpu_begin();
	64	if (len)
	65	sha1_base_do_update(desc, data, len,
85c66ecd	66	(sha1_block_fn *)sha1_xform);
85c66ecd	67	sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
824b4376	68	kernel_fpu_end();
66be8951	69
824b4376	70	return sha1_base_finish(desc, out);
66be8951 MK	71	}
66be8951 MK	72
85c66ecd	73	asmlinkage void sha1_transform_ssse3(u32 digest, const char data,
	74	unsigned int rounds);
	75
	76	static int sha1_ssse3_update(struct shash_desc desc, const u8 data,
	77	unsigned int len)
66be8951	78	{
85c66ecd	79	return sha1_update(desc, data, len,
85c66ecd	80	(sha1_transform_fn *) sha1_transform_ssse3);
66be8951 MK	81	}
66be8951 MK	82
85c66ecd	83	static int sha1_ssse3_finup(struct shash_desc desc, const u8 data,
85c66ecd	84	unsigned int len, u8 *out)
7c1da8d0	85	{
85c66ecd	86	return sha1_finup(desc, data, len, out,
	87	(sha1_transform_fn *) sha1_transform_ssse3);
	88	}
	89
	90	/* Add padding and return the message digest. */
	91	static int sha1_ssse3_final(struct shash_desc desc, u8 out)
	92	{
	93	return sha1_ssse3_finup(desc, NULL, 0, out);
7c1da8d0	94	}
7c1da8d0	95
85c66ecd	96	static struct shash_alg sha1_ssse3_alg = {
66be8951	97	.digestsize = SHA1_DIGEST_SIZE,
824b4376	98	.init = sha1_base_init,
66be8951 MK	99	.update = sha1_ssse3_update,
66be8951 MK	100	.final = sha1_ssse3_final,
824b4376	101	.finup = sha1_ssse3_finup,
66be8951	102	.descsize = sizeof(struct sha1_state),
66be8951 MK	103	.base = {
66be8951 MK	104	.cra_name = "sha1",
85c66ecd	105	.cra_driver_name = "sha1-ssse3",
66be8951	106	.cra_priority = 150,
66be8951 MK	107	.cra_blocksize = SHA1_BLOCK_SIZE,
	108	.cra_module = THIS_MODULE,
	109	}
	110	};
	111
85c66ecd	112	static int register_sha1_ssse3(void)
	113	{
	114	if (boot_cpu_has(X86_FEATURE_SSSE3))
	115	return crypto_register_shash(&sha1_ssse3_alg);
	116	return 0;
	117	}
	118
	119	static void unregister_sha1_ssse3(void)
	120	{
	121	if (boot_cpu_has(X86_FEATURE_SSSE3))
	122	crypto_unregister_shash(&sha1_ssse3_alg);
	123	}
	124
65df5774	125	#ifdef CONFIG_AS_AVX
85c66ecd	126	asmlinkage void sha1_transform_avx(u32 digest, const char data,
	127	unsigned int rounds);
	128
	129	static int sha1_avx_update(struct shash_desc desc, const u8 data,
	130	unsigned int len)
	131	{
	132	return sha1_update(desc, data, len,
	133	(sha1_transform_fn *) sha1_transform_avx);
	134	}
	135
	136	static int sha1_avx_finup(struct shash_desc desc, const u8 data,
	137	unsigned int len, u8 *out)
	138	{
	139	return sha1_finup(desc, data, len, out,
	140	(sha1_transform_fn *) sha1_transform_avx);
	141	}
	142
	143	static int sha1_avx_final(struct shash_desc desc, u8 out)
	144	{
	145	return sha1_avx_finup(desc, NULL, 0, out);
	146	}
	147
	148	static struct shash_alg sha1_avx_alg = {
	149	.digestsize = SHA1_DIGEST_SIZE,
	150	.init = sha1_base_init,
	151	.update = sha1_avx_update,
	152	.final = sha1_avx_final,
	153	.finup = sha1_avx_finup,
	154	.descsize = sizeof(struct sha1_state),
	155	.base = {
	156	.cra_name = "sha1",
	157	.cra_driver_name = "sha1-avx",
	158	.cra_priority = 160,
85c66ecd	159	.cra_blocksize = SHA1_BLOCK_SIZE,
	160	.cra_module = THIS_MODULE,
	161	}
	162	};
	163
	164	static bool avx_usable(void)
66be8951	165	{
d91cab78	166	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE \| XFEATURE_MASK_YMM, NULL)) {
da154e82	167	if (boot_cpu_has(X86_FEATURE_AVX))
d1e50966	168	pr_info("AVX detected but unusable.\n");
66be8951 MK	169	return false;
	170	}
	171
	172	return true;
	173	}
6ca5afb8	174
85c66ecd	175	static int register_sha1_avx(void)
	176	{
	177	if (avx_usable())
	178	return crypto_register_shash(&sha1_avx_alg);
	179	return 0;
	180	}
	181
	182	static void unregister_sha1_avx(void)
6ca5afb8	183	{
85c66ecd	184	if (avx_usable())
	185	crypto_unregister_shash(&sha1_avx_alg);
	186	}
	187
	188	#else /* CONFIG_AS_AVX */
	189	static inline int register_sha1_avx(void) { return 0; }
	190	static inline void unregister_sha1_avx(void) { }
	191	#endif /* CONFIG_AS_AVX */
	192
	193
	194	#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
	195	#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 464 bytes of SHA1 blocks /
	196
	197	asmlinkage void sha1_transform_avx2(u32 digest, const char data,
	198	unsigned int rounds);
	199
	200	static bool avx2_usable(void)
6ca5afb8	201	{
8861249c	202	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
85c66ecd	203	&& boot_cpu_has(X86_FEATURE_BMI1)
85c66ecd	204	&& boot_cpu_has(X86_FEATURE_BMI2))
6ca5afb8 MK	205	return true;
	206
	207	return false;
	208	}
66be8951	209
85c66ecd	210	static void sha1_apply_transform_avx2(u32 digest, const char data,
85c66ecd	211	unsigned int rounds)
66be8951	212	{
85c66ecd	213	/* Select the optimal transform based on data block size */
	214	if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
	215	sha1_transform_avx2(digest, data, rounds);
	216	else
	217	sha1_transform_avx(digest, data, rounds);
	218	}
6ca5afb8	219
85c66ecd	220	static int sha1_avx2_update(struct shash_desc desc, const u8 data,
	221	unsigned int len)
	222	{
	223	return sha1_update(desc, data, len,
	224	(sha1_transform_fn *) sha1_apply_transform_avx2);
	225	}
66be8951	226
85c66ecd	227	static int sha1_avx2_finup(struct shash_desc desc, const u8 data,
	228	unsigned int len, u8 *out)
	229	{
	230	return sha1_finup(desc, data, len, out,
	231	(sha1_transform_fn *) sha1_apply_transform_avx2);
	232	}
	233
	234	static int sha1_avx2_final(struct shash_desc desc, u8 out)
	235	{
	236	return sha1_avx2_finup(desc, NULL, 0, out);
	237	}
	238
	239	static struct shash_alg sha1_avx2_alg = {
	240	.digestsize = SHA1_DIGEST_SIZE,
	241	.init = sha1_base_init,
	242	.update = sha1_avx2_update,
	243	.final = sha1_avx2_final,
	244	.finup = sha1_avx2_finup,
	245	.descsize = sizeof(struct sha1_state),
	246	.base = {
	247	.cra_name = "sha1",
	248	.cra_driver_name = "sha1-avx2",
	249	.cra_priority = 170,
85c66ecd	250	.cra_blocksize = SHA1_BLOCK_SIZE,
85c66ecd	251	.cra_module = THIS_MODULE,
7c1da8d0	252	}
85c66ecd	253	};
	254
	255	static int register_sha1_avx2(void)
	256	{
	257	if (avx2_usable())
	258	return crypto_register_shash(&sha1_avx2_alg);
	259	return 0;
	260	}
	261
	262	static void unregister_sha1_avx2(void)
	263	{
	264	if (avx2_usable())
	265	crypto_unregister_shash(&sha1_avx2_alg);
	266	}
	267
	268	#else
	269	static inline int register_sha1_avx2(void) { return 0; }
	270	static inline void unregister_sha1_avx2(void) { }
6ca5afb8	271	#endif
85c66ecd	272
95fca7df	273	#ifdef CONFIG_AS_SHA1_NI
85c66ecd	274	asmlinkage void sha1_ni_transform(u32 digest, const char data,
	275	unsigned int rounds);
	276
	277	static int sha1_ni_update(struct shash_desc desc, const u8 data,
	278	unsigned int len)
	279	{
	280	return sha1_update(desc, data, len,
	281	(sha1_transform_fn *) sha1_ni_transform);
	282	}
	283
	284	static int sha1_ni_finup(struct shash_desc desc, const u8 data,
	285	unsigned int len, u8 *out)
	286	{
	287	return sha1_finup(desc, data, len, out,
	288	(sha1_transform_fn *) sha1_ni_transform);
	289	}
	290
	291	static int sha1_ni_final(struct shash_desc desc, u8 out)
	292	{
	293	return sha1_ni_finup(desc, NULL, 0, out);
	294	}
	295
	296	static struct shash_alg sha1_ni_alg = {
	297	.digestsize = SHA1_DIGEST_SIZE,
	298	.init = sha1_base_init,
	299	.update = sha1_ni_update,
	300	.final = sha1_ni_final,
	301	.finup = sha1_ni_finup,
	302	.descsize = sizeof(struct sha1_state),
	303	.base = {
	304	.cra_name = "sha1",
	305	.cra_driver_name = "sha1-ni",
	306	.cra_priority = 250,
85c66ecd	307	.cra_blocksize = SHA1_BLOCK_SIZE,
85c66ecd	308	.cra_module = THIS_MODULE,
95fca7df	309	}
85c66ecd	310	};
	311
	312	static int register_sha1_ni(void)
	313	{
	314	if (boot_cpu_has(X86_FEATURE_SHA_NI))
	315	return crypto_register_shash(&sha1_ni_alg);
	316	return 0;
	317	}
	318
	319	static void unregister_sha1_ni(void)
	320	{
	321	if (boot_cpu_has(X86_FEATURE_SHA_NI))
	322	crypto_unregister_shash(&sha1_ni_alg);
	323	}
	324
	325	#else
	326	static inline int register_sha1_ni(void) { return 0; }
	327	static inline void unregister_sha1_ni(void) { }
66be8951 MK	328	#endif
	329
	330	static int __init sha1_ssse3_mod_init(void)
	331	{
85c66ecd	332	if (register_sha1_ssse3())
85c66ecd	333	goto fail;
6ca5afb8	334
85c66ecd	335	if (register_sha1_avx()) {
	336	unregister_sha1_ssse3();
	337	goto fail;
7c1da8d0	338	}
66be8951	339
85c66ecd	340	if (register_sha1_avx2()) {
	341	unregister_sha1_avx();
	342	unregister_sha1_ssse3();
	343	goto fail;
7c1da8d0	344	}
66be8951	345
85c66ecd	346	if (register_sha1_ni()) {
	347	unregister_sha1_avx2();
	348	unregister_sha1_avx();
	349	unregister_sha1_ssse3();
	350	goto fail;
66be8951	351	}
66be8951	352
85c66ecd	353	return 0;
85c66ecd	354	fail:
66be8951 MK	355	return -ENODEV;
	356	}
	357
	358	static void __exit sha1_ssse3_mod_fini(void)
	359	{
85c66ecd	360	unregister_sha1_ni();
	361	unregister_sha1_avx2();
	362	unregister_sha1_avx();
	363	unregister_sha1_ssse3();
66be8951 MK	364	}
	365
	366	module_init(sha1_ssse3_mod_init);
	367	module_exit(sha1_ssse3_mod_fini);
	368
	369	MODULE_LICENSE("GPL");
	370	MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
	371
5d26a105	372	MODULE_ALIAS_CRYPTO("sha1");
1a445e8e SM	373	MODULE_ALIAS_CRYPTO("sha1-ssse3");
	374	MODULE_ALIAS_CRYPTO("sha1-avx");
	375	MODULE_ALIAS_CRYPTO("sha1-avx2");
	376	#ifdef CONFIG_AS_SHA1_NI
	377	MODULE_ALIAS_CRYPTO("sha1-ni");
	378	#endif