[linux-2.6-block.git] / arch / arm64 / crypto / crc32-arm64.c

/*
 * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
 *
 * Module based on crypto/crc32c_generic.c
 *
 * CRC32 loop taken from Ed Nevill's Hadoop CRC patch
 * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
 *
 * Using inline assembly instead of intrinsics in order to be backwards
 * compatible with older compilers.
 *
 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/unaligned/access_ok.h>
#include <linux/cpufeature.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>

#include <crypto/internal/hash.h>

MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
MODULE_LICENSE("GPL v2");

#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))

static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
{
	s64 length = len;

	while ((length -= sizeof(u64)) >= 0) {
		CRC32X(crc, get_unaligned_le64(p));
		p += sizeof(u64);
	}

	/* The following is more efficient than the straight loop */
	if (length & sizeof(u32)) {
		CRC32W(crc, get_unaligned_le32(p));
		p += sizeof(u32);
	}
	if (length & sizeof(u16)) {
		CRC32H(crc, get_unaligned_le16(p));
		p += sizeof(u16);
	}
	if (length & sizeof(u8))
		CRC32B(crc, *p);

	return crc;
}

static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
{
	s64 length = len;

	while ((length -= sizeof(u64)) >= 0) {
		CRC32CX(crc, get_unaligned_le64(p));
		p += sizeof(u64);
	}

	/* The following is more efficient than the straight loop */
	if (length & sizeof(u32)) {
		CRC32CW(crc, get_unaligned_le32(p));
		p += sizeof(u32);
	}
	if (length & sizeof(u16)) {
		CRC32CH(crc, get_unaligned_le16(p));
		p += sizeof(u16);
	}
	if (length & sizeof(u8))
		CRC32CB(crc, *p);

	return crc;
}

#define CHKSUM_BLOCK_SIZE	1
#define CHKSUM_DIGEST_SIZE	4

struct chksum_ctx {
	u32 key;
};

struct chksum_desc_ctx {
	u32 crc;
};

static int chksum_init(struct shash_desc *desc)
{
	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	ctx->crc = mctx->key;

	return 0;
}

/*
 * Setting the seed allows arbitrary accumulators and flexible XOR policy
 * If your algorithm starts with ~0, then XOR with ~0 before you set
 * the seed.
 */
static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
			 unsigned int keylen)
{
	struct chksum_ctx *mctx = crypto_shash_ctx(tfm);

	if (keylen != sizeof(mctx->key)) {
		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return -EINVAL;
	}
	mctx->key = get_unaligned_le32(key);
	return 0;
}

static int chksum_update(struct shash_desc *desc, const u8 *data,
			 unsigned int length)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
	return 0;
}

static int chksumc_update(struct shash_desc *desc, const u8 *data,
			 unsigned int length)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
	return 0;
}

static int chksum_final(struct shash_desc *desc, u8 *out)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	put_unaligned_le32(ctx->crc, out);
	return 0;
}

static int chksumc_final(struct shash_desc *desc, u8 *out)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	put_unaligned_le32(~ctx->crc, out);
	return 0;
}

static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
{
	put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
	return 0;
}

static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
{
	put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
	return 0;
}

static int chksum_finup(struct shash_desc *desc, const u8 *data,
			unsigned int len, u8 *out)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	return __chksum_finup(ctx->crc, data, len, out);
}

static int chksumc_finup(struct shash_desc *desc, const u8 *data,
			unsigned int len, u8 *out)
{
	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

	return __chksumc_finup(ctx->crc, data, len, out);
}

static int chksum_digest(struct shash_desc *desc, const u8 *data,
			 unsigned int length, u8 *out)
{
	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);

	return __chksum_finup(mctx->key, data, length, out);
}

static int chksumc_digest(struct shash_desc *desc, const u8 *data,
			 unsigned int length, u8 *out)
{
	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);

	return __chksumc_finup(mctx->key, data, length, out);
}

static int crc32_cra_init(struct crypto_tfm *tfm)
{
	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);

	mctx->key = 0;
	return 0;
}

static int crc32c_cra_init(struct crypto_tfm *tfm)
{
	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);

	mctx->key = ~0;
	return 0;
}

static struct shash_alg crc32_alg = {
	.digestsize		=	CHKSUM_DIGEST_SIZE,
	.setkey			=	chksum_setkey,
	.init			=	chksum_init,
	.update			=	chksum_update,
	.final			=	chksum_final,
	.finup			=	chksum_finup,
	.digest			=	chksum_digest,
	.descsize		=	sizeof(struct chksum_desc_ctx),
	.base			=	{
		.cra_name		=	"crc32",
		.cra_driver_name	=	"crc32-arm64-hw",
		.cra_priority		=	300,
		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
		.cra_alignmask		=	0,
		.cra_ctxsize		=	sizeof(struct chksum_ctx),
		.cra_module		=	THIS_MODULE,
		.cra_init		=	crc32_cra_init,
	}
};

static struct shash_alg crc32c_alg = {
	.digestsize		=	CHKSUM_DIGEST_SIZE,
	.setkey			=	chksum_setkey,
	.init			=	chksum_init,
	.update			=	chksumc_update,
	.final			=	chksumc_final,
	.finup			=	chksumc_finup,
	.digest			=	chksumc_digest,
	.descsize		=	sizeof(struct chksum_desc_ctx),
	.base			=	{
		.cra_name		=	"crc32c",
		.cra_driver_name	=	"crc32c-arm64-hw",
		.cra_priority		=	300,
		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
		.cra_alignmask		=	0,
		.cra_ctxsize		=	sizeof(struct chksum_ctx),
		.cra_module		=	THIS_MODULE,
		.cra_init		=	crc32c_cra_init,
	}
};

static int __init crc32_mod_init(void)
{
	int err;

	err = crypto_register_shash(&crc32_alg);

	if (err)
		return err;

	err = crypto_register_shash(&crc32c_alg);

	if (err) {
		crypto_unregister_shash(&crc32_alg);
		return err;
	}

	return 0;
}

static void __exit crc32_mod_exit(void)
{
	crypto_unregister_shash(&crc32_alg);
	crypto_unregister_shash(&crc32c_alg);
}

module_cpu_feature_match(CRC32, crc32_mod_init);
module_exit(crc32_mod_exit);
Commit	Line	Data
f6f203fa YG	1	/*
	2	* crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
	3	*
	4	* Module based on crypto/crc32c_generic.c
	5	*
	6	* CRC32 loop taken from Ed Nevill's Hadoop CRC patch
	7	* http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
	8	*
	9	* Using inline assembly instead of intrinsics in order to be backwards
	10	* compatible with older compilers.
	11	*
	12	* Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
	13	*
	14	* This program is free software; you can redistribute it and/or modify
	15	* it under the terms of the GNU General Public License version 2 as
	16	* published by the Free Software Foundation.
	17	*/
	18
	19	#include <linux/unaligned/access_ok.h>
	20	#include <linux/cpufeature.h>
	21	#include <linux/init.h>
	22	#include <linux/kernel.h>
	23	#include <linux/module.h>
	24	#include <linux/string.h>
	25
	26	#include <crypto/internal/hash.h>
	27
	28	MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
	29	MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
	30	MODULE_LICENSE("GPL v2");
	31
	32	#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
	33	#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	34	#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	35	#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	36	#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
	37	#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	38	#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	39	#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
	40
	41	static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
	42	{
	43	s64 length = len;
	44
	45	while ((length -= sizeof(u64)) >= 0) {
	46	CRC32X(crc, get_unaligned_le64(p));
	47	p += sizeof(u64);
	48	}
	49
	50	/* The following is more efficient than the straight loop */
	51	if (length & sizeof(u32)) {
	52	CRC32W(crc, get_unaligned_le32(p));
	53	p += sizeof(u32);
	54	}
	55	if (length & sizeof(u16)) {
	56	CRC32H(crc, get_unaligned_le16(p));
	57	p += sizeof(u16);
	58	}
	59	if (length & sizeof(u8))
	60	CRC32B(crc, *p);
	61
	62	return crc;
	63	}
	64
65	static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
66	{
67	s64 length = len;
68
69	while ((length -= sizeof(u64)) >= 0) {
70	CRC32CX(crc, get_unaligned_le64(p));
71	p += sizeof(u64);
72	}
73
74	/* The following is more efficient than the straight loop */
75	if (length & sizeof(u32)) {
76	CRC32CW(crc, get_unaligned_le32(p));
77	p += sizeof(u32);
78	}
79	if (length & sizeof(u16)) {
80	CRC32CH(crc, get_unaligned_le16(p));
81	p += sizeof(u16);
82	}
83	if (length & sizeof(u8))
84	CRC32CB(crc, *p);
85
86	return crc;
87	}
88
89	#define CHKSUM_BLOCK_SIZE 1
90	#define CHKSUM_DIGEST_SIZE 4
91
92	struct chksum_ctx {
93	u32 key;
94	};
95
96	struct chksum_desc_ctx {
97	u32 crc;
98	};
99
100	static int chksum_init(struct shash_desc *desc)
101	{
102	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
103	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
104
105	ctx->crc = mctx->key;
106
107	return 0;
108	}
109
110	/*
111	* Setting the seed allows arbitrary accumulators and flexible XOR policy
112	* If your algorithm starts with ~0, then XOR with ~0 before you set
113	* the seed.
114	*/
115	static int chksum_setkey(struct crypto_shash tfm, const u8 key,
116	unsigned int keylen)
117	{
118	struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
119
120	if (keylen != sizeof(mctx->key)) {
121	crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
122	return -EINVAL;
123	}
124	mctx->key = get_unaligned_le32(key);
125	return 0;
126	}
127
128	static int chksum_update(struct shash_desc desc, const u8 data,
129	unsigned int length)
130	{
131	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
132
133	ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
134	return 0;
135	}
136
137	static int chksumc_update(struct shash_desc desc, const u8 data,
138	unsigned int length)
139	{
140	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
141
142	ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
143	return 0;
144	}
145
146	static int chksum_final(struct shash_desc desc, u8 out)
147	{
148	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
149
ac02c6ea AB	150	put_unaligned_le32(ctx->crc, out);
	151	return 0;
	152	}
	153
	154	static int chksumc_final(struct shash_desc desc, u8 out)
	155	{
	156	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
	157
f6f203fa YG	158	put_unaligned_le32(~ctx->crc, out);
	159	return 0;
	160	}
	161
	162	static int __chksum_finup(u32 crc, const u8 data, unsigned int len, u8 out)
	163	{
ac02c6ea	164	put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
f6f203fa YG	165	return 0;
	166	}
	167
	168	static int __chksumc_finup(u32 crc, const u8 data, unsigned int len, u8 out)
	169	{
	170	put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
	171	return 0;
	172	}
	173
	174	static int chksum_finup(struct shash_desc desc, const u8 data,
	175	unsigned int len, u8 *out)
	176	{
	177	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
	178
	179	return __chksum_finup(ctx->crc, data, len, out);
	180	}
	181
	182	static int chksumc_finup(struct shash_desc desc, const u8 data,
	183	unsigned int len, u8 *out)
	184	{
	185	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
	186
	187	return __chksumc_finup(ctx->crc, data, len, out);
	188	}
	189
	190	static int chksum_digest(struct shash_desc desc, const u8 data,
	191	unsigned int length, u8 *out)
	192	{
	193	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
	194
	195	return __chksum_finup(mctx->key, data, length, out);
	196	}
	197
	198	static int chksumc_digest(struct shash_desc desc, const u8 data,
	199	unsigned int length, u8 *out)
	200	{
	201	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
	202
	203	return __chksumc_finup(mctx->key, data, length, out);
	204	}
	205
	206	static int crc32_cra_init(struct crypto_tfm *tfm)
	207	{
	208	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
	209
ac02c6ea AB	210	mctx->key = 0;
	211	return 0;
	212	}
	213
	214	static int crc32c_cra_init(struct crypto_tfm *tfm)
	215	{
	216	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
	217
f6f203fa YG	218	mctx->key = ~0;
	219	return 0;
	220	}
	221
	222	static struct shash_alg crc32_alg = {
	223	.digestsize = CHKSUM_DIGEST_SIZE,
	224	.setkey = chksum_setkey,
	225	.init = chksum_init,
	226	.update = chksum_update,
	227	.final = chksum_final,
	228	.finup = chksum_finup,
	229	.digest = chksum_digest,
	230	.descsize = sizeof(struct chksum_desc_ctx),
	231	.base = {
	232	.cra_name = "crc32",
	233	.cra_driver_name = "crc32-arm64-hw",
	234	.cra_priority = 300,
	235	.cra_blocksize = CHKSUM_BLOCK_SIZE,
	236	.cra_alignmask = 0,
	237	.cra_ctxsize = sizeof(struct chksum_ctx),
	238	.cra_module = THIS_MODULE,
	239	.cra_init = crc32_cra_init,
	240	}
	241	};
	242
	243	static struct shash_alg crc32c_alg = {
	244	.digestsize = CHKSUM_DIGEST_SIZE,
	245	.setkey = chksum_setkey,
	246	.init = chksum_init,
	247	.update = chksumc_update,
ac02c6ea	248	.final = chksumc_final,
f6f203fa YG	249	.finup = chksumc_finup,
	250	.digest = chksumc_digest,
	251	.descsize = sizeof(struct chksum_desc_ctx),
	252	.base = {
	253	.cra_name = "crc32c",
	254	.cra_driver_name = "crc32c-arm64-hw",
	255	.cra_priority = 300,
	256	.cra_blocksize = CHKSUM_BLOCK_SIZE,
	257	.cra_alignmask = 0,
	258	.cra_ctxsize = sizeof(struct chksum_ctx),
	259	.cra_module = THIS_MODULE,
ac02c6ea	260	.cra_init = crc32c_cra_init,
f6f203fa YG	261	}
	262	};
	263
	264	static int __init crc32_mod_init(void)
	265	{
	266	int err;
	267
	268	err = crypto_register_shash(&crc32_alg);
	269
	270	if (err)
	271	return err;
	272
	273	err = crypto_register_shash(&crc32c_alg);
	274
	275	if (err) {
	276	crypto_unregister_shash(&crc32_alg);
	277	return err;
	278	}
	279
	280	return 0;
	281	}
	282
	283	static void __exit crc32_mod_exit(void)
	284	{
	285	crypto_unregister_shash(&crc32_alg);
	286	crypto_unregister_shash(&crc32c_alg);
	287	}
	288
	289	module_cpu_feature_match(CRC32, crc32_mod_init);
	290	module_exit(crc32_mod_exit);