[linux-2.6-block.git] / arch / sparc / crypto / des_glue.c

// SPDX-License-Identifier: GPL-2.0-only
/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
 *
 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
 */

#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt

#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/algapi.h>
#include <crypto/des.h>

#include <asm/fpumacro.h>
#include <asm/pstate.h>
#include <asm/elf.h>

#include "opcodes.h"

struct des_sparc64_ctx {
	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
};

struct des3_ede_sparc64_ctx {
	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
};

static void encrypt_to_decrypt(u64 *d, const u64 *e)
{
	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
	int i;

	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
		*d++ = *s--;
}

extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);

static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
		       unsigned int keylen)
{
	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	u32 tmp[DES_EXPKEY_WORDS];
	int ret;

	/* Even though we have special instructions for key expansion,
	 * we call des_ekey() so that we don't have to write our own
	 * weak key detection code.
	 */
	ret = des_ekey(tmp, key);
	if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
		*flags |= CRYPTO_TFM_RES_WEAK_KEY;
		return -EINVAL;
	}

	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);

	return 0;
}

extern void des_sparc64_crypt(const u64 *key, const u64 *input,
			      u64 *output);

static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->encrypt_expkey;

	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->decrypt_expkey;

	des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

extern void des_sparc64_load_keys(const u64 *key);

extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
				  unsigned int len);

#define DES_BLOCK_MASK	(~(DES_BLOCK_SIZE - 1))

static int __ecb_crypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes, bool encrypt)
{
	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	if (encrypt)
		des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	else
		des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
					      (u64 *) walk.dst.virt.addr,
					      block_len);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

static int ecb_encrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	return __ecb_crypt(desc, dst, src, nbytes, true);
}

static int ecb_decrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	return __ecb_crypt(desc, dst, src, nbytes, false);
}

extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
				    unsigned int len, u64 *iv);

static int cbc_encrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
						(u64 *) walk.dst.virt.addr,
						block_len, (u64 *) walk.iv);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
				    unsigned int len, u64 *iv);

static int cbc_decrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
						(u64 *) walk.dst.virt.addr,
						block_len, (u64 *) walk.iv);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
			    unsigned int keylen)
{
	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
	u32 *flags = &tfm->crt_flags;
	u64 k1[DES_EXPKEY_WORDS / 2];
	u64 k2[DES_EXPKEY_WORDS / 2];
	u64 k3[DES_EXPKEY_WORDS / 2];
	int err;

	err = __des3_verify_key(flags, key);
	if (unlikely(err))
		return err;

	des_sparc64_key_expand((const u32 *)key, k1);
	key += DES_KEY_SIZE;
	des_sparc64_key_expand((const u32 *)key, k2);
	key += DES_KEY_SIZE;
	des_sparc64_key_expand((const u32 *)key, k3);

	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	       &k3[0], sizeof(k3));

	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
	       &k2[0], sizeof(k2));
	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
			   &k1[0]);

	return 0;
}

extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
				   u64 *output);

static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->encrypt_expkey;

	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	const u64 *K = ctx->decrypt_expkey;

	des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
}

extern void des3_ede_sparc64_load_keys(const u64 *key);

extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
				       u64 *output, unsigned int len);

static int __ecb3_crypt(struct blkcipher_desc *desc,
			struct scatterlist *dst, struct scatterlist *src,
			unsigned int nbytes, bool encrypt)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	const u64 *K;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	if (encrypt)
		K = &ctx->encrypt_expkey[0];
	else
		K = &ctx->decrypt_expkey[0];
	des3_ede_sparc64_load_keys(K);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			const u64 *src64 = (const u64 *)walk.src.virt.addr;
			des3_ede_sparc64_ecb_crypt(K, src64,
						   (u64 *) walk.dst.virt.addr,
						   block_len);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

static int ecb3_encrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	return __ecb3_crypt(desc, dst, src, nbytes, true);
}

static int ecb3_decrypt(struct blkcipher_desc *desc,
		       struct scatterlist *dst, struct scatterlist *src,
		       unsigned int nbytes)
{
	return __ecb3_crypt(desc, dst, src, nbytes, false);
}

extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
					 u64 *output, unsigned int len,
					 u64 *iv);

static int cbc3_encrypt(struct blkcipher_desc *desc,
			struct scatterlist *dst, struct scatterlist *src,
			unsigned int nbytes)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	const u64 *K;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	K = &ctx->encrypt_expkey[0];
	des3_ede_sparc64_load_keys(K);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			const u64 *src64 = (const u64 *)walk.src.virt.addr;
			des3_ede_sparc64_cbc_encrypt(K, src64,
						     (u64 *) walk.dst.virt.addr,
						     block_len,
						     (u64 *) walk.iv);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
					 u64 *output, unsigned int len,
					 u64 *iv);

static int cbc3_decrypt(struct blkcipher_desc *desc,
			struct scatterlist *dst, struct scatterlist *src,
			unsigned int nbytes)
{
	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	struct blkcipher_walk walk;
	const u64 *K;
	int err;

	blkcipher_walk_init(&walk, dst, src, nbytes);
	err = blkcipher_walk_virt(desc, &walk);
	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	K = &ctx->decrypt_expkey[0];
	des3_ede_sparc64_load_keys(K);
	while ((nbytes = walk.nbytes)) {
		unsigned int block_len = nbytes & DES_BLOCK_MASK;

		if (likely(block_len)) {
			const u64 *src64 = (const u64 *)walk.src.virt.addr;
			des3_ede_sparc64_cbc_decrypt(K, src64,
						     (u64 *) walk.dst.virt.addr,
						     block_len,
						     (u64 *) walk.iv);
		}
		nbytes &= DES_BLOCK_SIZE - 1;
		err = blkcipher_walk_done(desc, &walk, nbytes);
	}
	fprs_write(0);
	return err;
}

static struct crypto_alg algs[] = { {
	.cra_name		= "des",
	.cra_driver_name	= "des-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
	.cra_blocksize		= DES_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_module		= THIS_MODULE,
	.cra_u	= {
		.cipher	= {
			.cia_min_keysize	= DES_KEY_SIZE,
			.cia_max_keysize	= DES_KEY_SIZE,
			.cia_setkey		= des_set_key,
			.cia_encrypt		= des_encrypt,
			.cia_decrypt		= des_decrypt
		}
	}
}, {
	.cra_name		= "ecb(des)",
	.cra_driver_name	= "ecb-des-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= DES_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= DES_KEY_SIZE,
			.max_keysize	= DES_KEY_SIZE,
			.setkey		= des_set_key,
			.encrypt	= ecb_encrypt,
			.decrypt	= ecb_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(des)",
	.cra_driver_name	= "cbc-des-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= DES_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= DES_KEY_SIZE,
			.max_keysize	= DES_KEY_SIZE,
			.ivsize		= DES_BLOCK_SIZE,
			.setkey		= des_set_key,
			.encrypt	= cbc_encrypt,
			.decrypt	= cbc_decrypt,
		},
	},
}, {
	.cra_name		= "des3_ede",
	.cra_driver_name	= "des3_ede-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_module		= THIS_MODULE,
	.cra_u	= {
		.cipher	= {
			.cia_min_keysize	= DES3_EDE_KEY_SIZE,
			.cia_max_keysize	= DES3_EDE_KEY_SIZE,
			.cia_setkey		= des3_ede_set_key,
			.cia_encrypt		= des3_ede_encrypt,
			.cia_decrypt		= des3_ede_decrypt
		}
	}
}, {
	.cra_name		= "ecb(des3_ede)",
	.cra_driver_name	= "ecb-des3_ede-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= DES3_EDE_KEY_SIZE,
			.max_keysize	= DES3_EDE_KEY_SIZE,
			.setkey		= des3_ede_set_key,
			.encrypt	= ecb3_encrypt,
			.decrypt	= ecb3_decrypt,
		},
	},
}, {
	.cra_name		= "cbc(des3_ede)",
	.cra_driver_name	= "cbc-des3_ede-sparc64",
	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
	.cra_flags		= CRYPTO_ALG_TYPE_BLKCIPHER,
	.cra_blocksize		= DES3_EDE_BLOCK_SIZE,
	.cra_ctxsize		= sizeof(struct des3_ede_sparc64_ctx),
	.cra_alignmask		= 7,
	.cra_type		= &crypto_blkcipher_type,
	.cra_module		= THIS_MODULE,
	.cra_u = {
		.blkcipher = {
			.min_keysize	= DES3_EDE_KEY_SIZE,
			.max_keysize	= DES3_EDE_KEY_SIZE,
			.ivsize		= DES3_EDE_BLOCK_SIZE,
			.setkey		= des3_ede_set_key,
			.encrypt	= cbc3_encrypt,
			.decrypt	= cbc3_decrypt,
		},
	},
} };

static bool __init sparc64_has_des_opcode(void)
{
	unsigned long cfr;

	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
		return false;

	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
	if (!(cfr & CFR_DES))
		return false;

	return true;
}

static int __init des_sparc64_mod_init(void)
{
	if (sparc64_has_des_opcode()) {
		pr_info("Using sparc64 des opcodes optimized DES implementation\n");
		return crypto_register_algs(algs, ARRAY_SIZE(algs));
	}
	pr_info("sparc64 des opcodes not available.\n");
	return -ENODEV;
}

static void __exit des_sparc64_mod_fini(void)
{
	crypto_unregister_algs(algs, ARRAY_SIZE(algs));
}

module_init(des_sparc64_mod_init);
module_exit(des_sparc64_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");

MODULE_ALIAS_CRYPTO("des");
MODULE_ALIAS_CRYPTO("des3_ede");

#include "crop_devid.c"
Commit	Line	Data
09c434b8	1	// SPDX-License-Identifier: GPL-2.0-only
c5aac2df DM	2	/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
	3	*
	4	* Copyright (C) 2012 David S. Miller <davem@davemloft.net>
	5	*/
	6
71741680 DM	7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
71741680 DM	8
c5aac2df DM	9	#include <linux/crypto.h>
	10	#include <linux/init.h>
	11	#include <linux/module.h>
	12	#include <linux/mm.h>
	13	#include <linux/types.h>
	14	#include <crypto/algapi.h>
	15	#include <crypto/des.h>
	16
	17	#include <asm/fpumacro.h>
	18	#include <asm/pstate.h>
	19	#include <asm/elf.h>
	20
10803624 DM	21	#include "opcodes.h"
10803624 DM	22
c5aac2df DM	23	struct des_sparc64_ctx {
	24	u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
	25	u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
	26	};
	27
	28	struct des3_ede_sparc64_ctx {
	29	u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	30	u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
	31	};
	32
	33	static void encrypt_to_decrypt(u64 d, const u64 e)
	34	{
	35	const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
	36	int i;
	37
	38	for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
	39	d++ = s--;
	40	}
	41
	42	extern void des_sparc64_key_expand(const u32 input_key, u64 key);
	43
	44	static int des_set_key(struct crypto_tfm tfm, const u8 key,
	45	unsigned int keylen)
	46	{
	47	struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
	48	u32 *flags = &tfm->crt_flags;
	49	u32 tmp[DES_EXPKEY_WORDS];
	50	int ret;
	51
	52	/* Even though we have special instructions for key expansion,
	53	* we call des_ekey() so that we don't have to write our own
	54	* weak key detection code.
	55	*/
	56	ret = des_ekey(tmp, key);
231baecd	57	if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) {
c5aac2df DM	58	*flags \|= CRYPTO_TFM_RES_WEAK_KEY;
	59	return -EINVAL;
	60	}
	61
	62	des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
	63	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
	64
	65	return 0;
	66	}
	67
	68	extern void des_sparc64_crypt(const u64 key, const u64 input,
	69	u64 *output);
	70
	71	static void des_encrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
	72	{
	73	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	74	const u64 *K = ctx->encrypt_expkey;
	75
	76	des_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	77	}
	78
	79	static void des_decrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
	80	{
	81	struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	82	const u64 *K = ctx->decrypt_expkey;
	83
	84	des_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	85	}
	86
	87	extern void des_sparc64_load_keys(const u64 *key);
	88
	89	extern void des_sparc64_ecb_crypt(const u64 input, u64 output,
	90	unsigned int len);
	91
	92	#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
	93
	94	static int __ecb_crypt(struct blkcipher_desc *desc,
	95	struct scatterlist dst, struct scatterlist src,
	96	unsigned int nbytes, bool encrypt)
	97	{
	98	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	99	struct blkcipher_walk walk;
	100	int err;
	101
	102	blkcipher_walk_init(&walk, dst, src, nbytes);
	103	err = blkcipher_walk_virt(desc, &walk);
b3a37947	104	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	105
	106	if (encrypt)
	107	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	108	else
	109	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	110	while ((nbytes = walk.nbytes)) {
	111	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	112
	113	if (likely(block_len)) {
	114	des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
	115	(u64 *) walk.dst.virt.addr,
	116	block_len);
	117	}
	118	nbytes &= DES_BLOCK_SIZE - 1;
	119	err = blkcipher_walk_done(desc, &walk, nbytes);
	120	}
	121	fprs_write(0);
	122	return err;
	123	}
	124
	125	static int ecb_encrypt(struct blkcipher_desc *desc,
	126	struct scatterlist dst, struct scatterlist src,
	127	unsigned int nbytes)
	128	{
	129	return __ecb_crypt(desc, dst, src, nbytes, true);
	130	}
	131
	132	static int ecb_decrypt(struct blkcipher_desc *desc,
	133	struct scatterlist dst, struct scatterlist src,
	134	unsigned int nbytes)
	135	{
	136	return __ecb_crypt(desc, dst, src, nbytes, false);
	137	}
	138
	139	extern void des_sparc64_cbc_encrypt(const u64 input, u64 output,
	140	unsigned int len, u64 *iv);
	141
	142	static int cbc_encrypt(struct blkcipher_desc *desc,
	143	struct scatterlist dst, struct scatterlist src,
	144	unsigned int nbytes)
	145	{
	146	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	147	struct blkcipher_walk walk;
	148	int err;
	149
	150	blkcipher_walk_init(&walk, dst, src, nbytes);
	151	err = blkcipher_walk_virt(desc, &walk);
b3a37947	152	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	153
	154	des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
	155	while ((nbytes = walk.nbytes)) {
	156	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	157
	158	if (likely(block_len)) {
	159	des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
	160	(u64 *) walk.dst.virt.addr,
	161	block_len, (u64 *) walk.iv);
	162	}
	163	nbytes &= DES_BLOCK_SIZE - 1;
	164	err = blkcipher_walk_done(desc, &walk, nbytes);
	165	}
	166	fprs_write(0);
	167	return err;
	168	}
	169
	170	extern void des_sparc64_cbc_decrypt(const u64 input, u64 output,
	171	unsigned int len, u64 *iv);
	172
	173	static int cbc_decrypt(struct blkcipher_desc *desc,
	174	struct scatterlist dst, struct scatterlist src,
	175	unsigned int nbytes)
	176	{
	177	struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	178	struct blkcipher_walk walk;
	179	int err;
	180
	181	blkcipher_walk_init(&walk, dst, src, nbytes);
	182	err = blkcipher_walk_virt(desc, &walk);
b3a37947	183	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	184
	185	des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
	186	while ((nbytes = walk.nbytes)) {
	187	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	188
	189	if (likely(block_len)) {
	190	des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
	191	(u64 *) walk.dst.virt.addr,
	192	block_len, (u64 *) walk.iv);
	193	}
	194	nbytes &= DES_BLOCK_SIZE - 1;
	195	err = blkcipher_walk_done(desc, &walk, nbytes);
	196	}
	197	fprs_write(0);
	198	return err;
	199	}
	200
	201	static int des3_ede_set_key(struct crypto_tfm tfm, const u8 key,
	202	unsigned int keylen)
	203	{
	204	struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
c5aac2df DM	205	u32 *flags = &tfm->crt_flags;
	206	u64 k1[DES_EXPKEY_WORDS / 2];
	207	u64 k2[DES_EXPKEY_WORDS / 2];
	208	u64 k3[DES_EXPKEY_WORDS / 2];
eee25da5	209	int err;
c5aac2df	210
eee25da5 HX	211	err = __des3_verify_key(flags, key);
	212	if (unlikely(err))
	213	return err;
c5aac2df DM	214
	215	des_sparc64_key_expand((const u32 *)key, k1);
	216	key += DES_KEY_SIZE;
	217	des_sparc64_key_expand((const u32 *)key, k2);
	218	key += DES_KEY_SIZE;
	219	des_sparc64_key_expand((const u32 *)key, k3);
	220
	221	memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
	222	encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
	223	memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	224	&k3[0], sizeof(k3));
	225
	226	encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
	227	memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
	228	&k2[0], sizeof(k2));
	229	encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
	230	&k1[0]);
	231
	232	return 0;
	233	}
	234
	235	extern void des3_ede_sparc64_crypt(const u64 key, const u64 input,
	236	u64 *output);
	237
	238	static void des3_ede_encrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
	239	{
	240	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	241	const u64 *K = ctx->encrypt_expkey;
	242
	243	des3_ede_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	244	}
	245
	246	static void des3_ede_decrypt(struct crypto_tfm tfm, u8 dst, const u8 *src)
	247	{
	248	struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
	249	const u64 *K = ctx->decrypt_expkey;
	250
	251	des3_ede_sparc64_crypt(K, (const u64 ) src, (u64 ) dst);
	252	}
	253
	254	extern void des3_ede_sparc64_load_keys(const u64 *key);
	255
	256	extern void des3_ede_sparc64_ecb_crypt(const u64 expkey, const u64 input,
	257	u64 *output, unsigned int len);
	258
	259	static int __ecb3_crypt(struct blkcipher_desc *desc,
	260	struct scatterlist dst, struct scatterlist src,
	261	unsigned int nbytes, bool encrypt)
	262	{
	263	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	264	struct blkcipher_walk walk;
	265	const u64 *K;
	266	int err;
	267
	268	blkcipher_walk_init(&walk, dst, src, nbytes);
	269	err = blkcipher_walk_virt(desc, &walk);
b3a37947	270	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	271
	272	if (encrypt)
	273	K = &ctx->encrypt_expkey[0];
	274	else
	275	K = &ctx->decrypt_expkey[0];
	276	des3_ede_sparc64_load_keys(K);
	277	while ((nbytes = walk.nbytes)) {
	278	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	279
	280	if (likely(block_len)) {
	281	const u64 src64 = (const u64 )walk.src.virt.addr;
	282	des3_ede_sparc64_ecb_crypt(K, src64,
	283	(u64 *) walk.dst.virt.addr,
	284	block_len);
	285	}
	286	nbytes &= DES_BLOCK_SIZE - 1;
	287	err = blkcipher_walk_done(desc, &walk, nbytes);
	288	}
	289	fprs_write(0);
	290	return err;
	291	}
	292
	293	static int ecb3_encrypt(struct blkcipher_desc *desc,
	294	struct scatterlist dst, struct scatterlist src,
	295	unsigned int nbytes)
	296	{
	297	return __ecb3_crypt(desc, dst, src, nbytes, true);
	298	}
	299
	300	static int ecb3_decrypt(struct blkcipher_desc *desc,
	301	struct scatterlist dst, struct scatterlist src,
	302	unsigned int nbytes)
	303	{
	304	return __ecb3_crypt(desc, dst, src, nbytes, false);
	305	}
	306
	307	extern void des3_ede_sparc64_cbc_encrypt(const u64 expkey, const u64 input,
	308	u64 *output, unsigned int len,
	309	u64 *iv);
	310
	311	static int cbc3_encrypt(struct blkcipher_desc *desc,
	312	struct scatterlist dst, struct scatterlist src,
	313	unsigned int nbytes)
	314	{
	315	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	316	struct blkcipher_walk walk;
	317	const u64 *K;
	318	int err;
	319
	320	blkcipher_walk_init(&walk, dst, src, nbytes);
	321	err = blkcipher_walk_virt(desc, &walk);
b3a37947	322	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	323
	324	K = &ctx->encrypt_expkey[0];
	325	des3_ede_sparc64_load_keys(K);
	326	while ((nbytes = walk.nbytes)) {
	327	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	328
	329	if (likely(block_len)) {
	330	const u64 src64 = (const u64 )walk.src.virt.addr;
	331	des3_ede_sparc64_cbc_encrypt(K, src64,
	332	(u64 *) walk.dst.virt.addr,
	333	block_len,
	334	(u64 *) walk.iv);
	335	}
	336	nbytes &= DES_BLOCK_SIZE - 1;
	337	err = blkcipher_walk_done(desc, &walk, nbytes);
	338	}
	339	fprs_write(0);
	340	return err;
	341	}
	342
	343	extern void des3_ede_sparc64_cbc_decrypt(const u64 expkey, const u64 input,
	344	u64 *output, unsigned int len,
	345	u64 *iv);
	346
	347	static int cbc3_decrypt(struct blkcipher_desc *desc,
	348	struct scatterlist dst, struct scatterlist src,
	349	unsigned int nbytes)
	350	{
	351	struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
	352	struct blkcipher_walk walk;
	353	const u64 *K;
	354	int err;
	355
	356	blkcipher_walk_init(&walk, dst, src, nbytes);
	357	err = blkcipher_walk_virt(desc, &walk);
b3a37947	358	desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
c5aac2df DM	359
	360	K = &ctx->decrypt_expkey[0];
	361	des3_ede_sparc64_load_keys(K);
	362	while ((nbytes = walk.nbytes)) {
	363	unsigned int block_len = nbytes & DES_BLOCK_MASK;
	364
	365	if (likely(block_len)) {
	366	const u64 src64 = (const u64 )walk.src.virt.addr;
	367	des3_ede_sparc64_cbc_decrypt(K, src64,
	368	(u64 *) walk.dst.virt.addr,
	369	block_len,
	370	(u64 *) walk.iv);
	371	}
	372	nbytes &= DES_BLOCK_SIZE - 1;
	373	err = blkcipher_walk_done(desc, &walk, nbytes);
	374	}
	375	fprs_write(0);
	376	return err;
	377	}
	378
	379	static struct crypto_alg algs[] = { {
	380	.cra_name = "des",
	381	.cra_driver_name = "des-sparc64",
10803624	382	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	383	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
	384	.cra_blocksize = DES_BLOCK_SIZE,
	385	.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	386	.cra_alignmask = 7,
	387	.cra_module = THIS_MODULE,
	388	.cra_u = {
	389	.cipher = {
	390	.cia_min_keysize = DES_KEY_SIZE,
	391	.cia_max_keysize = DES_KEY_SIZE,
	392	.cia_setkey = des_set_key,
	393	.cia_encrypt = des_encrypt,
	394	.cia_decrypt = des_decrypt
	395	}
	396	}
	397	}, {
	398	.cra_name = "ecb(des)",
	399	.cra_driver_name = "ecb-des-sparc64",
10803624	400	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	401	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	402	.cra_blocksize = DES_BLOCK_SIZE,
	403	.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	404	.cra_alignmask = 7,
	405	.cra_type = &crypto_blkcipher_type,
	406	.cra_module = THIS_MODULE,
	407	.cra_u = {
	408	.blkcipher = {
	409	.min_keysize = DES_KEY_SIZE,
	410	.max_keysize = DES_KEY_SIZE,
	411	.setkey = des_set_key,
	412	.encrypt = ecb_encrypt,
	413	.decrypt = ecb_decrypt,
	414	},
	415	},
	416	}, {
	417	.cra_name = "cbc(des)",
	418	.cra_driver_name = "cbc-des-sparc64",
10803624	419	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	420	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	421	.cra_blocksize = DES_BLOCK_SIZE,
	422	.cra_ctxsize = sizeof(struct des_sparc64_ctx),
	423	.cra_alignmask = 7,
	424	.cra_type = &crypto_blkcipher_type,
	425	.cra_module = THIS_MODULE,
	426	.cra_u = {
	427	.blkcipher = {
	428	.min_keysize = DES_KEY_SIZE,
	429	.max_keysize = DES_KEY_SIZE,
a66d7f72	430	.ivsize = DES_BLOCK_SIZE,
c5aac2df DM	431	.setkey = des_set_key,
	432	.encrypt = cbc_encrypt,
	433	.decrypt = cbc_decrypt,
	434	},
	435	},
	436	}, {
	437	.cra_name = "des3_ede",
	438	.cra_driver_name = "des3_ede-sparc64",
10803624	439	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	440	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
	441	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	442	.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	443	.cra_alignmask = 7,
	444	.cra_module = THIS_MODULE,
	445	.cra_u = {
	446	.cipher = {
	447	.cia_min_keysize = DES3_EDE_KEY_SIZE,
	448	.cia_max_keysize = DES3_EDE_KEY_SIZE,
	449	.cia_setkey = des3_ede_set_key,
	450	.cia_encrypt = des3_ede_encrypt,
	451	.cia_decrypt = des3_ede_decrypt
	452	}
	453	}
	454	}, {
	455	.cra_name = "ecb(des3_ede)",
	456	.cra_driver_name = "ecb-des3_ede-sparc64",
10803624	457	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	458	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	459	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	460	.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	461	.cra_alignmask = 7,
	462	.cra_type = &crypto_blkcipher_type,
	463	.cra_module = THIS_MODULE,
	464	.cra_u = {
	465	.blkcipher = {
	466	.min_keysize = DES3_EDE_KEY_SIZE,
	467	.max_keysize = DES3_EDE_KEY_SIZE,
	468	.setkey = des3_ede_set_key,
	469	.encrypt = ecb3_encrypt,
	470	.decrypt = ecb3_decrypt,
	471	},
	472	},
	473	}, {
	474	.cra_name = "cbc(des3_ede)",
	475	.cra_driver_name = "cbc-des3_ede-sparc64",
10803624	476	.cra_priority = SPARC_CR_OPCODE_PRIORITY,
c5aac2df DM	477	.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
	478	.cra_blocksize = DES3_EDE_BLOCK_SIZE,
	479	.cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
	480	.cra_alignmask = 7,
	481	.cra_type = &crypto_blkcipher_type,
	482	.cra_module = THIS_MODULE,
	483	.cra_u = {
	484	.blkcipher = {
	485	.min_keysize = DES3_EDE_KEY_SIZE,
	486	.max_keysize = DES3_EDE_KEY_SIZE,
a66d7f72	487	.ivsize = DES3_EDE_BLOCK_SIZE,
c5aac2df DM	488	.setkey = des3_ede_set_key,
	489	.encrypt = cbc3_encrypt,
	490	.decrypt = cbc3_decrypt,
	491	},
	492	},
	493	} };
	494
	495	static bool __init sparc64_has_des_opcode(void)
	496	{
	497	unsigned long cfr;
	498
	499	if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
	500	return false;
	501
	502	__asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
	503	if (!(cfr & CFR_DES))
	504	return false;
	505
	506	return true;
	507	}
	508
	509	static int __init des_sparc64_mod_init(void)
	510	{
c5aac2df DM	511	if (sparc64_has_des_opcode()) {
	512	pr_info("Using sparc64 des opcodes optimized DES implementation\n");
	513	return crypto_register_algs(algs, ARRAY_SIZE(algs));
	514	}
	515	pr_info("sparc64 des opcodes not available.\n");
	516	return -ENODEV;
	517	}
	518
	519	static void __exit des_sparc64_mod_fini(void)
	520	{
	521	crypto_unregister_algs(algs, ARRAY_SIZE(algs));
	522	}
	523
	524	module_init(des_sparc64_mod_init);
	525	module_exit(des_sparc64_mod_fini);
	526
	527	MODULE_LICENSE("GPL");
	528	MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
	529
5d26a105	530	MODULE_ALIAS_CRYPTO("des");
5001323b	531	MODULE_ALIAS_CRYPTO("des3_ede");
226f7cea DM	532
226f7cea DM	533	#include "crop_devid.c"