[linux-block.git] / drivers / crypto / cavium / nitrox / nitrox_aead.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/printk.h>
#include <linux/crypto.h>
#include <linux/rtnetlink.h>

#include <crypto/aead.h>
#include <crypto/authenc.h>
#include <crypto/des.h>
#include <crypto/sha.h>
#include <crypto/internal/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/gcm.h>

#include "nitrox_dev.h"
#include "nitrox_common.h"
#include "nitrox_req.h"

#define GCM_AES_SALT_SIZE	4

union gph_p3 {
	struct {
#ifdef __BIG_ENDIAN_BITFIELD
		u16 iv_offset : 8;
		u16 auth_offset	: 8;
#else
		u16 auth_offset	: 8;
		u16 iv_offset : 8;
#endif
	};
	u16 param;
};

static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
				 unsigned int keylen)
{
	int aes_keylen;
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct flexi_crypto_context *fctx;
	union fc_ctx_flags flags;

	aes_keylen = flexi_aes_keylen(keylen);
	if (aes_keylen < 0)
		return -EINVAL;

	/* fill crypto context */
	fctx = nctx->u.fctx;
	flags.f = be64_to_cpu(fctx->flags.f);
	flags.w0.aes_keylen = aes_keylen;
	fctx->flags.f = cpu_to_be64(flags.f);

	/* copy enc key to context */
	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
	memcpy(fctx->crypto.u.key, key, keylen);

	return 0;
}

static int nitrox_aead_setauthsize(struct crypto_aead *aead,
				   unsigned int authsize)
{
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct flexi_crypto_context *fctx = nctx->u.fctx;
	union fc_ctx_flags flags;

	flags.f = be64_to_cpu(fctx->flags.f);
	flags.w0.mac_len = authsize;
	fctx->flags.f = cpu_to_be64(flags.f);

	aead->authsize = authsize;

	return 0;
}

static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
				      unsigned int authsize)
{
	switch (authsize) {
	case 4:
	case 8:
	case 12:
	case 13:
	case 14:
	case 15:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return nitrox_aead_setauthsize(aead, authsize);
}

static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
			    struct scatterlist *src, char *iv, int ivsize,
			    int buflen)
{
	int nents = sg_nents_for_len(src, buflen);
	int ret;

	if (nents < 0)
		return nents;

	/* IV entry */
	nents += 1;
	/* Allocate buffer to hold IV and input scatterlist array */
	ret = alloc_src_req_buf(nkreq, nents, ivsize);
	if (ret)
		return ret;

	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
	nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);

	return 0;
}

static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
			    struct scatterlist *dst, int ivsize, int buflen)
{
	int nents = sg_nents_for_len(dst, buflen);
	int ret;

	if (nents < 0)
		return nents;

	/* IV, ORH, COMPLETION entries */
	nents += 3;
	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
	 * array
	 */
	ret = alloc_dst_req_buf(nkreq, nents);
	if (ret)
		return ret;

	nitrox_creq_set_orh(nkreq);
	nitrox_creq_set_comp(nkreq);
	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);

	return 0;
}

static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
{
	kfree(nkreq->src);
}

static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
{
	kfree(nkreq->dst);
}

static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
{
	struct se_crypto_request *creq = &rctx->nkreq.creq;
	union gph_p3 param3;
	int ret;

	creq->flags = rctx->flags;
	creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
							       GFP_ATOMIC;

	creq->ctrl.value = 0;
	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
	creq->ctrl.s.arg = rctx->ctrl_arg;

	creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
	creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
	creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
	param3.iv_offset = 0;
	param3.auth_offset = rctx->ivsize;
	creq->gph.param3 = cpu_to_be16(param3.param);

	creq->ctx_handle = rctx->ctx_handle;
	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);

	ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
			       rctx->srclen);
	if (ret)
		return ret;

	ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
			       rctx->dstlen);
	if (ret) {
		free_src_sglist(&rctx->nkreq);
		return ret;
	}

	return 0;
}

static void nitrox_aead_callback(void *arg, int err)
{
	struct aead_request *areq = arg;
	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);

	free_src_sglist(&rctx->nkreq);
	free_dst_sglist(&rctx->nkreq);
	if (err) {
		pr_err_ratelimited("request failed status 0x%0x\n", err);
		err = -EINVAL;
	}

	areq->base.complete(&areq->base, err);
}

static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
{
	if (assoclen <= 512)
		return true;

	return false;
}

static int nitrox_aes_gcm_enc(struct aead_request *areq)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
	struct se_crypto_request *creq = &rctx->nkreq.creq;
	struct flexi_crypto_context *fctx = nctx->u.fctx;
	int ret;

	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
		return -EINVAL;

	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);

	rctx->cryptlen = areq->cryptlen;
	rctx->assoclen = areq->assoclen;
	rctx->srclen = areq->assoclen + areq->cryptlen;
	rctx->dstlen = rctx->srclen + aead->authsize;
	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	rctx->flags = areq->base.flags;
	rctx->ctx_handle = nctx->u.ctx_handle;
	rctx->src = areq->src;
	rctx->dst = areq->dst;
	rctx->ctrl_arg = ENCRYPT;
	ret = nitrox_set_creq(rctx);
	if (ret)
		return ret;

	/* send the crypto request */
	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
					 areq);
}

static int nitrox_aes_gcm_dec(struct aead_request *areq)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
	struct se_crypto_request *creq = &rctx->nkreq.creq;
	struct flexi_crypto_context *fctx = nctx->u.fctx;
	int ret;

	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
		return -EINVAL;

	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);

	rctx->cryptlen = areq->cryptlen - aead->authsize;
	rctx->assoclen = areq->assoclen;
	rctx->srclen = areq->cryptlen + areq->assoclen;
	rctx->dstlen = rctx->srclen - aead->authsize;
	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	rctx->flags = areq->base.flags;
	rctx->ctx_handle = nctx->u.ctx_handle;
	rctx->src = areq->src;
	rctx->dst = areq->dst;
	rctx->ctrl_arg = DECRYPT;
	ret = nitrox_set_creq(rctx);
	if (ret)
		return ret;

	/* send the crypto request */
	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
					 areq);
}

static int nitrox_aead_init(struct crypto_aead *aead)
{
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct crypto_ctx_hdr *chdr;

	/* get the first device */
	nctx->ndev = nitrox_get_first_device();
	if (!nctx->ndev)
		return -ENODEV;

	/* allocate nitrox crypto context */
	chdr = crypto_alloc_context(nctx->ndev);
	if (!chdr) {
		nitrox_put_device(nctx->ndev);
		return -ENOMEM;
	}
	nctx->chdr = chdr;
	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
					 sizeof(struct ctx_hdr));
	nctx->u.fctx->flags.f = 0;

	return 0;
}

static int nitrox_gcm_common_init(struct crypto_aead *aead)
{
	int ret;
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	union fc_ctx_flags *flags;

	ret = nitrox_aead_init(aead);
	if (ret)
		return ret;

	flags = &nctx->u.fctx->flags;
	flags->w0.cipher_type = CIPHER_AES_GCM;
	flags->w0.hash_type = AUTH_NULL;
	flags->w0.iv_source = IV_FROM_DPTR;
	/* ask microcode to calculate ipad/opad */
	flags->w0.auth_input_type = 1;
	flags->f = be64_to_cpu(flags->f);

	return 0;
}

static int nitrox_aes_gcm_init(struct crypto_aead *aead)
{
	int ret;

	ret = nitrox_gcm_common_init(aead);
	if (ret)
		return ret;

	crypto_aead_set_reqsize(aead,
				sizeof(struct aead_request) +
					sizeof(struct nitrox_aead_rctx));

	return 0;
}

static void nitrox_aead_exit(struct crypto_aead *aead)
{
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);

	/* free the nitrox crypto context */
	if (nctx->u.ctx_handle) {
		struct flexi_crypto_context *fctx = nctx->u.fctx;

		memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
		memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
		crypto_free_context((void *)nctx->chdr);
	}
	nitrox_put_device(nctx->ndev);

	nctx->u.ctx_handle = 0;
	nctx->ndev = NULL;
}

static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
				 unsigned int keylen)
{
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct flexi_crypto_context *fctx = nctx->u.fctx;
	int ret;

	if (keylen < GCM_AES_SALT_SIZE)
		return -EINVAL;

	keylen -= GCM_AES_SALT_SIZE;
	ret = nitrox_aes_gcm_setkey(aead, key, keylen);
	if (ret)
		return ret;

	memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
	return 0;
}

static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
				      unsigned int authsize)
{
	switch (authsize) {
	case 8:
	case 12:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return nitrox_aead_setauthsize(aead, authsize);
}

static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
{
	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
	unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
	struct scatterlist *sg;

	if (areq->assoclen != 16 && areq->assoclen != 20)
		return -EINVAL;

	scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
	sg_init_table(rctx->src, 3);
	sg_set_buf(rctx->src, rctx->assoc, assoclen);
	sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
	if (sg != rctx->src + 1)
		sg_chain(rctx->src, 2, sg);

	if (areq->src != areq->dst) {
		sg_init_table(rctx->dst, 3);
		sg_set_buf(rctx->dst, rctx->assoc, assoclen);
		sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
		if (sg != rctx->dst + 1)
			sg_chain(rctx->dst, 2, sg);
	}

	aead_rctx->src = rctx->src;
	aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;

	return 0;
}

static void nitrox_rfc4106_callback(void *arg, int err)
{
	struct aead_request *areq = arg;
	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
	struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;

	free_src_sglist(nkreq);
	free_dst_sglist(nkreq);
	if (err) {
		pr_err_ratelimited("request failed status 0x%0x\n", err);
		err = -EINVAL;
	}

	areq->base.complete(&areq->base, err);
}

static int nitrox_rfc4106_enc(struct aead_request *areq)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
	int ret;

	aead_rctx->cryptlen = areq->cryptlen;
	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
	aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
	aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
	aead_rctx->iv = areq->iv;
	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
	aead_rctx->flags = areq->base.flags;
	aead_rctx->ctx_handle = nctx->u.ctx_handle;
	aead_rctx->ctrl_arg = ENCRYPT;

	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
	if (ret)
		return ret;

	ret = nitrox_set_creq(aead_rctx);
	if (ret)
		return ret;

	/* send the crypto request */
	return nitrox_process_se_request(nctx->ndev, creq,
					 nitrox_rfc4106_callback, areq);
}

static int nitrox_rfc4106_dec(struct aead_request *areq)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
	int ret;

	aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
	aead_rctx->srclen =
		areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
	aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
	aead_rctx->iv = areq->iv;
	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
	aead_rctx->flags = areq->base.flags;
	aead_rctx->ctx_handle = nctx->u.ctx_handle;
	aead_rctx->ctrl_arg = DECRYPT;

	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
	if (ret)
		return ret;

	ret = nitrox_set_creq(aead_rctx);
	if (ret)
		return ret;

	/* send the crypto request */
	return nitrox_process_se_request(nctx->ndev, creq,
					 nitrox_rfc4106_callback, areq);
}

static int nitrox_rfc4106_init(struct crypto_aead *aead)
{
	int ret;

	ret = nitrox_gcm_common_init(aead);
	if (ret)
		return ret;

	crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
				sizeof(struct nitrox_rfc4106_rctx));

	return 0;
}

static struct aead_alg nitrox_aeads[] = { {
	.base = {
		.cra_name = "gcm(aes)",
		.cra_driver_name = "n5_aes_gcm",
		.cra_priority = PRIO,
		.cra_flags = CRYPTO_ALG_ASYNC,
		.cra_blocksize = 1,
		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
		.cra_alignmask = 0,
		.cra_module = THIS_MODULE,
	},
	.setkey = nitrox_aes_gcm_setkey,
	.setauthsize = nitrox_aes_gcm_setauthsize,
	.encrypt = nitrox_aes_gcm_enc,
	.decrypt = nitrox_aes_gcm_dec,
	.init = nitrox_aes_gcm_init,
	.exit = nitrox_aead_exit,
	.ivsize = GCM_AES_IV_SIZE,
	.maxauthsize = AES_BLOCK_SIZE,
}, {
	.base = {
		.cra_name = "rfc4106(gcm(aes))",
		.cra_driver_name = "n5_rfc4106",
		.cra_priority = PRIO,
		.cra_flags = CRYPTO_ALG_ASYNC,
		.cra_blocksize = 1,
		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
		.cra_alignmask = 0,
		.cra_module = THIS_MODULE,
	},
	.setkey = nitrox_rfc4106_setkey,
	.setauthsize = nitrox_rfc4106_setauthsize,
	.encrypt = nitrox_rfc4106_enc,
	.decrypt = nitrox_rfc4106_dec,
	.init = nitrox_rfc4106_init,
	.exit = nitrox_aead_exit,
	.ivsize = GCM_RFC4106_IV_SIZE,
	.maxauthsize = AES_BLOCK_SIZE,
} };

int nitrox_register_aeads(void)
{
	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
}

void nitrox_unregister_aeads(void)
{
	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
}
Commit	Line	Data
c9613335 NR	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/kernel.h>
	3	#include <linux/printk.h>
	4	#include <linux/crypto.h>
	5	#include <linux/rtnetlink.h>
	6
	7	#include <crypto/aead.h>
	8	#include <crypto/authenc.h>
	9	#include <crypto/des.h>
	10	#include <crypto/sha.h>
	11	#include <crypto/internal/aead.h>
	12	#include <crypto/scatterwalk.h>
	13	#include <crypto/gcm.h>
	14
	15	#include "nitrox_dev.h"
	16	#include "nitrox_common.h"
	17	#include "nitrox_req.h"
	18
	19	#define GCM_AES_SALT_SIZE 4
	20
c9613335 NR	21	union gph_p3 {
	22	struct {
	23	#ifdef __BIG_ENDIAN_BITFIELD
	24	u16 iv_offset : 8;
	25	u16 auth_offset : 8;
	26	#else
	27	u16 auth_offset : 8;
	28	u16 iv_offset : 8;
	29	#endif
	30	};
	31	u16 param;
	32	};
	33
	34	static int nitrox_aes_gcm_setkey(struct crypto_aead aead, const u8 key,
	35	unsigned int keylen)
	36	{
	37	int aes_keylen;
	38	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	39	struct flexi_crypto_context *fctx;
	40	union fc_ctx_flags flags;
	41
	42	aes_keylen = flexi_aes_keylen(keylen);
674f368a	43	if (aes_keylen < 0)
c9613335	44	return -EINVAL;
c9613335 NR	45
	46	/* fill crypto context */
	47	fctx = nctx->u.fctx;
	48	flags.f = be64_to_cpu(fctx->flags.f);
	49	flags.w0.aes_keylen = aes_keylen;
	50	fctx->flags.f = cpu_to_be64(flags.f);
	51
	52	/* copy enc key to context */
	53	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
	54	memcpy(fctx->crypto.u.key, key, keylen);
	55
	56	return 0;
	57	}
	58
	59	static int nitrox_aead_setauthsize(struct crypto_aead *aead,
	60	unsigned int authsize)
	61	{
	62	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	63	struct flexi_crypto_context *fctx = nctx->u.fctx;
	64	union fc_ctx_flags flags;
	65
	66	flags.f = be64_to_cpu(fctx->flags.f);
	67	flags.w0.mac_len = authsize;
	68	fctx->flags.f = cpu_to_be64(flags.f);
	69
	70	aead->authsize = authsize;
	71
	72	return 0;
	73	}
	74
3fed9baa NR	75	static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
	76	unsigned int authsize)
	77	{
	78	switch (authsize) {
	79	case 4:
	80	case 8:
	81	case 12:
	82	case 13:
	83	case 14:
	84	case 15:
	85	case 16:
	86	break;
	87	default:
	88	return -EINVAL;
	89	}
	90
	91	return nitrox_aead_setauthsize(aead, authsize);
	92	}
	93
8a3719a5 NR	94	static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
8a3719a5 NR	95	struct scatterlist src, char iv, int ivsize,
c9613335 NR	96	int buflen)
c9613335 NR	97	{
8a3719a5	98	int nents = sg_nents_for_len(src, buflen);
c9613335 NR	99	int ret;
	100
	101	if (nents < 0)
	102	return nents;
	103
8a3719a5 NR	104	/* IV entry */
8a3719a5 NR	105	nents += 1;
c9613335 NR	106	/* Allocate buffer to hold IV and input scatterlist array */
	107	ret = alloc_src_req_buf(nkreq, nents, ivsize);
	108	if (ret)
	109	return ret;
	110
	111	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
8a3719a5	112	nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
c9613335 NR	113
	114	return 0;
	115	}
	116
8a3719a5 NR	117	static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
8a3719a5 NR	118	struct scatterlist *dst, int ivsize, int buflen)
c9613335	119	{
8a3719a5	120	int nents = sg_nents_for_len(dst, buflen);
c9613335 NR	121	int ret;
	122
	123	if (nents < 0)
	124	return nents;
	125
8a3719a5 NR	126	/* IV, ORH, COMPLETION entries */
8a3719a5 NR	127	nents += 3;
c9613335 NR	128	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
	129	* array
	130	*/
	131	ret = alloc_dst_req_buf(nkreq, nents);
	132	if (ret)
	133	return ret;
	134
	135	nitrox_creq_set_orh(nkreq);
	136	nitrox_creq_set_comp(nkreq);
8a3719a5	137	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
c9613335 NR	138
	139	return 0;
	140	}
	141
8a3719a5	142	static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
c9613335	143	{
c9613335 NR	144	kfree(nkreq->src);
	145	}
	146
8a3719a5	147	static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
c9613335	148	{
c9613335 NR	149	kfree(nkreq->dst);
	150	}
	151
8a3719a5	152	static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
c9613335	153	{
8a3719a5	154	struct se_crypto_request *creq = &rctx->nkreq.creq;
c9613335	155	union gph_p3 param3;
c9613335 NR	156	int ret;
c9613335 NR	157
8a3719a5 NR	158	creq->flags = rctx->flags;
	159	creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
	160	GFP_ATOMIC;
c9613335 NR	161
	162	creq->ctrl.value = 0;
	163	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
8a3719a5	164	creq->ctrl.s.arg = rctx->ctrl_arg;
c9613335	165
8a3719a5 NR	166	creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
	167	creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
	168	creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
c9613335	169	param3.iv_offset = 0;
8a3719a5	170	param3.auth_offset = rctx->ivsize;
c9613335 NR	171	creq->gph.param3 = cpu_to_be16(param3.param);
c9613335 NR	172
8a3719a5	173	creq->ctx_handle = rctx->ctx_handle;
c9613335 NR	174	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
c9613335 NR	175
8a3719a5 NR	176	ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
8a3719a5 NR	177	rctx->srclen);
c9613335 NR	178	if (ret)
	179	return ret;
	180
8a3719a5 NR	181	ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
8a3719a5 NR	182	rctx->dstlen);
c9613335	183	if (ret) {
8a3719a5	184	free_src_sglist(&rctx->nkreq);
c9613335 NR	185	return ret;
	186	}
	187
	188	return 0;
	189	}
	190
	191	static void nitrox_aead_callback(void *arg, int err)
	192	{
	193	struct aead_request *areq = arg;
8a3719a5	194	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
c9613335	195
8a3719a5 NR	196	free_src_sglist(&rctx->nkreq);
8a3719a5 NR	197	free_dst_sglist(&rctx->nkreq);
c9613335 NR	198	if (err) {
	199	pr_err_ratelimited("request failed status 0x%0x\n", err);
	200	err = -EINVAL;
	201	}
	202
	203	areq->base.complete(&areq->base, err);
	204	}
	205
3fed9baa NR	206	static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
	207	{
	208	if (assoclen <= 512)
	209	return true;
	210
	211	return false;
	212	}
	213
c9613335 NR	214	static int nitrox_aes_gcm_enc(struct aead_request *areq)
	215	{
	216	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	217	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
8a3719a5 NR	218	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
8a3719a5 NR	219	struct se_crypto_request *creq = &rctx->nkreq.creq;
c9613335	220	struct flexi_crypto_context *fctx = nctx->u.fctx;
c9613335 NR	221	int ret;
c9613335 NR	222
3fed9baa NR	223	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
	224	return -EINVAL;
	225
c9613335 NR	226	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
c9613335 NR	227
8a3719a5 NR	228	rctx->cryptlen = areq->cryptlen;
	229	rctx->assoclen = areq->assoclen;
	230	rctx->srclen = areq->assoclen + areq->cryptlen;
	231	rctx->dstlen = rctx->srclen + aead->authsize;
	232	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	233	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	234	rctx->flags = areq->base.flags;
	235	rctx->ctx_handle = nctx->u.ctx_handle;
	236	rctx->src = areq->src;
	237	rctx->dst = areq->dst;
	238	rctx->ctrl_arg = ENCRYPT;
	239	ret = nitrox_set_creq(rctx);
c9613335 NR	240	if (ret)
	241	return ret;
	242
	243	/* send the crypto request */
	244	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
	245	areq);
	246	}
	247
	248	static int nitrox_aes_gcm_dec(struct aead_request *areq)
	249	{
	250	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
	251	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
8a3719a5 NR	252	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
8a3719a5 NR	253	struct se_crypto_request *creq = &rctx->nkreq.creq;
c9613335	254	struct flexi_crypto_context *fctx = nctx->u.fctx;
c9613335 NR	255	int ret;
c9613335 NR	256
3fed9baa NR	257	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
	258	return -EINVAL;
	259
c9613335 NR	260	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
c9613335 NR	261
8a3719a5 NR	262	rctx->cryptlen = areq->cryptlen - aead->authsize;
	263	rctx->assoclen = areq->assoclen;
	264	rctx->srclen = areq->cryptlen + areq->assoclen;
	265	rctx->dstlen = rctx->srclen - aead->authsize;
	266	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
	267	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
	268	rctx->flags = areq->base.flags;
	269	rctx->ctx_handle = nctx->u.ctx_handle;
	270	rctx->src = areq->src;
	271	rctx->dst = areq->dst;
	272	rctx->ctrl_arg = DECRYPT;
	273	ret = nitrox_set_creq(rctx);
c9613335 NR	274	if (ret)
	275	return ret;
	276
	277	/* send the crypto request */
	278	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
	279	areq);
	280	}
	281
	282	static int nitrox_aead_init(struct crypto_aead *aead)
	283	{
	284	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	285	struct crypto_ctx_hdr *chdr;
	286
	287	/* get the first device */
	288	nctx->ndev = nitrox_get_first_device();
	289	if (!nctx->ndev)
	290	return -ENODEV;
	291
	292	/* allocate nitrox crypto context */
	293	chdr = crypto_alloc_context(nctx->ndev);
	294	if (!chdr) {
	295	nitrox_put_device(nctx->ndev);
	296	return -ENOMEM;
	297	}
	298	nctx->chdr = chdr;
	299	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
	300	sizeof(struct ctx_hdr));
	301	nctx->u.fctx->flags.f = 0;
	302
	303	return 0;
	304	}
	305
8a3719a5	306	static int nitrox_gcm_common_init(struct crypto_aead *aead)
c9613335 NR	307	{
	308	int ret;
	309	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	310	union fc_ctx_flags *flags;
	311
	312	ret = nitrox_aead_init(aead);
	313	if (ret)
	314	return ret;
	315
	316	flags = &nctx->u.fctx->flags;
	317	flags->w0.cipher_type = CIPHER_AES_GCM;
	318	flags->w0.hash_type = AUTH_NULL;
	319	flags->w0.iv_source = IV_FROM_DPTR;
	320	/* ask microcode to calculate ipad/opad */
	321	flags->w0.auth_input_type = 1;
	322	flags->f = be64_to_cpu(flags->f);
	323
8a3719a5 NR	324	return 0;
	325	}
	326
	327	static int nitrox_aes_gcm_init(struct crypto_aead *aead)
	328	{
	329	int ret;
	330
	331	ret = nitrox_gcm_common_init(aead);
	332	if (ret)
	333	return ret;
	334
	335	crypto_aead_set_reqsize(aead,
	336	sizeof(struct aead_request) +
	337	sizeof(struct nitrox_aead_rctx));
c9613335 NR	338
	339	return 0;
	340	}
	341
	342	static void nitrox_aead_exit(struct crypto_aead *aead)
	343	{
	344	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	345
	346	/* free the nitrox crypto context */
	347	if (nctx->u.ctx_handle) {
	348	struct flexi_crypto_context *fctx = nctx->u.fctx;
	349
	350	memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
	351	memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
	352	crypto_free_context((void *)nctx->chdr);
	353	}
	354	nitrox_put_device(nctx->ndev);
	355
	356	nctx->u.ctx_handle = 0;
	357	nctx->ndev = NULL;
	358	}
	359
8a3719a5 NR	360	static int nitrox_rfc4106_setkey(struct crypto_aead aead, const u8 key,
	361	unsigned int keylen)
	362	{
	363	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
	364	struct flexi_crypto_context *fctx = nctx->u.fctx;
	365	int ret;
	366
	367	if (keylen < GCM_AES_SALT_SIZE)
	368	return -EINVAL;
	369
	370	keylen -= GCM_AES_SALT_SIZE;
	371	ret = nitrox_aes_gcm_setkey(aead, key, keylen);
	372	if (ret)
	373	return ret;
	374
	375	memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
	376	return 0;
	377	}
	378
	379	static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
	380	unsigned int authsize)
	381	{
	382	switch (authsize) {
	383	case 8:
	384	case 12:
	385	case 16:
	386	break;
	387	default:
	388	return -EINVAL;
	389	}
	390
	391	return nitrox_aead_setauthsize(aead, authsize);
	392	}
	393
	394	static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
	395	{
	396	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
	397	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
	398	unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
	399	struct scatterlist *sg;
	400
	401	if (areq->assoclen != 16 && areq->assoclen != 20)
	402	return -EINVAL;
	403
	404	scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
	405	sg_init_table(rctx->src, 3);
	406	sg_set_buf(rctx->src, rctx->assoc, assoclen);
	407	sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
	408	if (sg != rctx->src + 1)
	409	sg_chain(rctx->src, 2, sg);
	410
	411	if (areq->src != areq->dst) {
	412	sg_init_table(rctx->dst, 3);
	413	sg_set_buf(rctx->dst, rctx->assoc, assoclen);
	414	sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
	415	if (sg != rctx->dst + 1)
	416	sg_chain(rctx->dst, 2, sg);
	417	}
	418
	419	aead_rctx->src = rctx->src;
	420	aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
	421
	422	return 0;
	423	}
424
425	static void nitrox_rfc4106_callback(void *arg, int err)
426	{
427	struct aead_request *areq = arg;
428	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
429	struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
430
431	free_src_sglist(nkreq);
432	free_dst_sglist(nkreq);
433	if (err) {
434	pr_err_ratelimited("request failed status 0x%0x\n", err);
435	err = -EINVAL;
436	}
437
438	areq->base.complete(&areq->base, err);
439	}
440
441	static int nitrox_rfc4106_enc(struct aead_request *areq)
442	{
443	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
444	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
445	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
446	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
447	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
448	int ret;
449
450	aead_rctx->cryptlen = areq->cryptlen;
451	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
452	aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
453	aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
454	aead_rctx->iv = areq->iv;
455	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
456	aead_rctx->flags = areq->base.flags;
457	aead_rctx->ctx_handle = nctx->u.ctx_handle;
458	aead_rctx->ctrl_arg = ENCRYPT;
459
460	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
461	if (ret)
462	return ret;
463
464	ret = nitrox_set_creq(aead_rctx);
465	if (ret)
466	return ret;
467
468	/* send the crypto request */
469	return nitrox_process_se_request(nctx->ndev, creq,
470	nitrox_rfc4106_callback, areq);
471	}
472
473	static int nitrox_rfc4106_dec(struct aead_request *areq)
474	{
475	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
476	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
477	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
478	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
479	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
480	int ret;
481
482	aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
483	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
484	aead_rctx->srclen =
485	areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
486	aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
487	aead_rctx->iv = areq->iv;
488	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
489	aead_rctx->flags = areq->base.flags;
490	aead_rctx->ctx_handle = nctx->u.ctx_handle;
491	aead_rctx->ctrl_arg = DECRYPT;
492
493	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
494	if (ret)
495	return ret;
496
497	ret = nitrox_set_creq(aead_rctx);
498	if (ret)
499	return ret;
500
501	/* send the crypto request */
502	return nitrox_process_se_request(nctx->ndev, creq,
503	nitrox_rfc4106_callback, areq);
504	}
505
506	static int nitrox_rfc4106_init(struct crypto_aead *aead)
507	{
508	int ret;
509
510	ret = nitrox_gcm_common_init(aead);
511	if (ret)
512	return ret;
513
514	crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
515	sizeof(struct nitrox_rfc4106_rctx));
516
517	return 0;
518	}
519
c9613335 NR	520	static struct aead_alg nitrox_aeads[] = { {
	521	.base = {
	522	.cra_name = "gcm(aes)",
	523	.cra_driver_name = "n5_aes_gcm",
	524	.cra_priority = PRIO,
	525	.cra_flags = CRYPTO_ALG_ASYNC,
3fed9baa	526	.cra_blocksize = 1,
c9613335 NR	527	.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
	528	.cra_alignmask = 0,
	529	.cra_module = THIS_MODULE,
	530	},
	531	.setkey = nitrox_aes_gcm_setkey,
3fed9baa	532	.setauthsize = nitrox_aes_gcm_setauthsize,
c9613335 NR	533	.encrypt = nitrox_aes_gcm_enc,
	534	.decrypt = nitrox_aes_gcm_dec,
	535	.init = nitrox_aes_gcm_init,
	536	.exit = nitrox_aead_exit,
	537	.ivsize = GCM_AES_IV_SIZE,
	538	.maxauthsize = AES_BLOCK_SIZE,
8a3719a5 NR	539	}, {
	540	.base = {
	541	.cra_name = "rfc4106(gcm(aes))",
	542	.cra_driver_name = "n5_rfc4106",
	543	.cra_priority = PRIO,
	544	.cra_flags = CRYPTO_ALG_ASYNC,
3fed9baa	545	.cra_blocksize = 1,
8a3719a5 NR	546	.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
	547	.cra_alignmask = 0,
	548	.cra_module = THIS_MODULE,
	549	},
	550	.setkey = nitrox_rfc4106_setkey,
	551	.setauthsize = nitrox_rfc4106_setauthsize,
	552	.encrypt = nitrox_rfc4106_enc,
	553	.decrypt = nitrox_rfc4106_dec,
	554	.init = nitrox_rfc4106_init,
	555	.exit = nitrox_aead_exit,
	556	.ivsize = GCM_RFC4106_IV_SIZE,
	557	.maxauthsize = AES_BLOCK_SIZE,
c9613335 NR	558	} };
	559
	560	int nitrox_register_aeads(void)
	561	{
	562	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
	563	}
	564
	565	void nitrox_unregister_aeads(void)
	566	{
	567	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
	568	}