[linux-2.6-block.git] / drivers / crypto / omap-aes-gcm.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Cryptographic API.
 *
 * Support for OMAP AES GCM HW acceleration.
 *
 * Copyright (c) 2016 Texas Instruments Incorporated
 */

#include <crypto/aes.h>
#include <crypto/engine.h>
#include <crypto/gcm.h>
#include <crypto/internal/aead.h>
#include <crypto/scatterwalk.h>
#include <crypto/skcipher.h>
#include <linux/errno.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/omap-dma.h>
#include <linux/pm_runtime.h>
#include <linux/scatterlist.h>
#include <linux/string.h>

#include "omap-crypto.h"
#include "omap-aes.h"

static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
				     struct aead_request *req);

static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
{
	struct aead_request *req = dd->aead_req;

	dd->in_sg = NULL;
	dd->out_sg = NULL;

	crypto_finalize_aead_request(dd->engine, req, ret);

	pm_runtime_mark_last_busy(dd->dev);
	pm_runtime_put_autosuspend(dd->dev);
}

static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
{
	u8 *tag;
	int alen, clen, i, ret = 0, nsg;
	struct omap_aes_reqctx *rctx;

	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
	clen = ALIGN(dd->total, AES_BLOCK_SIZE);
	rctx = aead_request_ctx(dd->aead_req);

	nsg = !!(dd->assoc_len && dd->total);

	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
			       DMA_FROM_DEVICE);
	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
	omap_aes_crypt_dma_stop(dd);

	omap_crypto_cleanup(dd->out_sg, dd->orig_out,
			    dd->aead_req->assoclen, dd->total,
			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);

	if (dd->flags & FLAGS_ENCRYPT)
		scatterwalk_map_and_copy(rctx->auth_tag,
					 dd->aead_req->dst,
					 dd->total + dd->aead_req->assoclen,
					 dd->authsize, 1);

	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
			    FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);

	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);

	if (!(dd->flags & FLAGS_ENCRYPT)) {
		tag = (u8 *)rctx->auth_tag;
		for (i = 0; i < dd->authsize; i++) {
			if (tag[i]) {
				ret = -EBADMSG;
			}
		}
	}

	omap_aes_gcm_finish_req(dd, ret);
}

static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
				     struct aead_request *req)
{
	int alen, clen, cryptlen, assoclen, ret;
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	unsigned int authlen = crypto_aead_authsize(aead);
	struct scatterlist *tmp, sg_arr[2];
	int nsg;
	u16 flags;

	assoclen = req->assoclen;
	cryptlen = req->cryptlen;

	if (dd->flags & FLAGS_RFC4106_GCM)
		assoclen -= 8;

	if (!(dd->flags & FLAGS_ENCRYPT))
		cryptlen -= authlen;

	alen = ALIGN(assoclen, AES_BLOCK_SIZE);
	clen = ALIGN(cryptlen, AES_BLOCK_SIZE);

	nsg = !!(assoclen && cryptlen);

	omap_aes_clear_copy_flags(dd);

	sg_init_table(dd->in_sgl, nsg + 1);
	if (assoclen) {
		tmp = req->src;
		ret = omap_crypto_align_sg(&tmp, assoclen,
					   AES_BLOCK_SIZE, dd->in_sgl,
					   OMAP_CRYPTO_COPY_DATA |
					   OMAP_CRYPTO_ZERO_BUF |
					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
					   FLAGS_ASSOC_DATA_ST_SHIFT,
					   &dd->flags);
		if (ret)
			return ret;
	}

	if (cryptlen) {
		tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);

		if (nsg)
			sg_unmark_end(dd->in_sgl);

		ret = omap_crypto_align_sg(&tmp, cryptlen,
					   AES_BLOCK_SIZE, &dd->in_sgl[nsg],
					   OMAP_CRYPTO_COPY_DATA |
					   OMAP_CRYPTO_ZERO_BUF |
					   OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
					   FLAGS_IN_DATA_ST_SHIFT,
					   &dd->flags);
		if (ret)
			return ret;
	}

	dd->in_sg = dd->in_sgl;
	dd->total = cryptlen;
	dd->assoc_len = assoclen;
	dd->authsize = authlen;

	dd->out_sg = req->dst;
	dd->orig_out = req->dst;

	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);

	flags = 0;
	if (req->src == req->dst || dd->out_sg == sg_arr)
		flags |= OMAP_CRYPTO_FORCE_COPY;

	if (cryptlen) {
		ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
					   AES_BLOCK_SIZE, &dd->out_sgl,
					   flags,
					   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
		if (ret)
			return ret;
	}

	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);

	return 0;
}

static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
{
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));

	aes_encrypt(&ctx->actx, (u8 *)tag, (u8 *)iv);
	return 0;
}

void omap_aes_gcm_dma_out_callback(void *data)
{
	struct omap_aes_dev *dd = data;
	struct omap_aes_reqctx *rctx;
	int i, val;
	u32 *auth_tag, tag[4];

	if (!(dd->flags & FLAGS_ENCRYPT))
		scatterwalk_map_and_copy(tag, dd->aead_req->src,
					 dd->total + dd->aead_req->assoclen,
					 dd->authsize, 0);

	rctx = aead_request_ctx(dd->aead_req);
	auth_tag = (u32 *)rctx->auth_tag;
	for (i = 0; i < 4; i++) {
		val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
		auth_tag[i] = val ^ auth_tag[i];
		if (!(dd->flags & FLAGS_ENCRYPT))
			auth_tag[i] = auth_tag[i] ^ tag[i];
	}

	omap_aes_gcm_done_task(dd);
}

static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
				     struct aead_request *req)
{
	if (req)
		return crypto_transfer_aead_request_to_engine(dd->engine, req);

	return 0;
}

static int omap_aes_gcm_prepare_req(struct aead_request *req,
				    struct omap_aes_dev *dd)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	int err;

	dd->aead_req = req;

	rctx->mode &= FLAGS_MODE_MASK;
	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;

	err = omap_aes_gcm_copy_buffers(dd, req);
	if (err)
		return err;

	dd->ctx = &ctx->octx;

	return omap_aes_write_ctrl(dd);
}

static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	unsigned int authlen = crypto_aead_authsize(aead);
	struct omap_aes_dev *dd;
	__be32 counter = cpu_to_be32(1);
	int err, assoclen;

	memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
	memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);

	err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
	if (err)
		return err;

	if (mode & FLAGS_RFC4106_GCM)
		assoclen = req->assoclen - 8;
	else
		assoclen = req->assoclen;
	if (assoclen + req->cryptlen == 0) {
		scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
					 1);
		return 0;
	}

	dd = omap_aes_find_dev(rctx);
	if (!dd)
		return -ENODEV;
	rctx->mode = mode;

	return omap_aes_gcm_handle_queue(dd, req);
}

int omap_aes_gcm_encrypt(struct aead_request *req)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
}

int omap_aes_gcm_decrypt(struct aead_request *req)
{
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
	return omap_aes_gcm_crypt(req, FLAGS_GCM);
}

int omap_aes_4106gcm_encrypt(struct aead_request *req)
{
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, ctx->octx.nonce, 4);
	memcpy(rctx->iv + 4, req->iv, 8);
	return crypto_ipsec_check_assoclen(req->assoclen) ?:
	       omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
				  FLAGS_RFC4106_GCM);
}

int omap_aes_4106gcm_decrypt(struct aead_request *req)
{
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);

	memcpy(rctx->iv, ctx->octx.nonce, 4);
	memcpy(rctx->iv + 4, req->iv, 8);
	return crypto_ipsec_check_assoclen(req->assoclen) ?:
	       omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
}

int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
			unsigned int keylen)
{
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
	int ret;

	ret = aes_expandkey(&ctx->actx, key, keylen);
	if (ret)
		return ret;

	memcpy(ctx->octx.key, key, keylen);
	ctx->octx.keylen = keylen;

	return 0;
}

int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
			    unsigned int keylen)
{
	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
	int ret;

	if (keylen < 4)
		return -EINVAL;
	keylen -= 4;

	ret = aes_expandkey(&ctx->actx, key, keylen);
	if (ret)
		return ret;

	memcpy(ctx->octx.key, key, keylen);
	memcpy(ctx->octx.nonce, key + keylen, 4);
	ctx->octx.keylen = keylen;

	return 0;
}

int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
	return crypto_gcm_check_authsize(authsize);
}

int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
				 unsigned int authsize)
{
	return crypto_rfc4106_check_authsize(authsize);
}

int omap_aes_gcm_crypt_req(struct crypto_engine *engine, void *areq)
{
	struct aead_request *req = container_of(areq, struct aead_request,
						base);
	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	struct omap_aes_dev *dd = rctx->dd;
	int ret;

	if (!dd)
		return -ENODEV;

	ret = omap_aes_gcm_prepare_req(req, dd);
	if (ret)
		return ret;

	if (dd->in_sg_len)
		ret = omap_aes_crypt_dma_start(dd);
	else
		omap_aes_gcm_dma_out_callback(dd);

	return ret;
}

int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
{
	crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));

	return 0;
}
Commit	Line	Data
d2912cb1	1	// SPDX-License-Identifier: GPL-2.0-only
ad18cc9d TK	2	/*
	3	* Cryptographic API.
	4	*
	5	* Support for OMAP AES GCM HW acceleration.
	6	*
	7	* Copyright (c) 2016 Texas Instruments Incorporated
ad18cc9d TK	8	*/
ad18cc9d TK	9
03906fba HX	10	#include <crypto/aes.h>
	11	#include <crypto/engine.h>
	12	#include <crypto/gcm.h>
	13	#include <crypto/internal/aead.h>
	14	#include <crypto/scatterwalk.h>
	15	#include <crypto/skcipher.h>
ad18cc9d	16	#include <linux/errno.h>
ad18cc9d TK	17	#include <linux/dma-mapping.h>
ad18cc9d TK	18	#include <linux/dmaengine.h>
ad18cc9d	19	#include <linux/interrupt.h>
03906fba HX	20	#include <linux/kernel.h>
03906fba HX	21	#include <linux/omap-dma.h>
04a4616e	22	#include <linux/pm_runtime.h>
03906fba HX	23	#include <linux/scatterlist.h>
03906fba HX	24	#include <linux/string.h>
ad18cc9d TK	25
	26	#include "omap-crypto.h"
	27	#include "omap-aes.h"
	28
	29	static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
	30	struct aead_request *req);
	31
	32	static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
	33	{
	34	struct aead_request *req = dd->aead_req;
	35
ad18cc9d TK	36	dd->in_sg = NULL;
	37	dd->out_sg = NULL;
	38
04a4616e TK	39	crypto_finalize_aead_request(dd->engine, req, ret);
	40
	41	pm_runtime_mark_last_busy(dd->dev);
	42	pm_runtime_put_autosuspend(dd->dev);
ad18cc9d TK	43	}
	44
	45	static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
	46	{
	47	u8 *tag;
	48	int alen, clen, i, ret = 0, nsg;
	49	struct omap_aes_reqctx *rctx;
	50
	51	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
	52	clen = ALIGN(dd->total, AES_BLOCK_SIZE);
	53	rctx = aead_request_ctx(dd->aead_req);
	54
	55	nsg = !!(dd->assoc_len && dd->total);
	56
	57	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
	58	DMA_FROM_DEVICE);
	59	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
	60	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
	61	omap_aes_crypt_dma_stop(dd);
	62
	63	omap_crypto_cleanup(dd->out_sg, dd->orig_out,
	64	dd->aead_req->assoclen, dd->total,
	65	FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
	66
	67	if (dd->flags & FLAGS_ENCRYPT)
	68	scatterwalk_map_and_copy(rctx->auth_tag,
	69	dd->aead_req->dst,
	70	dd->total + dd->aead_req->assoclen,
	71	dd->authsize, 1);
	72
	73	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
	74	FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
	75
	76	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
	77	FLAGS_IN_DATA_ST_SHIFT, dd->flags);
	78
	79	if (!(dd->flags & FLAGS_ENCRYPT)) {
	80	tag = (u8 *)rctx->auth_tag;
	81	for (i = 0; i < dd->authsize; i++) {
	82	if (tag[i]) {
ad18cc9d TK	83	ret = -EBADMSG;
	84	}
	85	}
	86	}
	87
	88	omap_aes_gcm_finish_req(dd, ret);
ad18cc9d TK	89	}
	90
	91	static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
	92	struct aead_request *req)
	93	{
	94	int alen, clen, cryptlen, assoclen, ret;
	95	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	96	unsigned int authlen = crypto_aead_authsize(aead);
	97	struct scatterlist *tmp, sg_arr[2];
	98	int nsg;
	99	u16 flags;
	100
	101	assoclen = req->assoclen;
	102	cryptlen = req->cryptlen;
	103
	104	if (dd->flags & FLAGS_RFC4106_GCM)
	105	assoclen -= 8;
	106
	107	if (!(dd->flags & FLAGS_ENCRYPT))
	108	cryptlen -= authlen;
	109
	110	alen = ALIGN(assoclen, AES_BLOCK_SIZE);
	111	clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
	112
	113	nsg = !!(assoclen && cryptlen);
	114
	115	omap_aes_clear_copy_flags(dd);
	116
	117	sg_init_table(dd->in_sgl, nsg + 1);
	118	if (assoclen) {
	119	tmp = req->src;
	120	ret = omap_crypto_align_sg(&tmp, assoclen,
	121	AES_BLOCK_SIZE, dd->in_sgl,
	122	OMAP_CRYPTO_COPY_DATA \|
	123	OMAP_CRYPTO_ZERO_BUF \|
	124	OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
	125	FLAGS_ASSOC_DATA_ST_SHIFT,
	126	&dd->flags);
46d57443 AB	127	if (ret)
46d57443 AB	128	return ret;
ad18cc9d TK	129	}
	130
	131	if (cryptlen) {
	132	tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
	133
04a4616e TK	134	if (nsg)
	135	sg_unmark_end(dd->in_sgl);
	136
ad18cc9d TK	137	ret = omap_crypto_align_sg(&tmp, cryptlen,
	138	AES_BLOCK_SIZE, &dd->in_sgl[nsg],
	139	OMAP_CRYPTO_COPY_DATA \|
	140	OMAP_CRYPTO_ZERO_BUF \|
	141	OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
	142	FLAGS_IN_DATA_ST_SHIFT,
	143	&dd->flags);
46d57443 AB	144	if (ret)
46d57443 AB	145	return ret;
ad18cc9d TK	146	}
	147
	148	dd->in_sg = dd->in_sgl;
	149	dd->total = cryptlen;
	150	dd->assoc_len = assoclen;
	151	dd->authsize = authlen;
	152
	153	dd->out_sg = req->dst;
	154	dd->orig_out = req->dst;
	155
04a4616e	156	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);
ad18cc9d TK	157
	158	flags = 0;
	159	if (req->src == req->dst \|\| dd->out_sg == sg_arr)
	160	flags \|= OMAP_CRYPTO_FORCE_COPY;
	161
a9befcf4 TK	162	if (cryptlen) {
	163	ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
	164	AES_BLOCK_SIZE, &dd->out_sgl,
	165	flags,
	166	FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
	167	if (ret)
	168	return ret;
	169	}
ad18cc9d TK	170
	171	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
	172	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
	173
	174	return 0;
	175	}
	176
ad18cc9d TK	177	static int do_encrypt_iv(struct aead_request req, u32 tag, u32 *iv)
ad18cc9d TK	178	{
f0956d42	179	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
ad18cc9d	180
f0956d42 AB	181	aes_encrypt(&ctx->actx, (u8 )tag, (u8 )iv);
f0956d42 AB	182	return 0;
ad18cc9d TK	183	}
	184
	185	void omap_aes_gcm_dma_out_callback(void *data)
	186	{
	187	struct omap_aes_dev *dd = data;
	188	struct omap_aes_reqctx *rctx;
	189	int i, val;
	190	u32 *auth_tag, tag[4];
	191
	192	if (!(dd->flags & FLAGS_ENCRYPT))
	193	scatterwalk_map_and_copy(tag, dd->aead_req->src,
	194	dd->total + dd->aead_req->assoclen,
	195	dd->authsize, 0);
	196
	197	rctx = aead_request_ctx(dd->aead_req);
	198	auth_tag = (u32 *)rctx->auth_tag;
	199	for (i = 0; i < 4; i++) {
	200	val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
	201	auth_tag[i] = val ^ auth_tag[i];
	202	if (!(dd->flags & FLAGS_ENCRYPT))
	203	auth_tag[i] = auth_tag[i] ^ tag[i];
	204	}
	205
	206	omap_aes_gcm_done_task(dd);
	207	}
	208
	209	static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
	210	struct aead_request *req)
	211	{
ad18cc9d	212	if (req)
04a4616e	213	return crypto_transfer_aead_request_to_engine(dd->engine, req);
ad18cc9d	214
04a4616e TK	215	return 0;
04a4616e TK	216	}
ad18cc9d	217
c752c013 HX	218	static int omap_aes_gcm_prepare_req(struct aead_request *req,
c752c013 HX	219	struct omap_aes_dev *dd)
04a4616e	220	{
04a4616e	221	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
04a4616e TK	222	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
04a4616e TK	223	int err;
ad18cc9d	224
ad18cc9d TK	225	dd->aead_req = req;
	226
	227	rctx->mode &= FLAGS_MODE_MASK;
	228	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) \| rctx->mode;
	229
	230	err = omap_aes_gcm_copy_buffers(dd, req);
	231	if (err)
	232	return err;
	233
04a4616e	234	dd->ctx = &ctx->octx;
ad18cc9d	235
04a4616e	236	return omap_aes_write_ctrl(dd);
ad18cc9d TK	237	}
	238
	239	static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
	240	{
	241	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	242	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	243	unsigned int authlen = crypto_aead_authsize(aead);
	244	struct omap_aes_dev *dd;
	245	__be32 counter = cpu_to_be32(1);
	246	int err, assoclen;
	247
	248	memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
cb3f3817	249	memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
ad18cc9d TK	250
	251	err = do_encrypt_iv(req, (u32 )rctx->auth_tag, (u32 )rctx->iv);
	252	if (err)
	253	return err;
	254
	255	if (mode & FLAGS_RFC4106_GCM)
	256	assoclen = req->assoclen - 8;
	257	else
	258	assoclen = req->assoclen;
	259	if (assoclen + req->cryptlen == 0) {
	260	scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
	261	1);
	262	return 0;
	263	}
	264
	265	dd = omap_aes_find_dev(rctx);
	266	if (!dd)
	267	return -ENODEV;
	268	rctx->mode = mode;
	269
	270	return omap_aes_gcm_handle_queue(dd, req);
	271	}
	272
	273	int omap_aes_gcm_encrypt(struct aead_request *req)
	274	{
	275	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	276
cb3f3817	277	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
ad18cc9d TK	278	return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT \| FLAGS_GCM);
	279	}
	280
	281	int omap_aes_gcm_decrypt(struct aead_request *req)
	282	{
	283	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	284
cb3f3817	285	memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
ad18cc9d TK	286	return omap_aes_gcm_crypt(req, FLAGS_GCM);
	287	}
	288
	289	int omap_aes_4106gcm_encrypt(struct aead_request *req)
	290	{
f0956d42	291	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
ad18cc9d TK	292	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
ad18cc9d TK	293
f0956d42	294	memcpy(rctx->iv, ctx->octx.nonce, 4);
ad18cc9d	295	memcpy(rctx->iv + 4, req->iv, 8);
b877ad1a AB	296	return crypto_ipsec_check_assoclen(req->assoclen) ?:
b877ad1a AB	297	omap_aes_gcm_crypt(req, FLAGS_ENCRYPT \| FLAGS_GCM \|
ad18cc9d TK	298	FLAGS_RFC4106_GCM);
	299	}
	300
	301	int omap_aes_4106gcm_decrypt(struct aead_request *req)
	302	{
f0956d42	303	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
ad18cc9d TK	304	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
ad18cc9d TK	305
f0956d42	306	memcpy(rctx->iv, ctx->octx.nonce, 4);
ad18cc9d	307	memcpy(rctx->iv + 4, req->iv, 8);
b877ad1a AB	308	return crypto_ipsec_check_assoclen(req->assoclen) ?:
b877ad1a AB	309	omap_aes_gcm_crypt(req, FLAGS_GCM \| FLAGS_RFC4106_GCM);
ad18cc9d TK	310	}
	311
	312	int omap_aes_gcm_setkey(struct crypto_aead tfm, const u8 key,
	313	unsigned int keylen)
	314	{
f0956d42 AB	315	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
f0956d42 AB	316	int ret;
ad18cc9d	317
f0956d42 AB	318	ret = aes_expandkey(&ctx->actx, key, keylen);
	319	if (ret)
	320	return ret;
ad18cc9d	321
f0956d42 AB	322	memcpy(ctx->octx.key, key, keylen);
f0956d42 AB	323	ctx->octx.keylen = keylen;
ad18cc9d TK	324
	325	return 0;
	326	}
	327
	328	int omap_aes_4106gcm_setkey(struct crypto_aead tfm, const u8 key,
	329	unsigned int keylen)
	330	{
f0956d42 AB	331	struct omap_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
f0956d42 AB	332	int ret;
ad18cc9d TK	333
	334	if (keylen < 4)
	335	return -EINVAL;
ad18cc9d	336	keylen -= 4;
ad18cc9d	337
f0956d42 AB	338	ret = aes_expandkey(&ctx->actx, key, keylen);
	339	if (ret)
	340	return ret;
	341
	342	memcpy(ctx->octx.key, key, keylen);
	343	memcpy(ctx->octx.nonce, key + keylen, 4);
	344	ctx->octx.keylen = keylen;
ad18cc9d TK	345
	346	return 0;
	347	}
12adf9d6 AB	348
	349	int omap_aes_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
	350	{
	351	return crypto_gcm_check_authsize(authsize);
	352	}
	353
	354	int omap_aes_4106gcm_setauthsize(struct crypto_aead *parent,
	355	unsigned int authsize)
	356	{
	357	return crypto_rfc4106_check_authsize(authsize);
	358	}
04a4616e	359
03906fba	360	int omap_aes_gcm_crypt_req(struct crypto_engine engine, void areq)
04a4616e TK	361	{
	362	struct aead_request *req = container_of(areq, struct aead_request,
	363	base);
	364	struct omap_aes_reqctx *rctx = aead_request_ctx(req);
	365	struct omap_aes_dev *dd = rctx->dd;
c752c013	366	int ret;
04a4616e TK	367
	368	if (!dd)
	369	return -ENODEV;
	370
c752c013 HX	371	ret = omap_aes_gcm_prepare_req(req, dd);
	372	if (ret)
	373	return ret;
	374
04a4616e TK	375	if (dd->in_sg_len)
	376	ret = omap_aes_crypt_dma_start(dd);
	377	else
	378	omap_aes_gcm_dma_out_callback(dd);
	379
	380	return ret;
	381	}
	382
	383	int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
	384	{
04a4616e TK	385	crypto_aead_set_reqsize(tfm, sizeof(struct omap_aes_reqctx));
	386
	387	return 0;
	388	}