[linux-2.6-block.git] / crypto / gcm.c

/*
 * GCM: Galois/Counter Mode.
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <crypto/algapi.h>
#include <crypto/gf128mul.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>

#include "internal.h"

struct gcm_instance_ctx {
	struct crypto_spawn ctr;
};

struct crypto_gcm_ctx {
	struct crypto_ablkcipher *ctr;
	struct gf128mul_4k *gf128;
};

struct crypto_gcm_ghash_ctx {
	u32 bytes;
	u32 flags;
	struct gf128mul_4k *gf128;
	u8 buffer[16];
};

struct crypto_gcm_req_priv_ctx {
	u8 auth_tag[16];
	u8 iauth_tag[16];
	u8 counter[16];
	struct crypto_gcm_ghash_ctx ghash;
	struct ablkcipher_request abreq;
};

static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	struct aead_request *req)
{
	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));

	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
}

static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
				  struct gf128mul_4k *gf128)
{
	ctx->bytes = 0;
	ctx->flags = flags;
	ctx->gf128 = gf128;
	memset(ctx->buffer, 0, 16);
}

static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
				    const u8 *src, unsigned int srclen)
{
	u8 *dst = ctx->buffer;

	if (ctx->bytes) {
		int n = min(srclen, ctx->bytes);
		u8 *pos = dst + (16 - ctx->bytes);

		ctx->bytes -= n;
		srclen -= n;

		while (n--)
			*pos++ ^= *src++;

		if (!ctx->bytes)
			gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	}

	while (srclen >= 16) {
		crypto_xor(dst, src, 16);
		gf128mul_4k_lle((be128 *)dst, ctx->gf128);
		src += 16;
		srclen -= 16;
	}

	if (srclen) {
		ctx->bytes = 16 - srclen;
		while (srclen--)
			*dst++ ^= *src++;
	}
}

static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
				       struct scatterlist *sg, int len)
{
	struct scatter_walk walk;
	u8 *src;
	int n;

	if (!len)
		return;

	scatterwalk_start(&walk, sg);

	while (len) {
		n = scatterwalk_clamp(&walk, len);

		if (!n) {
			scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
			n = scatterwalk_clamp(&walk, len);
		}

		src = scatterwalk_map(&walk, 0);

		crypto_gcm_ghash_update(ctx, src, n);
		len -= n;

		scatterwalk_unmap(src, 0);
		scatterwalk_advance(&walk, n);
		scatterwalk_done(&walk, 0, len);
		if (len)
			crypto_yield(ctx->flags);
	}
}

static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
{
	u8 *dst = ctx->buffer;

	if (ctx->bytes) {
		u8 *tmp = dst + (16 - ctx->bytes);

		while (ctx->bytes--)
			*tmp++ ^= 0;

		gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	}

	ctx->bytes = 0;
}

static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
				       unsigned int authlen,
				       unsigned int cryptlen, u8 *dst)
{
	u8 *buf = ctx->buffer;
	u128 lengths;

	lengths.a = cpu_to_be64(authlen * 8);
	lengths.b = cpu_to_be64(cryptlen * 8);

	crypto_gcm_ghash_flush(ctx);
	crypto_xor(buf, (u8 *)&lengths, 16);
	gf128mul_4k_lle((be128 *)buf, ctx->gf128);
	crypto_xor(dst, buf, 16);
}

static inline void crypto_gcm_set_counter(u8 *counterblock, u32 value)
{
	*((u32 *)&counterblock[12]) = cpu_to_be32(value + 1);
}

static int crypto_gcm_encrypt_counter(struct crypto_aead *aead, u8 *block,
				       u32 value, const u8 *iv)
{
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_ablkcipher *ctr = ctx->ctr;
	struct ablkcipher_request req;
	struct scatterlist sg;
	u8 counterblock[16];

	if (iv == NULL)
		memset(counterblock, 0, 12);
	else
		memcpy(counterblock, iv, 12);

	crypto_gcm_set_counter(counterblock, value);

	sg_init_one(&sg, block, 16);
	ablkcipher_request_set_tfm(&req, ctr);
	ablkcipher_request_set_crypt(&req, &sg, &sg, 16, counterblock);
	ablkcipher_request_set_callback(&req, 0, NULL, NULL);
	memset(block, 0, 16);
	return crypto_ablkcipher_encrypt(&req);
}

static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
			     unsigned int keylen)
{
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_ablkcipher *ctr = ctx->ctr;
	int alignmask = crypto_ablkcipher_alignmask(ctr);
	u8 alignbuf[16+alignmask];
	u8 *hash = (u8 *)ALIGN((unsigned long)alignbuf, alignmask+1);
	int err = 0;

	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
				   CRYPTO_TFM_REQ_MASK);

	err = crypto_ablkcipher_setkey(ctr, key, keylen);
	if (err)
		goto out;

	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
				       CRYPTO_TFM_RES_MASK);

	err = crypto_gcm_encrypt_counter(aead, hash, -1, NULL);
	if (err)
		goto out;

	if (ctx->gf128 != NULL)
		gf128mul_free_4k(ctx->gf128);

	ctx->gf128 = gf128mul_init_4k_lle((be128 *)hash);

	if (ctx->gf128 == NULL)
		err = -ENOMEM;

 out:
	return err;
}

static int crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
				 struct aead_request *req,
				 unsigned int cryptlen,
				 void (*done)(struct crypto_async_request *,
					      int))
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	u32 flags = req->base.tfm->crt_flags;
	u8 *auth_tag = pctx->auth_tag;
	u8 *counter = pctx->counter;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	int err = 0;

	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
	ablkcipher_request_set_callback(ablk_req, aead_request_flags(req),
					done, req);
	ablkcipher_request_set_crypt(ablk_req, req->src, req->dst,
				     cryptlen, counter);

	err = crypto_gcm_encrypt_counter(aead, auth_tag, 0, req->iv);
	if (err)
		goto out;

	memcpy(counter, req->iv, 12);
	crypto_gcm_set_counter(counter, 1);

	crypto_gcm_ghash_init(ghash, flags, ctx->gf128);

	crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
	crypto_gcm_ghash_flush(ghash);

 out:
	return err;
}

static int crypto_gcm_hash(struct aead_request *req)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	u8 *auth_tag = pctx->auth_tag;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;

	crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
	crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
				   auth_tag);

	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
				 crypto_aead_authsize(aead), 1);
	return 0;
}

static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;

	if (!err)
		err = crypto_gcm_hash(req);

	aead_request_complete(req, err);
}

static int crypto_gcm_encrypt(struct aead_request *req)
{
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct ablkcipher_request *abreq = &pctx->abreq;
	int err = 0;

	err = crypto_gcm_init_crypt(abreq, req, req->cryptlen,
				    crypto_gcm_encrypt_done);
	if (err)
		return err;

	if (req->cryptlen) {
		err = crypto_ablkcipher_encrypt(abreq);
		if (err)
			return err;
	}

	return crypto_gcm_hash(req);
}

static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
{
	aead_request_complete(areq->data, err);
}

static int crypto_gcm_decrypt(struct aead_request *req)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct ablkcipher_request *abreq = &pctx->abreq;
	u8 *auth_tag = pctx->auth_tag;
	u8 *iauth_tag = pctx->iauth_tag;
	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	unsigned int cryptlen = req->cryptlen;
	unsigned int authsize = crypto_aead_authsize(aead);
	int err;

	if (cryptlen < authsize)
		return -EINVAL;
	cryptlen -= authsize;

	err = crypto_gcm_init_crypt(abreq, req, cryptlen,
				    crypto_gcm_decrypt_done);
	if (err)
		return err;

	crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
	crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);

	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
	if (memcmp(iauth_tag, auth_tag, authsize))
		return -EBADMSG;

	return crypto_ablkcipher_decrypt(abreq);
}

static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
{
	struct crypto_instance *inst = (void *)tfm->__crt_alg;
	struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_ablkcipher *ctr;
	unsigned long align;
	int err;

	ctr = crypto_spawn_ablkcipher(&ictx->ctr);
	err = PTR_ERR(ctr);
	if (IS_ERR(ctr))
		return err;

	ctx->ctr = ctr;
	ctx->gf128 = NULL;

	align = crypto_tfm_alg_alignmask(tfm);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
	tfm->crt_aead.reqsize = align +
				sizeof(struct crypto_gcm_req_priv_ctx) +
				crypto_ablkcipher_reqsize(ctr);

	return 0;
}

static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
{
	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);

	if (ctx->gf128 != NULL)
		gf128mul_free_4k(ctx->gf128);

	crypto_free_ablkcipher(ctx->ctr);
}

static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
{
	struct crypto_instance *inst;
	struct crypto_alg *ctr;
	struct crypto_alg *cipher;
	struct gcm_instance_ctx *ctx;
	int err;
	char ctr_name[CRYPTO_MAX_ALG_NAME];

	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD);
	if (err)
		return ERR_PTR(err);

	cipher = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER,
			      CRYPTO_ALG_TYPE_MASK);

	inst = ERR_PTR(PTR_ERR(cipher));
	if (IS_ERR(cipher))
		return inst;

	inst = ERR_PTR(ENAMETOOLONG);
	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
		     cipher->cra_name) >= CRYPTO_MAX_ALG_NAME)
		return inst;

	ctr = crypto_alg_mod_lookup(ctr_name, CRYPTO_ALG_TYPE_BLKCIPHER,
				    CRYPTO_ALG_TYPE_MASK);

	if (IS_ERR(ctr))
		return ERR_PTR(PTR_ERR(ctr));

	if (cipher->cra_blocksize != 16)
		goto out_put_ctr;

	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
	err = -ENOMEM;
	if (!inst)
		goto out_put_ctr;

	err = -ENAMETOOLONG;
	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
		     "gcm(%s)", cipher->cra_name) >= CRYPTO_MAX_ALG_NAME ||
	    snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "gcm(%s)", cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
		goto err_free_inst;


	ctx = crypto_instance_ctx(inst);
	err = crypto_init_spawn(&ctx->ctr, ctr, inst, CRYPTO_ALG_TYPE_MASK);
	if (err)
		goto err_free_inst;

	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
	inst->alg.cra_priority = ctr->cra_priority;
	inst->alg.cra_blocksize = 16;
	inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
	inst->alg.cra_type = &crypto_aead_type;
	inst->alg.cra_aead.ivsize = 12;
	inst->alg.cra_aead.maxauthsize = 16;
	inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
	inst->alg.cra_init = crypto_gcm_init_tfm;
	inst->alg.cra_exit = crypto_gcm_exit_tfm;
	inst->alg.cra_aead.setkey = crypto_gcm_setkey;
	inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
	inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;

out:
	crypto_mod_put(ctr);
	return inst;
err_free_inst:
	kfree(inst);
out_put_ctr:
	inst = ERR_PTR(err);
	goto out;
}

static void crypto_gcm_free(struct crypto_instance *inst)
{
	struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);

	crypto_drop_spawn(&ctx->ctr);
	kfree(inst);
}

static struct crypto_template crypto_gcm_tmpl = {
	.name = "gcm",
	.alloc = crypto_gcm_alloc,
	.free = crypto_gcm_free,
	.module = THIS_MODULE,
};

static int __init crypto_gcm_module_init(void)
{
	return crypto_register_template(&crypto_gcm_tmpl);
}

static void __exit crypto_gcm_module_exit(void)
{
	crypto_unregister_template(&crypto_gcm_tmpl);
}

module_init(crypto_gcm_module_init);
module_exit(crypto_gcm_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Galois/Counter Mode");
MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
Commit	Line	Data
28db8e3e MH	1	/*
	2	* GCM: Galois/Counter Mode.
	3	*
	4	* Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
	5	*
	6	* This program is free software; you can redistribute it and/or modify it
	7	* under the terms of the GNU General Public License version 2 as published
	8	* by the Free Software Foundation.
	9	*/
	10
	11	#include <crypto/algapi.h>
	12	#include <crypto/gf128mul.h>
42c271c6	13	#include <crypto/scatterwalk.h>
28db8e3e MH	14	#include <linux/err.h>
	15	#include <linux/init.h>
	16	#include <linux/kernel.h>
	17	#include <linux/module.h>
	18	#include <linux/slab.h>
	19
42c271c6	20	#include "internal.h"
28db8e3e MH	21
	22	struct gcm_instance_ctx {
	23	struct crypto_spawn ctr;
	24	};
	25
	26	struct crypto_gcm_ctx {
	27	struct crypto_ablkcipher *ctr;
	28	struct gf128mul_4k *gf128;
	29	};
	30
	31	struct crypto_gcm_ghash_ctx {
	32	u32 bytes;
	33	u32 flags;
	34	struct gf128mul_4k *gf128;
	35	u8 buffer[16];
	36	};
	37
	38	struct crypto_gcm_req_priv_ctx {
	39	u8 auth_tag[16];
6160b289	40	u8 iauth_tag[16];
28db8e3e MH	41	u8 counter[16];
28db8e3e MH	42	struct crypto_gcm_ghash_ctx ghash;
7f681378	43	struct ablkcipher_request abreq;
28db8e3e MH	44	};
28db8e3e MH	45
2589469d HX	46	static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	47	struct aead_request *req)
	48	{
	49	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
	50
	51	return (void )PTR_ALIGN((u8 )aead_request_ctx(req), align + 1);
	52	}
	53
28db8e3e MH	54	static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
	55	struct gf128mul_4k *gf128)
	56	{
	57	ctx->bytes = 0;
	58	ctx->flags = flags;
	59	ctx->gf128 = gf128;
	60	memset(ctx->buffer, 0, 16);
	61	}
	62
	63	static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
	64	const u8 *src, unsigned int srclen)
	65	{
	66	u8 *dst = ctx->buffer;
	67
	68	if (ctx->bytes) {
	69	int n = min(srclen, ctx->bytes);
	70	u8 *pos = dst + (16 - ctx->bytes);
	71
	72	ctx->bytes -= n;
	73	srclen -= n;
	74
	75	while (n--)
	76	pos++ ^= src++;
	77
	78	if (!ctx->bytes)
	79	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	80	}
	81
	82	while (srclen >= 16) {
	83	crypto_xor(dst, src, 16);
	84	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	85	src += 16;
	86	srclen -= 16;
	87	}
	88
	89	if (srclen) {
	90	ctx->bytes = 16 - srclen;
	91	while (srclen--)
	92	dst++ ^= src++;
	93	}
	94	}
	95
	96	static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
	97	struct scatterlist *sg, int len)
	98	{
	99	struct scatter_walk walk;
	100	u8 *src;
	101	int n;
	102
6160b289 HX	103	if (!len)
	104	return;
	105
28db8e3e MH	106	scatterwalk_start(&walk, sg);
	107
	108	while (len) {
	109	n = scatterwalk_clamp(&walk, len);
	110
	111	if (!n) {
b2ab4a57	112	scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
28db8e3e MH	113	n = scatterwalk_clamp(&walk, len);
	114	}
	115
	116	src = scatterwalk_map(&walk, 0);
	117
	118	crypto_gcm_ghash_update(ctx, src, n);
	119	len -= n;
	120
	121	scatterwalk_unmap(src, 0);
	122	scatterwalk_advance(&walk, n);
	123	scatterwalk_done(&walk, 0, len);
	124	if (len)
	125	crypto_yield(ctx->flags);
	126	}
	127	}
	128
	129	static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
	130	{
	131	u8 *dst = ctx->buffer;
	132
	133	if (ctx->bytes) {
	134	u8 *tmp = dst + (16 - ctx->bytes);
	135
	136	while (ctx->bytes--)
	137	*tmp++ ^= 0;
	138
	139	gf128mul_4k_lle((be128 *)dst, ctx->gf128);
	140	}
	141
	142	ctx->bytes = 0;
	143	}
	144
	145	static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
	146	unsigned int authlen,
	147	unsigned int cryptlen, u8 *dst)
	148	{
	149	u8 *buf = ctx->buffer;
	150	u128 lengths;
	151
	152	lengths.a = cpu_to_be64(authlen * 8);
	153	lengths.b = cpu_to_be64(cryptlen * 8);
	154
	155	crypto_gcm_ghash_flush(ctx);
	156	crypto_xor(buf, (u8 *)&lengths, 16);
	157	gf128mul_4k_lle((be128 *)buf, ctx->gf128);
	158	crypto_xor(dst, buf, 16);
	159	}
	160
	161	static inline void crypto_gcm_set_counter(u8 *counterblock, u32 value)
	162	{
5311f248	163	((u32 )&counterblock[12]) = cpu_to_be32(value + 1);
28db8e3e MH	164	}
	165
	166	static int crypto_gcm_encrypt_counter(struct crypto_aead aead, u8 block,
	167	u32 value, const u8 *iv)
	168	{
	169	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	170	struct crypto_ablkcipher *ctr = ctx->ctr;
	171	struct ablkcipher_request req;
	172	struct scatterlist sg;
	173	u8 counterblock[16];
	174
	175	if (iv == NULL)
	176	memset(counterblock, 0, 12);
	177	else
	178	memcpy(counterblock, iv, 12);
	179
	180	crypto_gcm_set_counter(counterblock, value);
	181
	182	sg_init_one(&sg, block, 16);
	183	ablkcipher_request_set_tfm(&req, ctr);
	184	ablkcipher_request_set_crypt(&req, &sg, &sg, 16, counterblock);
	185	ablkcipher_request_set_callback(&req, 0, NULL, NULL);
	186	memset(block, 0, 16);
	187	return crypto_ablkcipher_encrypt(&req);
	188	}
	189
	190	static int crypto_gcm_setkey(struct crypto_aead aead, const u8 key,
	191	unsigned int keylen)
	192	{
	193	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	194	struct crypto_ablkcipher *ctr = ctx->ctr;
	195	int alignmask = crypto_ablkcipher_alignmask(ctr);
	196	u8 alignbuf[16+alignmask];
	197	u8 hash = (u8 )ALIGN((unsigned long)alignbuf, alignmask+1);
	198	int err = 0;
	199
	200	crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	201	crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
	202	CRYPTO_TFM_REQ_MASK);
	203
	204	err = crypto_ablkcipher_setkey(ctr, key, keylen);
	205	if (err)
	206	goto out;
	207
	208	crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
	209	CRYPTO_TFM_RES_MASK);
	210
	211	err = crypto_gcm_encrypt_counter(aead, hash, -1, NULL);
	212	if (err)
	213	goto out;
	214
	215	if (ctx->gf128 != NULL)
	216	gf128mul_free_4k(ctx->gf128);
	217
	218	ctx->gf128 = gf128mul_init_4k_lle((be128 *)hash);
	219
	220	if (ctx->gf128 == NULL)
	221	err = -ENOMEM;
	222
	223	out:
	224	return err;
	225	}
	226
	227	static int crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
6160b289 HX	228	struct aead_request *req,
	229	unsigned int cryptlen,
	230	void (done)(struct crypto_async_request ,
	231	int))
28db8e3e MH	232	{
	233	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	234	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
2589469d	235	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
28db8e3e MH	236	u32 flags = req->base.tfm->crt_flags;
	237	u8 *auth_tag = pctx->auth_tag;
	238	u8 *counter = pctx->counter;
	239	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	240	int err = 0;
	241
	242	ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
	243	ablkcipher_request_set_callback(ablk_req, aead_request_flags(req),
	244	done, req);
	245	ablkcipher_request_set_crypt(ablk_req, req->src, req->dst,
6160b289	246	cryptlen, counter);
28db8e3e MH	247
	248	err = crypto_gcm_encrypt_counter(aead, auth_tag, 0, req->iv);
	249	if (err)
	250	goto out;
	251
	252	memcpy(counter, req->iv, 12);
	253	crypto_gcm_set_counter(counter, 1);
	254
	255	crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
	256
6160b289 HX	257	crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
6160b289 HX	258	crypto_gcm_ghash_flush(ghash);
28db8e3e MH	259
	260	out:
	261	return err;
	262	}
	263
6160b289	264	static int crypto_gcm_hash(struct aead_request *req)
28db8e3e	265	{
6160b289	266	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2589469d	267	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
28db8e3e MH	268	u8 *auth_tag = pctx->auth_tag;
	269	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
	270
	271	crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
	272	crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
	273	auth_tag);
	274
6160b289 HX	275	scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
	276	crypto_aead_authsize(aead), 1);
	277	return 0;
	278	}
	279
	280	static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
	281	{
	282	struct aead_request *req = areq->data;
	283
	284	if (!err)
	285	err = crypto_gcm_hash(req);
	286
28db8e3e MH	287	aead_request_complete(req, err);
	288	}
	289
	290	static int crypto_gcm_encrypt(struct aead_request *req)
	291	{
2589469d	292	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
7f681378	293	struct ablkcipher_request *abreq = &pctx->abreq;
28db8e3e MH	294	int err = 0;
28db8e3e MH	295
7f681378	296	err = crypto_gcm_init_crypt(abreq, req, req->cryptlen,
6160b289	297	crypto_gcm_encrypt_done);
28db8e3e MH	298	if (err)
	299	return err;
	300
	301	if (req->cryptlen) {
7f681378	302	err = crypto_ablkcipher_encrypt(abreq);
28db8e3e MH	303	if (err)
28db8e3e MH	304	return err;
28db8e3e MH	305	}
28db8e3e MH	306
6160b289	307	return crypto_gcm_hash(req);
28db8e3e MH	308	}
	309
	310	static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
	311	{
	312	aead_request_complete(areq->data, err);
	313	}
	314
	315	static int crypto_gcm_decrypt(struct aead_request *req)
	316	{
6160b289	317	struct crypto_aead *aead = crypto_aead_reqtfm(req);
2589469d	318	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
7f681378	319	struct ablkcipher_request *abreq = &pctx->abreq;
28db8e3e	320	u8 *auth_tag = pctx->auth_tag;
6160b289	321	u8 *iauth_tag = pctx->iauth_tag;
28db8e3e	322	struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
6160b289 HX	323	unsigned int cryptlen = req->cryptlen;
6160b289 HX	324	unsigned int authsize = crypto_aead_authsize(aead);
28db8e3e MH	325	int err;
28db8e3e MH	326
6160b289	327	if (cryptlen < authsize)
28db8e3e	328	return -EINVAL;
6160b289	329	cryptlen -= authsize;
28db8e3e	330
7f681378	331	err = crypto_gcm_init_crypt(abreq, req, cryptlen,
6160b289	332	crypto_gcm_decrypt_done);
28db8e3e MH	333	if (err)
	334	return err;
	335
6160b289 HX	336	crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
6160b289 HX	337	crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
28db8e3e	338
6160b289 HX	339	scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
6160b289 HX	340	if (memcmp(iauth_tag, auth_tag, authsize))
fe70f5df	341	return -EBADMSG;
28db8e3e	342
7f681378	343	return crypto_ablkcipher_decrypt(abreq);
28db8e3e MH	344	}
	345
	346	static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
	347	{
	348	struct crypto_instance inst = (void )tfm->__crt_alg;
	349	struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
	350	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	351	struct crypto_ablkcipher *ctr;
	352	unsigned long align;
	353	int err;
	354
	355	ctr = crypto_spawn_ablkcipher(&ictx->ctr);
	356	err = PTR_ERR(ctr);
	357	if (IS_ERR(ctr))
	358	return err;
	359
	360	ctx->ctr = ctr;
	361	ctx->gf128 = NULL;
	362
2589469d	363	align = crypto_tfm_alg_alignmask(tfm);
28db8e3e	364	align &= ~(crypto_tfm_ctx_alignment() - 1);
7f681378 HX	365	tfm->crt_aead.reqsize = align +
	366	sizeof(struct crypto_gcm_req_priv_ctx) +
	367	crypto_ablkcipher_reqsize(ctr);
28db8e3e MH	368
	369	return 0;
	370	}
	371
	372	static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
	373	{
	374	struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
	375
	376	if (ctx->gf128 != NULL)
	377	gf128mul_free_4k(ctx->gf128);
	378
	379	crypto_free_ablkcipher(ctx->ctr);
	380	}
	381
	382	static struct crypto_instance crypto_gcm_alloc(struct rtattr *tb)
	383	{
	384	struct crypto_instance *inst;
	385	struct crypto_alg *ctr;
	386	struct crypto_alg *cipher;
	387	struct gcm_instance_ctx *ctx;
	388	int err;
	389	char ctr_name[CRYPTO_MAX_ALG_NAME];
	390
	391	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD);
	392	if (err)
	393	return ERR_PTR(err);
	394
	395	cipher = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER,
	396	CRYPTO_ALG_TYPE_MASK);
	397
	398	inst = ERR_PTR(PTR_ERR(cipher));
	399	if (IS_ERR(cipher))
	400	return inst;
	401
	402	inst = ERR_PTR(ENAMETOOLONG);
5311f248 HX	403	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
5311f248 HX	404	cipher->cra_name) >= CRYPTO_MAX_ALG_NAME)
28db8e3e MH	405	return inst;
	406
	407	ctr = crypto_alg_mod_lookup(ctr_name, CRYPTO_ALG_TYPE_BLKCIPHER,
	408	CRYPTO_ALG_TYPE_MASK);
	409
	410	if (IS_ERR(ctr))
	411	return ERR_PTR(PTR_ERR(ctr));
	412
	413	if (cipher->cra_blocksize != 16)
	414	goto out_put_ctr;
	415
	416	inst = kzalloc(sizeof(inst) + sizeof(ctx), GFP_KERNEL);
	417	err = -ENOMEM;
	418	if (!inst)
	419	goto out_put_ctr;
	420
	421	err = -ENAMETOOLONG;
	422	if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
	423	"gcm(%s)", cipher->cra_name) >= CRYPTO_MAX_ALG_NAME \|\|
	424	snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
	425	"gcm(%s)", cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
	426	goto err_free_inst;
	427
	428
	429	ctx = crypto_instance_ctx(inst);
	430	err = crypto_init_spawn(&ctx->ctr, ctr, inst, CRYPTO_ALG_TYPE_MASK);
	431	if (err)
	432	goto err_free_inst;
	433
	434	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD \| CRYPTO_ALG_ASYNC;
	435	inst->alg.cra_priority = ctr->cra_priority;
	436	inst->alg.cra_blocksize = 16;
2589469d	437	inst->alg.cra_alignmask = ctr->cra_alignmask \| (__alignof__(u64) - 1);
28db8e3e MH	438	inst->alg.cra_type = &crypto_aead_type;
28db8e3e MH	439	inst->alg.cra_aead.ivsize = 12;
7ba683a6	440	inst->alg.cra_aead.maxauthsize = 16;
28db8e3e MH	441	inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
	442	inst->alg.cra_init = crypto_gcm_init_tfm;
	443	inst->alg.cra_exit = crypto_gcm_exit_tfm;
	444	inst->alg.cra_aead.setkey = crypto_gcm_setkey;
	445	inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
	446	inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
	447
	448	out:
	449	crypto_mod_put(ctr);
	450	return inst;
	451	err_free_inst:
	452	kfree(inst);
	453	out_put_ctr:
	454	inst = ERR_PTR(err);
	455	goto out;
	456	}
	457
	458	static void crypto_gcm_free(struct crypto_instance *inst)
	459	{
	460	struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
	461
	462	crypto_drop_spawn(&ctx->ctr);
	463	kfree(inst);
	464	}
	465
	466	static struct crypto_template crypto_gcm_tmpl = {
	467	.name = "gcm",
	468	.alloc = crypto_gcm_alloc,
	469	.free = crypto_gcm_free,
	470	.module = THIS_MODULE,
	471	};
	472
	473	static int __init crypto_gcm_module_init(void)
	474	{
	475	return crypto_register_template(&crypto_gcm_tmpl);
	476	}
	477
	478	static void __exit crypto_gcm_module_exit(void)
	479	{
	480	crypto_unregister_template(&crypto_gcm_tmpl);
	481	}
	482
	483	module_init(crypto_gcm_module_init);
	484	module_exit(crypto_gcm_module_exit);
	485
	486	MODULE_LICENSE("GPL");
	487	MODULE_DESCRIPTION("Galois/Counter Mode");
	488	MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");