[linux-2.6-block.git] / crypto / vmac.c

/*
 * Modified to interface to the Linux kernel
 * Copyright (c) 2009, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 * Place - Suite 330, Boston, MA 02111-1307 USA.
 */

/* --------------------------------------------------------------------------
 * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
 * This implementation is herby placed in the public domain.
 * The authors offers no warranty. Use at your own risk.
 * Please send bug reports to the authors.
 * Last modified: 17 APR 08, 1700 PDT
 * ----------------------------------------------------------------------- */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <asm/byteorder.h>
#include <crypto/scatterwalk.h>
#include <crypto/vmac.h>
#include <crypto/internal/hash.h>

/*
 * Constants and masks
 */
#define UINT64_C(x) x##ULL
const u64 p64   = UINT64_C(0xfffffffffffffeff);  /* 2^64 - 257 prime  */
const u64 m62   = UINT64_C(0x3fffffffffffffff);  /* 62-bit mask       */
const u64 m63   = UINT64_C(0x7fffffffffffffff);  /* 63-bit mask       */
const u64 m64   = UINT64_C(0xffffffffffffffff);  /* 64-bit mask       */
const u64 mpoly = UINT64_C(0x1fffffff1fffffff);  /* Poly key mask     */

#define pe64_to_cpup le64_to_cpup		/* Prefer little endian */

#ifdef __LITTLE_ENDIAN
#define INDEX_HIGH 1
#define INDEX_LOW 0
#else
#define INDEX_HIGH 0
#define INDEX_LOW 1
#endif

/*
 * The following routines are used in this implementation. They are
 * written via macros to simulate zero-overhead call-by-reference.
 *
 * MUL64: 64x64->128-bit multiplication
 * PMUL64: assumes top bits cleared on inputs
 * ADD128: 128x128->128-bit addition
 */

#define ADD128(rh, rl, ih, il)						\
	do {								\
		u64 _il = (il);						\
		(rl) += (_il);						\
		if ((rl) < (_il))					\
			(rh)++;						\
		(rh) += (ih);						\
	} while (0)

#define MUL32(i1, i2)	((u64)(u32)(i1)*(u32)(i2))

#define PMUL64(rh, rl, i1, i2)	/* Assumes m doesn't overflow */	\
	do {								\
		u64 _i1 = (i1), _i2 = (i2);				\
		u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2);	\
		rh = MUL32(_i1>>32, _i2>>32);				\
		rl = MUL32(_i1, _i2);					\
		ADD128(rh, rl, (m >> 32), (m << 32));			\
	} while (0)

#define MUL64(rh, rl, i1, i2)						\
	do {								\
		u64 _i1 = (i1), _i2 = (i2);				\
		u64 m1 = MUL32(_i1, _i2>>32);				\
		u64 m2 = MUL32(_i1>>32, _i2);				\
		rh = MUL32(_i1>>32, _i2>>32);				\
		rl = MUL32(_i1, _i2);					\
		ADD128(rh, rl, (m1 >> 32), (m1 << 32));			\
		ADD128(rh, rl, (m2 >> 32), (m2 << 32));			\
	} while (0)

/*
 * For highest performance the L1 NH and L2 polynomial hashes should be
 * carefully implemented to take advantage of one's target architecture.
 * Here these two hash functions are defined multiple time; once for
 * 64-bit architectures, once for 32-bit SSE2 architectures, and once
 * for the rest (32-bit) architectures.
 * For each, nh_16 *must* be defined (works on multiples of 16 bytes).
 * Optionally, nh_vmac_nhbytes can be defined (for multiples of
 * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
 * NH computations at once).
 */

#ifdef CONFIG_64BIT

#define nh_16(mp, kp, nw, rh, rl)					\
	do {								\
		int i; u64 th, tl;					\
		rh = rl = 0;						\
		for (i = 0; i < nw; i += 2) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
		}							\
	} while (0)

#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1)				\
	do {								\
		int i; u64 th, tl;					\
		rh1 = rl1 = rh = rl = 0;				\
		for (i = 0; i < nw; i += 2) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
			ADD128(rh1, rl1, th, tl);			\
		}							\
	} while (0)

#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
#define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
	do {								\
		int i; u64 th, tl;					\
		rh = rl = 0;						\
		for (i = 0; i < nw; i += 8) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
			ADD128(rh, rl, th, tl);				\
		}							\
	} while (0)

#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1)			\
	do {								\
		int i; u64 th, tl;					\
		rh1 = rl1 = rh = rl = 0;				\
		for (i = 0; i < nw; i += 8) {				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+1]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+1)+(kp)[i+3]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+3]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+3)+(kp)[i+5]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+5]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+5)+(kp)[i+7]);	\
			ADD128(rh1, rl1, th, tl);			\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+7]);	\
			ADD128(rh, rl, th, tl);				\
			MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8],	\
				pe64_to_cpup((mp)+i+7)+(kp)[i+9]);	\
			ADD128(rh1, rl1, th, tl);			\
		}							\
	} while (0)
#endif

#define poly_step(ah, al, kh, kl, mh, ml)				\
	do {								\
		u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0;		\
		/* compute ab*cd, put bd into result registers */	\
		PMUL64(t3h, t3l, al, kh);				\
		PMUL64(t2h, t2l, ah, kl);				\
		PMUL64(t1h, t1l, ah, 2*kh);				\
		PMUL64(ah, al, al, kl);					\
		/* add 2 * ac to result */				\
		ADD128(ah, al, t1h, t1l);				\
		/* add together ad + bc */				\
		ADD128(t2h, t2l, t3h, t3l);				\
		/* now (ah,al), (t2l,2*t2h) need summing */		\
		/* first add the high registers, carrying into t2h */	\
		ADD128(t2h, ah, z, t2l);				\
		/* double t2h and add top bit of ah */			\
		t2h = 2 * t2h + (ah >> 63);				\
		ah &= m63;						\
		/* now add the low registers */				\
		ADD128(ah, al, mh, ml);					\
		ADD128(ah, al, z, t2h);					\
	} while (0)

#else /* ! CONFIG_64BIT */

#ifndef nh_16
#define nh_16(mp, kp, nw, rh, rl)					\
	do {								\
		u64 t1, t2, m1, m2, t;					\
		int i;							\
		rh = rl = t = 0;					\
		for (i = 0; i < nw; i += 2)  {				\
			t1 = pe64_to_cpup(mp+i) + kp[i];		\
			t2 = pe64_to_cpup(mp+i+1) + kp[i+1];		\
			m2 = MUL32(t1 >> 32, t2);			\
			m1 = MUL32(t1, t2 >> 32);			\
			ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32),	\
				MUL32(t1, t2));				\
			rh += (u64)(u32)(m1 >> 32)			\
				+ (u32)(m2 >> 32);			\
			t += (u64)(u32)m1 + (u32)m2;			\
		}							\
		ADD128(rh, rl, (t >> 32), (t << 32));			\
	} while (0)
#endif

static void poly_step_func(u64 *ahi, u64 *alo,
			const u64 *kh, const u64 *kl,
			const u64 *mh, const u64 *ml)
{
#define a0 (*(((u32 *)alo)+INDEX_LOW))
#define a1 (*(((u32 *)alo)+INDEX_HIGH))
#define a2 (*(((u32 *)ahi)+INDEX_LOW))
#define a3 (*(((u32 *)ahi)+INDEX_HIGH))
#define k0 (*(((u32 *)kl)+INDEX_LOW))
#define k1 (*(((u32 *)kl)+INDEX_HIGH))
#define k2 (*(((u32 *)kh)+INDEX_LOW))
#define k3 (*(((u32 *)kh)+INDEX_HIGH))

	u64 p, q, t;
	u32 t2;

	p = MUL32(a3, k3);
	p += p;
	p += *(u64 *)mh;
	p += MUL32(a0, k2);
	p += MUL32(a1, k1);
	p += MUL32(a2, k0);
	t = (u32)(p);
	p >>= 32;
	p += MUL32(a0, k3);
	p += MUL32(a1, k2);
	p += MUL32(a2, k1);
	p += MUL32(a3, k0);
	t |= ((u64)((u32)p & 0x7fffffff)) << 32;
	p >>= 31;
	p += (u64)(((u32 *)ml)[INDEX_LOW]);
	p += MUL32(a0, k0);
	q =  MUL32(a1, k3);
	q += MUL32(a2, k2);
	q += MUL32(a3, k1);
	q += q;
	p += q;
	t2 = (u32)(p);
	p >>= 32;
	p += (u64)(((u32 *)ml)[INDEX_HIGH]);
	p += MUL32(a0, k1);
	p += MUL32(a1, k0);
	q =  MUL32(a2, k3);
	q += MUL32(a3, k2);
	q += q;
	p += q;
	*(u64 *)(alo) = (p << 32) | t2;
	p >>= 32;
	*(u64 *)(ahi) = p + t;

#undef a0
#undef a1
#undef a2
#undef a3
#undef k0
#undef k1
#undef k2
#undef k3
}

#define poly_step(ah, al, kh, kl, mh, ml)				\
	poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))

#endif  /* end of specialized NH and poly definitions */

/* At least nh_16 is defined. Defined others as needed here */
#ifndef nh_16_2
#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2)				\
	do { 								\
		nh_16(mp, kp, nw, rh, rl);				\
		nh_16(mp, ((kp)+2), nw, rh2, rl2);			\
	} while (0)
#endif
#ifndef nh_vmac_nhbytes
#define nh_vmac_nhbytes(mp, kp, nw, rh, rl)				\
	nh_16(mp, kp, nw, rh, rl)
#endif
#ifndef nh_vmac_nhbytes_2
#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2)			\
	do {								\
		nh_vmac_nhbytes(mp, kp, nw, rh, rl);			\
		nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2);		\
	} while (0)
#endif

static void vhash_abort(struct vmac_ctx *ctx)
{
	ctx->polytmp[0] = ctx->polykey[0] ;
	ctx->polytmp[1] = ctx->polykey[1] ;
	ctx->first_block_processed = 0;
}

static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
{
	u64 rh, rl, t, z = 0;

	/* fully reduce (p1,p2)+(len,0) mod p127 */
	t = p1 >> 63;
	p1 &= m63;
	ADD128(p1, p2, len, t);
	/* At this point, (p1,p2) is at most 2^127+(len<<64) */
	t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
	ADD128(p1, p2, z, t);
	p1 &= m63;

	/* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
	t = p1 + (p2 >> 32);
	t += (t >> 32);
	t += (u32)t > 0xfffffffeu;
	p1 += (t >> 32);
	p2 += (p1 << 32);

	/* compute (p1+k1)%p64 and (p2+k2)%p64 */
	p1 += k1;
	p1 += (0 - (p1 < k1)) & 257;
	p2 += k2;
	p2 += (0 - (p2 < k2)) & 257;

	/* compute (p1+k1)*(p2+k2)%p64 */
	MUL64(rh, rl, p1, p2);
	t = rh >> 56;
	ADD128(t, rl, z, rh);
	rh <<= 8;
	ADD128(t, rl, z, rh);
	t += t << 8;
	rl += t;
	rl += (0 - (rl < t)) & 257;
	rl += (0 - (rl > p64-1)) & 257;
	return rl;
}

static void vhash_update(const unsigned char *m,
			unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
			struct vmac_ctx *ctx)
{
	u64 rh, rl, *mptr;
	const u64 *kptr = (u64 *)ctx->nhkey;
	int i;
	u64 ch, cl;
	u64 pkh = ctx->polykey[0];
	u64 pkl = ctx->polykey[1];

	mptr = (u64 *)m;
	i = mbytes / VMAC_NHBYTES;  /* Must be non-zero */

	ch = ctx->polytmp[0];
	cl = ctx->polytmp[1];

	if (!ctx->first_block_processed) {
		ctx->first_block_processed = 1;
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		ADD128(ch, cl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		i--;
	}

	while (i--) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
	}

	ctx->polytmp[0] = ch;
	ctx->polytmp[1] = cl;
}

static u64 vhash(unsigned char m[], unsigned int mbytes,
			u64 *tagl, struct vmac_ctx *ctx)
{
	u64 rh, rl, *mptr;
	const u64 *kptr = (u64 *)ctx->nhkey;
	int i, remaining;
	u64 ch, cl;
	u64 pkh = ctx->polykey[0];
	u64 pkl = ctx->polykey[1];

	mptr = (u64 *)m;
	i = mbytes / VMAC_NHBYTES;
	remaining = mbytes % VMAC_NHBYTES;

	if (ctx->first_block_processed) {
		ch = ctx->polytmp[0];
		cl = ctx->polytmp[1];
	} else if (i) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
		ch &= m62;
		ADD128(ch, cl, pkh, pkl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		i--;
	} else if (remaining) {
		nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
		ch &= m62;
		ADD128(ch, cl, pkh, pkl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
		goto do_l3;
	} else {/* Empty String */
		ch = pkh; cl = pkl;
		goto do_l3;
	}

	while (i--) {
		nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
		mptr += (VMAC_NHBYTES/sizeof(u64));
	}
	if (remaining) {
		nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
		rh &= m62;
		poly_step(ch, cl, pkh, pkl, rh, rl);
	}

do_l3:
	vhash_abort(ctx);
	remaining *= 8;
	return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
}

static u64 vmac(unsigned char m[], unsigned int mbytes,
			unsigned char n[16], u64 *tagl,
			struct vmac_ctx_t *ctx)
{
	u64 *in_n, *out_p;
	u64 p, h;
	int i;

	in_n = ctx->__vmac_ctx.cached_nonce;
	out_p = ctx->__vmac_ctx.cached_aes;

	i = n[15] & 1;
	if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) {
		in_n[0] = *(u64 *)(n);
		in_n[1] = *(u64 *)(n+8);
		((unsigned char *)in_n)[15] &= 0xFE;
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out_p, (unsigned char *)in_n);

		((unsigned char *)in_n)[15] |= (unsigned char)(1-i);
	}
	p = be64_to_cpup(out_p + i);
	h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
	return le64_to_cpu(p + h);
}

static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
{
	u64 in[2] = {0}, out[2];
	unsigned i;
	int err = 0;

	err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
	if (err)
		return err;

	/* Fill nh key */
	((unsigned char *)in)[0] = 0x80;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out, (unsigned char *)in);
		ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
		ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
		((unsigned char *)in)[15] += 1;
	}

	/* Fill poly key */
	((unsigned char *)in)[0] = 0xC0;
	in[1] = 0;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
		crypto_cipher_encrypt_one(ctx->child,
			(unsigned char *)out, (unsigned char *)in);
		ctx->__vmac_ctx.polytmp[i] =
			ctx->__vmac_ctx.polykey[i] =
				be64_to_cpup(out) & mpoly;
		ctx->__vmac_ctx.polytmp[i+1] =
			ctx->__vmac_ctx.polykey[i+1] =
				be64_to_cpup(out+1) & mpoly;
		((unsigned char *)in)[15] += 1;
	}

	/* Fill ip key */
	((unsigned char *)in)[0] = 0xE0;
	in[1] = 0;
	for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
		do {
			crypto_cipher_encrypt_one(ctx->child,
				(unsigned char *)out, (unsigned char *)in);
			ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
			ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
			((unsigned char *)in)[15] += 1;
		} while (ctx->__vmac_ctx.l3key[i] >= p64
			|| ctx->__vmac_ctx.l3key[i+1] >= p64);
	}

	/* Invalidate nonce/aes cache and reset other elements */
	ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
	ctx->__vmac_ctx.cached_nonce[1] = (u64)0;  /* Ensure illegal nonce */
	ctx->__vmac_ctx.first_block_processed = 0;

	return err;
}

static int vmac_setkey(struct crypto_shash *parent,
		const u8 *key, unsigned int keylen)
{
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);

	if (keylen != VMAC_KEY_LEN) {
		crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
		return -EINVAL;
	}

	return vmac_set_key((u8 *)key, ctx);
}

static int vmac_init(struct shash_desc *pdesc)
{
	return 0;
}

static int vmac_update(struct shash_desc *pdesc, const u8 *p,
		unsigned int len)
{
	struct crypto_shash *parent = pdesc->tfm;
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);

	vhash_update(p, len, &ctx->__vmac_ctx);

	return 0;
}

static int vmac_final(struct shash_desc *pdesc, u8 *out)
{
	struct crypto_shash *parent = pdesc->tfm;
	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	vmac_t mac;
	u8 nonce[16] = {};

	mac = vmac(NULL, 0, nonce, NULL, ctx);
	memcpy(out, &mac, sizeof(vmac_t));
	memset(&mac, 0, sizeof(vmac_t));
	memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
	return 0;
}

static int vmac_init_tfm(struct crypto_tfm *tfm)
{
	struct crypto_cipher *cipher;
	struct crypto_instance *inst = (void *)tfm->__crt_alg;
	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);

	cipher = crypto_spawn_cipher(spawn);
	if (IS_ERR(cipher))
		return PTR_ERR(cipher);

	ctx->child = cipher;
	return 0;
}

static void vmac_exit_tfm(struct crypto_tfm *tfm)
{
	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	crypto_free_cipher(ctx->child);
}

static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb)
{
	struct shash_instance *inst;
	struct crypto_alg *alg;
	int err;

	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
	if (err)
		return err;

	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
			CRYPTO_ALG_TYPE_MASK);
	if (IS_ERR(alg))
		return PTR_ERR(alg);

	inst = shash_alloc_instance("vmac", alg);
	err = PTR_ERR(inst);
	if (IS_ERR(inst))
		goto out_put_alg;

	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
			shash_crypto_instance(inst),
			CRYPTO_ALG_TYPE_MASK);
	if (err)
		goto out_free_inst;

	inst->alg.base.cra_priority = alg->cra_priority;
	inst->alg.base.cra_blocksize = alg->cra_blocksize;
	inst->alg.base.cra_alignmask = alg->cra_alignmask;

	inst->alg.digestsize = sizeof(vmac_t);
	inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
	inst->alg.base.cra_init = vmac_init_tfm;
	inst->alg.base.cra_exit = vmac_exit_tfm;

	inst->alg.init = vmac_init;
	inst->alg.update = vmac_update;
	inst->alg.final = vmac_final;
	inst->alg.setkey = vmac_setkey;

	err = shash_register_instance(tmpl, inst);
	if (err) {
out_free_inst:
		shash_free_instance(shash_crypto_instance(inst));
	}

out_put_alg:
	crypto_mod_put(alg);
	return err;
}

static struct crypto_template vmac_tmpl = {
	.name = "vmac",
	.create = vmac_create,
	.free = shash_free_instance,
	.module = THIS_MODULE,
};

static int __init vmac_module_init(void)
{
	return crypto_register_template(&vmac_tmpl);
}

static void __exit vmac_module_exit(void)
{
	crypto_unregister_template(&vmac_tmpl);
}

module_init(vmac_module_init);
module_exit(vmac_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("VMAC hash algorithm");
Commit	Line	Data
f1939f7c SW	1	/*
	2	* Modified to interface to the Linux kernel
	3	* Copyright (c) 2009, Intel Corporation.
	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms and conditions of the GNU General Public License,
	7	* version 2, as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
	14	* You should have received a copy of the GNU General Public License along with
	15	* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
	16	* Place - Suite 330, Boston, MA 02111-1307 USA.
	17	*/
	18
	19	/* --------------------------------------------------------------------------
	20	* VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai.
	21	* This implementation is herby placed in the public domain.
	22	* The authors offers no warranty. Use at your own risk.
	23	* Please send bug reports to the authors.
	24	* Last modified: 17 APR 08, 1700 PDT
	25	* ----------------------------------------------------------------------- */
	26
	27	#include <linux/init.h>
	28	#include <linux/types.h>
	29	#include <linux/crypto.h>
	30	#include <linux/scatterlist.h>
	31	#include <asm/byteorder.h>
	32	#include <crypto/scatterwalk.h>
	33	#include <crypto/vmac.h>
	34	#include <crypto/internal/hash.h>
	35
	36	/*
	37	* Constants and masks
	38	*/
	39	#define UINT64_C(x) x##ULL
	40	const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */
	41	const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */
	42	const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */
	43	const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */
	44	const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
	45
304a204e SW	46	#define pe64_to_cpup le64_to_cpup /* Prefer little endian */
304a204e SW	47
f1939f7c SW	48	#ifdef __LITTLE_ENDIAN
	49	#define INDEX_HIGH 1
	50	#define INDEX_LOW 0
	51	#else
	52	#define INDEX_HIGH 0
	53	#define INDEX_LOW 1
	54	#endif
	55
	56	/*
	57	* The following routines are used in this implementation. They are
	58	* written via macros to simulate zero-overhead call-by-reference.
	59	*
	60	* MUL64: 64x64->128-bit multiplication
	61	* PMUL64: assumes top bits cleared on inputs
	62	* ADD128: 128x128->128-bit addition
	63	*/
	64
	65	#define ADD128(rh, rl, ih, il) \
	66	do { \
	67	u64 _il = (il); \
	68	(rl) += (_il); \
	69	if ((rl) < (_il)) \
	70	(rh)++; \
	71	(rh) += (ih); \
	72	} while (0)
	73
	74	#define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2))
	75
	76	#define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \
	77	do { \
	78	u64 _i1 = (i1), _i2 = (i2); \
	79	u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \
	80	rh = MUL32(_i1>>32, _i2>>32); \
	81	rl = MUL32(_i1, _i2); \
	82	ADD128(rh, rl, (m >> 32), (m << 32)); \
	83	} while (0)
	84
	85	#define MUL64(rh, rl, i1, i2) \
	86	do { \
	87	u64 _i1 = (i1), _i2 = (i2); \
	88	u64 m1 = MUL32(_i1, _i2>>32); \
	89	u64 m2 = MUL32(_i1>>32, _i2); \
	90	rh = MUL32(_i1>>32, _i2>>32); \
	91	rl = MUL32(_i1, _i2); \
	92	ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \
	93	ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \
	94	} while (0)
	95
	96	/*
	97	* For highest performance the L1 NH and L2 polynomial hashes should be
25985edc	98	* carefully implemented to take advantage of one's target architecture.
f1939f7c SW	99	* Here these two hash functions are defined multiple time; once for
	100	* 64-bit architectures, once for 32-bit SSE2 architectures, and once
	101	* for the rest (32-bit) architectures.
	102	* For each, nh_16 must be defined (works on multiples of 16 bytes).
	103	* Optionally, nh_vmac_nhbytes can be defined (for multiples of
	104	* VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two
	105	* NH computations at once).
	106	*/
	107
	108	#ifdef CONFIG_64BIT
	109
	110	#define nh_16(mp, kp, nw, rh, rl) \
	111	do { \
	112	int i; u64 th, tl; \
	113	rh = rl = 0; \
	114	for (i = 0; i < nw; i += 2) { \
304a204e SW	115	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	116	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c SW	117	ADD128(rh, rl, th, tl); \
	118	} \
	119	} while (0)
	120
	121	#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \
	122	do { \
	123	int i; u64 th, tl; \
	124	rh1 = rl1 = rh = rl = 0; \
	125	for (i = 0; i < nw; i += 2) { \
304a204e SW	126	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	127	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	128	ADD128(rh, rl, th, tl); \
304a204e SW	129	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
304a204e SW	130	pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
f1939f7c SW	131	ADD128(rh1, rl1, th, tl); \
	132	} \
	133	} while (0)
	134
	135	#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */
	136	#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
	137	do { \
	138	int i; u64 th, tl; \
	139	rh = rl = 0; \
	140	for (i = 0; i < nw; i += 8) { \
304a204e SW	141	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	142	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	143	ADD128(rh, rl, th, tl); \
304a204e SW	144	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
304a204e SW	145	pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
f1939f7c	146	ADD128(rh, rl, th, tl); \
304a204e SW	147	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
304a204e SW	148	pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
f1939f7c	149	ADD128(rh, rl, th, tl); \
304a204e SW	150	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
304a204e SW	151	pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
f1939f7c SW	152	ADD128(rh, rl, th, tl); \
	153	} \
	154	} while (0)
	155
	156	#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \
	157	do { \
	158	int i; u64 th, tl; \
	159	rh1 = rl1 = rh = rl = 0; \
	160	for (i = 0; i < nw; i += 8) { \
304a204e SW	161	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
304a204e SW	162	pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
f1939f7c	163	ADD128(rh, rl, th, tl); \
304a204e SW	164	MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
304a204e SW	165	pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
f1939f7c	166	ADD128(rh1, rl1, th, tl); \
304a204e SW	167	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
304a204e SW	168	pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
f1939f7c	169	ADD128(rh, rl, th, tl); \
304a204e SW	170	MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
304a204e SW	171	pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \
f1939f7c	172	ADD128(rh1, rl1, th, tl); \
304a204e SW	173	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
304a204e SW	174	pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
f1939f7c	175	ADD128(rh, rl, th, tl); \
304a204e SW	176	MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
304a204e SW	177	pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \
f1939f7c	178	ADD128(rh1, rl1, th, tl); \
304a204e SW	179	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
304a204e SW	180	pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
f1939f7c	181	ADD128(rh, rl, th, tl); \
304a204e SW	182	MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
304a204e SW	183	pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \
f1939f7c SW	184	ADD128(rh1, rl1, th, tl); \
	185	} \
	186	} while (0)
	187	#endif
	188
	189	#define poly_step(ah, al, kh, kl, mh, ml) \
	190	do { \
	191	u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \
	192	/* compute abcd, put bd into result registers / \
	193	PMUL64(t3h, t3l, al, kh); \
	194	PMUL64(t2h, t2l, ah, kl); \
	195	PMUL64(t1h, t1l, ah, 2*kh); \
	196	PMUL64(ah, al, al, kl); \
	197	/* add 2 * ac to result */ \
	198	ADD128(ah, al, t1h, t1l); \
	199	/* add together ad + bc */ \
	200	ADD128(t2h, t2l, t3h, t3l); \
	201	/* now (ah,al), (t2l,2t2h) need summing / \
	202	/* first add the high registers, carrying into t2h */ \
	203	ADD128(t2h, ah, z, t2l); \
	204	/* double t2h and add top bit of ah */ \
	205	t2h = 2 * t2h + (ah >> 63); \
	206	ah &= m63; \
	207	/* now add the low registers */ \
	208	ADD128(ah, al, mh, ml); \
	209	ADD128(ah, al, z, t2h); \
	210	} while (0)
	211
	212	#else /* ! CONFIG_64BIT */
	213
	214	#ifndef nh_16
	215	#define nh_16(mp, kp, nw, rh, rl) \
	216	do { \
	217	u64 t1, t2, m1, m2, t; \
	218	int i; \
	219	rh = rl = t = 0; \
	220	for (i = 0; i < nw; i += 2) { \
304a204e SW	221	t1 = pe64_to_cpup(mp+i) + kp[i]; \
304a204e SW	222	t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \
f1939f7c SW	223	m2 = MUL32(t1 >> 32, t2); \
	224	m1 = MUL32(t1, t2 >> 32); \
	225	ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \
	226	MUL32(t1, t2)); \
	227	rh += (u64)(u32)(m1 >> 32) \
	228	+ (u32)(m2 >> 32); \
	229	t += (u64)(u32)m1 + (u32)m2; \
	230	} \
	231	ADD128(rh, rl, (t >> 32), (t << 32)); \
	232	} while (0)
	233	#endif
	234
	235	static void poly_step_func(u64 ahi, u64 alo,
	236	const u64 kh, const u64 kl,
	237	const u64 mh, const u64 ml)
	238	{
	239	#define a0 ((((u32 )alo)+INDEX_LOW))
	240	#define a1 ((((u32 )alo)+INDEX_HIGH))
	241	#define a2 ((((u32 )ahi)+INDEX_LOW))
	242	#define a3 ((((u32 )ahi)+INDEX_HIGH))
	243	#define k0 ((((u32 )kl)+INDEX_LOW))
	244	#define k1 ((((u32 )kl)+INDEX_HIGH))
	245	#define k2 ((((u32 )kh)+INDEX_LOW))
	246	#define k3 ((((u32 )kh)+INDEX_HIGH))
	247
	248	u64 p, q, t;
	249	u32 t2;
	250
	251	p = MUL32(a3, k3);
	252	p += p;
	253	p += (u64 )mh;
	254	p += MUL32(a0, k2);
	255	p += MUL32(a1, k1);
	256	p += MUL32(a2, k0);
	257	t = (u32)(p);
	258	p >>= 32;
	259	p += MUL32(a0, k3);
	260	p += MUL32(a1, k2);
	261	p += MUL32(a2, k1);
	262	p += MUL32(a3, k0);
	263	t \|= ((u64)((u32)p & 0x7fffffff)) << 32;
	264	p >>= 31;
	265	p += (u64)(((u32 *)ml)[INDEX_LOW]);
	266	p += MUL32(a0, k0);
	267	q = MUL32(a1, k3);
	268	q += MUL32(a2, k2);
	269	q += MUL32(a3, k1);
	270	q += q;
	271	p += q;
	272	t2 = (u32)(p);
	273	p >>= 32;
	274	p += (u64)(((u32 *)ml)[INDEX_HIGH]);
	275	p += MUL32(a0, k1);
	276	p += MUL32(a1, k0);
	277	q = MUL32(a2, k3);
	278	q += MUL32(a3, k2);
	279	q += q;
	280	p += q;
	281	(u64 )(alo) = (p << 32) \| t2;
	282	p >>= 32;
	283	(u64 )(ahi) = p + t;
	284
	285	#undef a0
	286	#undef a1
287	#undef a2
288	#undef a3
289	#undef k0
290	#undef k1
291	#undef k2
292	#undef k3
293	}
294
295	#define poly_step(ah, al, kh, kl, mh, ml) \
296	poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml))
297
298	#endif /* end of specialized NH and poly definitions */
299
300	/* At least nh_16 is defined. Defined others as needed here */
301	#ifndef nh_16_2
302	#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \
303	do { \
304	nh_16(mp, kp, nw, rh, rl); \
305	nh_16(mp, ((kp)+2), nw, rh2, rl2); \
306	} while (0)
307	#endif
308	#ifndef nh_vmac_nhbytes
309	#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \
310	nh_16(mp, kp, nw, rh, rl)
311	#endif
312	#ifndef nh_vmac_nhbytes_2
313	#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \
314	do { \
315	nh_vmac_nhbytes(mp, kp, nw, rh, rl); \
316	nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \
317	} while (0)
318	#endif
319
320	static void vhash_abort(struct vmac_ctx *ctx)
321	{
322	ctx->polytmp[0] = ctx->polykey[0] ;
323	ctx->polytmp[1] = ctx->polykey[1] ;
324	ctx->first_block_processed = 0;
325	}
326
304a204e	327	static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
f1939f7c SW	328	{
	329	u64 rh, rl, t, z = 0;
	330
	331	/* fully reduce (p1,p2)+(len,0) mod p127 */
	332	t = p1 >> 63;
	333	p1 &= m63;
	334	ADD128(p1, p2, len, t);
	335	/* At this point, (p1,p2) is at most 2^127+(len<<64) */
	336	t = (p1 > m63) + ((p1 == m63) && (p2 == m64));
	337	ADD128(p1, p2, z, t);
	338	p1 &= m63;
	339
	340	/* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */
	341	t = p1 + (p2 >> 32);
	342	t += (t >> 32);
	343	t += (u32)t > 0xfffffffeu;
	344	p1 += (t >> 32);
	345	p2 += (p1 << 32);
	346
	347	/* compute (p1+k1)%p64 and (p2+k2)%p64 */
	348	p1 += k1;
	349	p1 += (0 - (p1 < k1)) & 257;
	350	p2 += k2;
	351	p2 += (0 - (p2 < k2)) & 257;
	352
	353	/* compute (p1+k1)(p2+k2)%p64 /
	354	MUL64(rh, rl, p1, p2);
	355	t = rh >> 56;
	356	ADD128(t, rl, z, rh);
	357	rh <<= 8;
	358	ADD128(t, rl, z, rh);
	359	t += t << 8;
	360	rl += t;
	361	rl += (0 - (rl < t)) & 257;
	362	rl += (0 - (rl > p64-1)) & 257;
	363	return rl;
	364	}
	365
	366	static void vhash_update(const unsigned char *m,
	367	unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */
	368	struct vmac_ctx *ctx)
	369	{
	370	u64 rh, rl, *mptr;
	371	const u64 kptr = (u64 )ctx->nhkey;
	372	int i;
	373	u64 ch, cl;
	374	u64 pkh = ctx->polykey[0];
	375	u64 pkl = ctx->polykey[1];
	376
	377	mptr = (u64 *)m;
	378	i = mbytes / VMAC_NHBYTES; /* Must be non-zero */
	379
	380	ch = ctx->polytmp[0];
	381	cl = ctx->polytmp[1];
	382
	383	if (!ctx->first_block_processed) {
	384	ctx->first_block_processed = 1;
	385	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
	386	rh &= m62;
	387	ADD128(ch, cl, rh, rl);
	388	mptr += (VMAC_NHBYTES/sizeof(u64));
	389	i--;
	390	}
	391
392	while (i--) {
393	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
394	rh &= m62;
395	poly_step(ch, cl, pkh, pkl, rh, rl);
396	mptr += (VMAC_NHBYTES/sizeof(u64));
397	}
398
399	ctx->polytmp[0] = ch;
400	ctx->polytmp[1] = cl;
401	}
402
403	static u64 vhash(unsigned char m[], unsigned int mbytes,
404	u64 tagl, struct vmac_ctx ctx)
405	{
406	u64 rh, rl, *mptr;
407	const u64 kptr = (u64 )ctx->nhkey;
408	int i, remaining;
409	u64 ch, cl;
410	u64 pkh = ctx->polykey[0];
411	u64 pkl = ctx->polykey[1];
412
413	mptr = (u64 *)m;
414	i = mbytes / VMAC_NHBYTES;
415	remaining = mbytes % VMAC_NHBYTES;
416
417	if (ctx->first_block_processed) {
418	ch = ctx->polytmp[0];
419	cl = ctx->polytmp[1];
420	} else if (i) {
421	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl);
422	ch &= m62;
423	ADD128(ch, cl, pkh, pkl);
424	mptr += (VMAC_NHBYTES/sizeof(u64));
425	i--;
426	} else if (remaining) {
427	nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl);
428	ch &= m62;
429	ADD128(ch, cl, pkh, pkl);
430	mptr += (VMAC_NHBYTES/sizeof(u64));
431	goto do_l3;
432	} else {/* Empty String */
433	ch = pkh; cl = pkl;
434	goto do_l3;
435	}
436
437	while (i--) {
438	nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl);
439	rh &= m62;
440	poly_step(ch, cl, pkh, pkl, rh, rl);
441	mptr += (VMAC_NHBYTES/sizeof(u64));
442	}
443	if (remaining) {
444	nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl);
445	rh &= m62;
446	poly_step(ch, cl, pkh, pkl, rh, rl);
447	}
448
449	do_l3:
450	vhash_abort(ctx);
451	remaining *= 8;
452	return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining);
453	}
454
455	static u64 vmac(unsigned char m[], unsigned int mbytes,
456	unsigned char n[16], u64 *tagl,
457	struct vmac_ctx_t *ctx)
458	{
459	u64 in_n, out_p;
460	u64 p, h;
461	int i;
462
463	in_n = ctx->__vmac_ctx.cached_nonce;
464	out_p = ctx->__vmac_ctx.cached_aes;
465
466	i = n[15] & 1;
467	if (((u64 )(n+8) != in_n[1]) \|\| ((u64 )(n) != in_n[0])) {
468	in_n[0] = (u64 )(n);
469	in_n[1] = (u64 )(n+8);
470	((unsigned char *)in_n)[15] &= 0xFE;
471	crypto_cipher_encrypt_one(ctx->child,
472	(unsigned char )out_p, (unsigned char )in_n);
473
474	((unsigned char *)in_n)[15] \|= (unsigned char)(1-i);
475	}
476	p = be64_to_cpup(out_p + i);
477	h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
304a204e	478	return le64_to_cpu(p + h);
f1939f7c SW	479	}
	480
	481	static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
	482	{
	483	u64 in[2] = {0}, out[2];
	484	unsigned i;
	485	int err = 0;
	486
	487	err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN);
	488	if (err)
	489	return err;
	490
	491	/* Fill nh key */
	492	((unsigned char *)in)[0] = 0x80;
	493	for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) {
	494	crypto_cipher_encrypt_one(ctx->child,
	495	(unsigned char )out, (unsigned char )in);
	496	ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out);
	497	ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1);
	498	((unsigned char *)in)[15] += 1;
	499	}
	500
	501	/* Fill poly key */
	502	((unsigned char *)in)[0] = 0xC0;
	503	in[1] = 0;
	504	for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) {
	505	crypto_cipher_encrypt_one(ctx->child,
	506	(unsigned char )out, (unsigned char )in);
	507	ctx->__vmac_ctx.polytmp[i] =
	508	ctx->__vmac_ctx.polykey[i] =
	509	be64_to_cpup(out) & mpoly;
	510	ctx->__vmac_ctx.polytmp[i+1] =
	511	ctx->__vmac_ctx.polykey[i+1] =
	512	be64_to_cpup(out+1) & mpoly;
	513	((unsigned char *)in)[15] += 1;
	514	}
	515
	516	/* Fill ip key */
	517	((unsigned char *)in)[0] = 0xE0;
	518	in[1] = 0;
	519	for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) {
	520	do {
	521	crypto_cipher_encrypt_one(ctx->child,
	522	(unsigned char )out, (unsigned char )in);
	523	ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out);
	524	ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1);
	525	((unsigned char *)in)[15] += 1;
	526	} while (ctx->__vmac_ctx.l3key[i] >= p64
	527	\|\| ctx->__vmac_ctx.l3key[i+1] >= p64);
	528	}
	529
	530	/* Invalidate nonce/aes cache and reset other elements */
	531	ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */
	532	ctx->__vmac_ctx.cached_nonce[1] = (u64)0; /* Ensure illegal nonce */
	533	ctx->__vmac_ctx.first_block_processed = 0;
	534
	535	return err;
	536	}
	537
	538	static int vmac_setkey(struct crypto_shash *parent,
	539	const u8 *key, unsigned int keylen)
	540	{
	541	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	542
543	if (keylen != VMAC_KEY_LEN) {
544	crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN);
545	return -EINVAL;
546	}
547
548	return vmac_set_key((u8 *)key, ctx);
549	}
550
551	static int vmac_init(struct shash_desc *pdesc)
552	{
f1939f7c SW	553	return 0;
	554	}
	555
	556	static int vmac_update(struct shash_desc pdesc, const u8 p,
	557	unsigned int len)
	558	{
	559	struct crypto_shash *parent = pdesc->tfm;
	560	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	561
	562	vhash_update(p, len, &ctx->__vmac_ctx);
	563
	564	return 0;
	565	}
	566
	567	static int vmac_final(struct shash_desc pdesc, u8 out)
	568	{
	569	struct crypto_shash *parent = pdesc->tfm;
	570	struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
	571	vmac_t mac;
	572	u8 nonce[16] = {};
	573
	574	mac = vmac(NULL, 0, nonce, NULL, ctx);
	575	memcpy(out, &mac, sizeof(vmac_t));
	576	memset(&mac, 0, sizeof(vmac_t));
	577	memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
	578	return 0;
	579	}
	580
	581	static int vmac_init_tfm(struct crypto_tfm *tfm)
	582	{
	583	struct crypto_cipher *cipher;
	584	struct crypto_instance inst = (void )tfm->__crt_alg;
	585	struct crypto_spawn *spawn = crypto_instance_ctx(inst);
	586	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	587
	588	cipher = crypto_spawn_cipher(spawn);
	589	if (IS_ERR(cipher))
	590	return PTR_ERR(cipher);
	591
	592	ctx->child = cipher;
	593	return 0;
	594	}
	595
	596	static void vmac_exit_tfm(struct crypto_tfm *tfm)
	597	{
	598	struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm);
	599	crypto_free_cipher(ctx->child);
	600	}
	601
	602	static int vmac_create(struct crypto_template tmpl, struct rtattr *tb)
	603	{
	604	struct shash_instance *inst;
	605	struct crypto_alg *alg;
	606	int err;
	607
	608	err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
	609	if (err)
	610	return err;
	611
	612	alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
	613	CRYPTO_ALG_TYPE_MASK);
	614	if (IS_ERR(alg))
	615	return PTR_ERR(alg);
	616
617	inst = shash_alloc_instance("vmac", alg);
618	err = PTR_ERR(inst);
619	if (IS_ERR(inst))
620	goto out_put_alg;
621
622	err = crypto_init_spawn(shash_instance_ctx(inst), alg,
623	shash_crypto_instance(inst),
624	CRYPTO_ALG_TYPE_MASK);
625	if (err)
626	goto out_free_inst;
627
628	inst->alg.base.cra_priority = alg->cra_priority;
629	inst->alg.base.cra_blocksize = alg->cra_blocksize;
630	inst->alg.base.cra_alignmask = alg->cra_alignmask;
631
632	inst->alg.digestsize = sizeof(vmac_t);
633	inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t);
634	inst->alg.base.cra_init = vmac_init_tfm;
635	inst->alg.base.cra_exit = vmac_exit_tfm;
636
637	inst->alg.init = vmac_init;
638	inst->alg.update = vmac_update;
639	inst->alg.final = vmac_final;
640	inst->alg.setkey = vmac_setkey;
641
642	err = shash_register_instance(tmpl, inst);
643	if (err) {
644	out_free_inst:
645	shash_free_instance(shash_crypto_instance(inst));
646	}
647
648	out_put_alg:
649	crypto_mod_put(alg);
650	return err;
651	}
652
653	static struct crypto_template vmac_tmpl = {
654	.name = "vmac",
655	.create = vmac_create,
656	.free = shash_free_instance,
657	.module = THIS_MODULE,
658	};
659
660	static int __init vmac_module_init(void)
661	{
662	return crypto_register_template(&vmac_tmpl);
663	}
664
665	static void __exit vmac_module_exit(void)
666	{
667	crypto_unregister_template(&vmac_tmpl);
668	}
669
670	module_init(vmac_module_init);
671	module_exit(vmac_module_exit);
672
673	MODULE_LICENSE("GPL");
674	MODULE_DESCRIPTION("VMAC hash algorithm");
675