[linux-2.6-block.git] / lib / crypto / chacha20poly1305.c

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
 *
 * Information: https://tools.ietf.org/html/rfc8439
 */

#include <crypto/algapi.h>
#include <crypto/chacha20poly1305.h>
#include <crypto/chacha.h>
#include <crypto/poly1305.h>
#include <crypto/scatterwalk.h>

#include <asm/unaligned.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/module.h>

#define CHACHA_KEY_WORDS	(CHACHA_KEY_SIZE / sizeof(u32))

static void chacha_load_key(u32 *k, const u8 *in)
{
	k[0] = get_unaligned_le32(in);
	k[1] = get_unaligned_le32(in + 4);
	k[2] = get_unaligned_le32(in + 8);
	k[3] = get_unaligned_le32(in + 12);
	k[4] = get_unaligned_le32(in + 16);
	k[5] = get_unaligned_le32(in + 20);
	k[6] = get_unaligned_le32(in + 24);
	k[7] = get_unaligned_le32(in + 28);
}

static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
{
	u32 k[CHACHA_KEY_WORDS];
	u8 iv[CHACHA_IV_SIZE];

	memset(iv, 0, 8);
	memcpy(iv + 8, nonce + 16, 8);

	chacha_load_key(k, key);

	/* Compute the subkey given the original key and first 128 nonce bits */
	chacha_init(chacha_state, k, nonce);
	hchacha_block(chacha_state, k, 20);

	chacha_init(chacha_state, k, iv);

	memzero_explicit(k, sizeof(k));
	memzero_explicit(iv, sizeof(iv));
}

static void
__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	union {
		u8 block0[POLY1305_KEY_SIZE];
		__le64 lens[2];
	} b;

	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	poly1305_update(&poly1305_state, ad, ad_len);
	if (ad_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));

	chacha20_crypt(chacha_state, dst, src, src_len);

	poly1305_update(&poly1305_state, dst, src_len);
	if (src_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(src_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	poly1305_final(&poly1305_state, dst + src_len);

	memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
	memzero_explicit(&b, sizeof(b));
}

void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			      const u8 *ad, const size_t ad_len,
			      const u64 nonce,
			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];
	u32 k[CHACHA_KEY_WORDS];
	__le64 iv[2];

	chacha_load_key(k, key);

	iv[0] = 0;
	iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, k, (u8 *)iv);
	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);

	memzero_explicit(iv, sizeof(iv));
	memzero_explicit(k, sizeof(k));
}
EXPORT_SYMBOL(chacha20poly1305_encrypt);

void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			       const u8 *ad, const size_t ad_len,
			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];

	xchacha_init(chacha_state, key, nonce);
	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_encrypt);

static bool
__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	size_t dst_len;
	int ret;
	union {
		u8 block0[POLY1305_KEY_SIZE];
		u8 mac[POLY1305_DIGEST_SIZE];
		__le64 lens[2];
	} b;

	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
		return false;

	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	poly1305_update(&poly1305_state, ad, ad_len);
	if (ad_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));

	dst_len = src_len - POLY1305_DIGEST_SIZE;
	poly1305_update(&poly1305_state, src, dst_len);
	if (dst_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(dst_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	poly1305_final(&poly1305_state, b.mac);

	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
	if (likely(!ret))
		chacha20_crypt(chacha_state, dst, src, dst_len);

	memzero_explicit(&b, sizeof(b));

	return !ret;
}

bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			      const u8 *ad, const size_t ad_len,
			      const u64 nonce,
			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];
	u32 k[CHACHA_KEY_WORDS];
	__le64 iv[2];
	bool ret;

	chacha_load_key(k, key);

	iv[0] = 0;
	iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, k, (u8 *)iv);
	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
					 chacha_state);

	memzero_explicit(chacha_state, sizeof(chacha_state));
	memzero_explicit(iv, sizeof(iv));
	memzero_explicit(k, sizeof(k));
	return ret;
}
EXPORT_SYMBOL(chacha20poly1305_decrypt);

bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			       const u8 *ad, const size_t ad_len,
			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];

	xchacha_init(chacha_state, key, nonce);
	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
					  chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_decrypt);

static
bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
				       const size_t src_len,
				       const u8 *ad, const size_t ad_len,
				       const u64 nonce,
				       const u8 key[CHACHA20POLY1305_KEY_SIZE],
				       int encrypt)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	u32 chacha_state[CHACHA_STATE_WORDS];
	struct sg_mapping_iter miter;
	size_t partial = 0;
	unsigned int flags;
	bool ret = true;
	int sl;
	union {
		struct {
			u32 k[CHACHA_KEY_WORDS];
			__le64 iv[2];
		};
		u8 block0[POLY1305_KEY_SIZE];
		u8 chacha_stream[CHACHA_BLOCK_SIZE];
		struct {
			u8 mac[2][POLY1305_DIGEST_SIZE];
		};
		__le64 lens[2];
	} b __aligned(16);

	if (WARN_ON(src_len > INT_MAX))
		return false;

	chacha_load_key(b.k, key);

	b.iv[0] = 0;
	b.iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, b.k, (u8 *)b.iv);
	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	if (unlikely(ad_len)) {
		poly1305_update(&poly1305_state, ad, ad_len);
		if (ad_len & 0xf)
			poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	}

	flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;

	sg_miter_start(&miter, src, sg_nents(src), flags);

	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
		u8 *addr = miter.addr;
		size_t length = min_t(size_t, sl, miter.length);

		if (!encrypt)
			poly1305_update(&poly1305_state, addr, length);

		if (unlikely(partial)) {
			size_t l = min(length, CHACHA_BLOCK_SIZE - partial);

			crypto_xor(addr, b.chacha_stream + partial, l);
			partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);

			addr += l;
			length -= l;
		}

		if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
			size_t l = length;

			if (unlikely(length < sl))
				l &= ~(CHACHA_BLOCK_SIZE - 1);
			chacha20_crypt(chacha_state, addr, addr, l);
			addr += l;
			length -= l;
		}

		if (unlikely(length > 0)) {
			chacha20_crypt(chacha_state, b.chacha_stream, pad0,
				       CHACHA_BLOCK_SIZE);
			crypto_xor(addr, b.chacha_stream, length);
			partial = length;
		}

		if (encrypt)
			poly1305_update(&poly1305_state, miter.addr,
					min_t(size_t, sl, miter.length));
	}

	if (src_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(src_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
		if (encrypt) {
			poly1305_final(&poly1305_state,
				       miter.addr + miter.length + sl);
			ret = true;
		} else {
			poly1305_final(&poly1305_state, b.mac[0]);
			ret = !crypto_memneq(b.mac[0],
					     miter.addr + miter.length + sl,
					     POLY1305_DIGEST_SIZE);
		}
	}

	sg_miter_stop(&miter);

	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
		poly1305_final(&poly1305_state, b.mac[1]);
		scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
					 sizeof(b.mac[1]), encrypt);
		ret = encrypt ||
		      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
	}

	memzero_explicit(chacha_state, sizeof(chacha_state));
	memzero_explicit(&b, sizeof(b));

	return ret;
}

bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
					 const u8 *ad, const size_t ad_len,
					 const u64 nonce,
					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
						 nonce, key, 1);
}
EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);

bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
					 const u8 *ad, const size_t ad_len,
					 const u64 nonce,
					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
		return false;

	return chacha20poly1305_crypt_sg_inplace(src,
						 src_len - POLY1305_DIGEST_SIZE,
						 ad, ad_len, nonce, key, 0);
}
EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);

static int __init chacha20poly1305_init(void)
{
	if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
	    WARN_ON(!chacha20poly1305_selftest()))
		return -ENODEV;
	return 0;
}

static void __exit chacha20poly1305_exit(void)
{
}

module_init(chacha20poly1305_init);
module_exit(chacha20poly1305_exit);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
Commit	Line	Data
ed20078b AB	1	// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	/*
	3	* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	4	*
	5	* This is an implementation of the ChaCha20Poly1305 AEAD construction.
	6	*
	7	* Information: https://tools.ietf.org/html/rfc8439
	8	*/
	9
	10	#include <crypto/algapi.h>
	11	#include <crypto/chacha20poly1305.h>
	12	#include <crypto/chacha.h>
	13	#include <crypto/poly1305.h>
d95312a3	14	#include <crypto/scatterwalk.h>
ed20078b AB	15
	16	#include <asm/unaligned.h>
	17	#include <linux/kernel.h>
	18	#include <linux/init.h>
	19	#include <linux/mm.h>
	20	#include <linux/module.h>
	21
	22	#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
	23
ed20078b AB	24	static void chacha_load_key(u32 k, const u8 in)
	25	{
	26	k[0] = get_unaligned_le32(in);
	27	k[1] = get_unaligned_le32(in + 4);
	28	k[2] = get_unaligned_le32(in + 8);
	29	k[3] = get_unaligned_le32(in + 12);
	30	k[4] = get_unaligned_le32(in + 16);
	31	k[5] = get_unaligned_le32(in + 20);
	32	k[6] = get_unaligned_le32(in + 24);
	33	k[7] = get_unaligned_le32(in + 28);
	34	}
	35
	36	static void xchacha_init(u32 chacha_state, const u8 key, const u8 *nonce)
	37	{
	38	u32 k[CHACHA_KEY_WORDS];
	39	u8 iv[CHACHA_IV_SIZE];
	40
	41	memset(iv, 0, 8);
	42	memcpy(iv + 8, nonce + 16, 8);
	43
	44	chacha_load_key(k, key);
	45
	46	/* Compute the subkey given the original key and first 128 nonce bits */
	47	chacha_init(chacha_state, k, nonce);
	48	hchacha_block(chacha_state, k, 20);
	49
	50	chacha_init(chacha_state, k, iv);
	51
	52	memzero_explicit(k, sizeof(k));
	53	memzero_explicit(iv, sizeof(iv));
	54	}
	55
	56	static void
	57	__chacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	58	const u8 ad, const size_t ad_len, u32 chacha_state)
	59	{
	60	const u8 *pad0 = page_address(ZERO_PAGE(0));
	61	struct poly1305_desc_ctx poly1305_state;
	62	union {
	63	u8 block0[POLY1305_KEY_SIZE];
	64	__le64 lens[2];
	65	} b;
	66
413808b7	67	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
ed20078b AB	68	poly1305_init(&poly1305_state, b.block0);
	69
	70	poly1305_update(&poly1305_state, ad, ad_len);
	71	if (ad_len & 0xf)
	72	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	73
413808b7	74	chacha20_crypt(chacha_state, dst, src, src_len);
ed20078b AB	75
	76	poly1305_update(&poly1305_state, dst, src_len);
	77	if (src_len & 0xf)
	78	poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
	79
	80	b.lens[0] = cpu_to_le64(ad_len);
	81	b.lens[1] = cpu_to_le64(src_len);
	82	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	83
	84	poly1305_final(&poly1305_state, dst + src_len);
	85
	86	memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
	87	memzero_explicit(&b, sizeof(b));
	88	}
	89
	90	void chacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	91	const u8 *ad, const size_t ad_len,
	92	const u64 nonce,
	93	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	94	{
	95	u32 chacha_state[CHACHA_STATE_WORDS];
	96	u32 k[CHACHA_KEY_WORDS];
	97	__le64 iv[2];
	98
	99	chacha_load_key(k, key);
	100
	101	iv[0] = 0;
	102	iv[1] = cpu_to_le64(nonce);
	103
	104	chacha_init(chacha_state, k, (u8 *)iv);
	105	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
	106
	107	memzero_explicit(iv, sizeof(iv));
	108	memzero_explicit(k, sizeof(k));
	109	}
	110	EXPORT_SYMBOL(chacha20poly1305_encrypt);
	111
	112	void xchacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	113	const u8 *ad, const size_t ad_len,
	114	const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
	115	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	116	{
	117	u32 chacha_state[CHACHA_STATE_WORDS];
	118
	119	xchacha_init(chacha_state, key, nonce);
	120	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
	121	}
	122	EXPORT_SYMBOL(xchacha20poly1305_encrypt);
	123
	124	static bool
	125	__chacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	126	const u8 ad, const size_t ad_len, u32 chacha_state)
	127	{
	128	const u8 *pad0 = page_address(ZERO_PAGE(0));
	129	struct poly1305_desc_ctx poly1305_state;
	130	size_t dst_len;
	131	int ret;
	132	union {
	133	u8 block0[POLY1305_KEY_SIZE];
	134	u8 mac[POLY1305_DIGEST_SIZE];
	135	__le64 lens[2];
	136	} b;
	137
	138	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
139	return false;
140
413808b7	141	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
ed20078b AB	142	poly1305_init(&poly1305_state, b.block0);
	143
	144	poly1305_update(&poly1305_state, ad, ad_len);
	145	if (ad_len & 0xf)
	146	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	147
	148	dst_len = src_len - POLY1305_DIGEST_SIZE;
	149	poly1305_update(&poly1305_state, src, dst_len);
	150	if (dst_len & 0xf)
	151	poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
	152
	153	b.lens[0] = cpu_to_le64(ad_len);
	154	b.lens[1] = cpu_to_le64(dst_len);
	155	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	156
	157	poly1305_final(&poly1305_state, b.mac);
	158
	159	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
	160	if (likely(!ret))
413808b7	161	chacha20_crypt(chacha_state, dst, src, dst_len);
ed20078b AB	162
	163	memzero_explicit(&b, sizeof(b));
	164
	165	return !ret;
	166	}
	167
	168	bool chacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	169	const u8 *ad, const size_t ad_len,
	170	const u64 nonce,
	171	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	172	{
	173	u32 chacha_state[CHACHA_STATE_WORDS];
	174	u32 k[CHACHA_KEY_WORDS];
	175	__le64 iv[2];
	176	bool ret;
	177
	178	chacha_load_key(k, key);
	179
	180	iv[0] = 0;
	181	iv[1] = cpu_to_le64(nonce);
	182
	183	chacha_init(chacha_state, k, (u8 *)iv);
	184	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
	185	chacha_state);
	186
	187	memzero_explicit(chacha_state, sizeof(chacha_state));
	188	memzero_explicit(iv, sizeof(iv));
	189	memzero_explicit(k, sizeof(k));
	190	return ret;
	191	}
	192	EXPORT_SYMBOL(chacha20poly1305_decrypt);
	193
	194	bool xchacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	195	const u8 *ad, const size_t ad_len,
	196	const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
	197	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	198	{
	199	u32 chacha_state[CHACHA_STATE_WORDS];
	200
	201	xchacha_init(chacha_state, key, nonce);
	202	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
	203	chacha_state);
	204	}
	205	EXPORT_SYMBOL(xchacha20poly1305_decrypt);
	206
d95312a3 AB	207	static
	208	bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
	209	const size_t src_len,
	210	const u8 *ad, const size_t ad_len,
	211	const u64 nonce,
	212	const u8 key[CHACHA20POLY1305_KEY_SIZE],
	213	int encrypt)
	214	{
	215	const u8 *pad0 = page_address(ZERO_PAGE(0));
	216	struct poly1305_desc_ctx poly1305_state;
	217	u32 chacha_state[CHACHA_STATE_WORDS];
	218	struct sg_mapping_iter miter;
	219	size_t partial = 0;
	220	unsigned int flags;
	221	bool ret = true;
	222	int sl;
	223	union {
	224	struct {
	225	u32 k[CHACHA_KEY_WORDS];
	226	__le64 iv[2];
	227	};
	228	u8 block0[POLY1305_KEY_SIZE];
	229	u8 chacha_stream[CHACHA_BLOCK_SIZE];
	230	struct {
	231	u8 mac[2][POLY1305_DIGEST_SIZE];
	232	};
	233	__le64 lens[2];
	234	} b __aligned(16);
	235
c9cc0517 JD	236	if (WARN_ON(src_len > INT_MAX))
	237	return false;
	238
d95312a3 AB	239	chacha_load_key(b.k, key);
	240
	241	b.iv[0] = 0;
	242	b.iv[1] = cpu_to_le64(nonce);
	243
	244	chacha_init(chacha_state, b.k, (u8 *)b.iv);
413808b7	245	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
d95312a3 AB	246	poly1305_init(&poly1305_state, b.block0);
	247
	248	if (unlikely(ad_len)) {
	249	poly1305_update(&poly1305_state, ad, ad_len);
	250	if (ad_len & 0xf)
	251	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	252	}
	253
255f6c2e	254	flags = SG_MITER_TO_SG \| SG_MITER_ATOMIC;
d95312a3 AB	255
	256	sg_miter_start(&miter, src, sg_nents(src), flags);
	257
	258	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
	259	u8 *addr = miter.addr;
	260	size_t length = min_t(size_t, sl, miter.length);
	261
	262	if (!encrypt)
	263	poly1305_update(&poly1305_state, addr, length);
	264
	265	if (unlikely(partial)) {
	266	size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
	267
	268	crypto_xor(addr, b.chacha_stream + partial, l);
	269	partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
	270
	271	addr += l;
	272	length -= l;
	273	}
	274
	275	if (likely(length >= CHACHA_BLOCK_SIZE \|\| length == sl)) {
	276	size_t l = length;
	277
	278	if (unlikely(length < sl))
	279	l &= ~(CHACHA_BLOCK_SIZE - 1);
413808b7	280	chacha20_crypt(chacha_state, addr, addr, l);
d95312a3 AB	281	addr += l;
	282	length -= l;
	283	}
	284
	285	if (unlikely(length > 0)) {
413808b7 EB	286	chacha20_crypt(chacha_state, b.chacha_stream, pad0,
413808b7 EB	287	CHACHA_BLOCK_SIZE);
d95312a3 AB	288	crypto_xor(addr, b.chacha_stream, length);
	289	partial = length;
	290	}
	291
	292	if (encrypt)
	293	poly1305_update(&poly1305_state, miter.addr,
	294	min_t(size_t, sl, miter.length));
	295	}
	296
	297	if (src_len & 0xf)
	298	poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
	299
	300	b.lens[0] = cpu_to_le64(ad_len);
	301	b.lens[1] = cpu_to_le64(src_len);
	302	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	303
	304	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
	305	if (encrypt) {
	306	poly1305_final(&poly1305_state,
	307	miter.addr + miter.length + sl);
	308	ret = true;
	309	} else {
	310	poly1305_final(&poly1305_state, b.mac[0]);
	311	ret = !crypto_memneq(b.mac[0],
	312	miter.addr + miter.length + sl,
	313	POLY1305_DIGEST_SIZE);
	314	}
	315	}
	316
	317	sg_miter_stop(&miter);
	318
	319	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
	320	poly1305_final(&poly1305_state, b.mac[1]);
	321	scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
	322	sizeof(b.mac[1]), encrypt);
	323	ret = encrypt \|\|
	324	!crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
	325	}
	326
	327	memzero_explicit(chacha_state, sizeof(chacha_state));
	328	memzero_explicit(&b, sizeof(b));
	329
	330	return ret;
	331	}
	332
	333	bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
	334	const u8 *ad, const size_t ad_len,
	335	const u64 nonce,
	336	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	337	{
	338	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
	339	nonce, key, 1);
	340	}
	341	EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
	342
	343	bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
	344	const u8 *ad, const size_t ad_len,
	345	const u64 nonce,
	346	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	347	{
	348	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
	349	return false;
	350
	351	return chacha20poly1305_crypt_sg_inplace(src,
352	src_len - POLY1305_DIGEST_SIZE,
353	ad, ad_len, nonce, key, 0);
354	}
355	EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
356
f03a3cab	357	static int __init chacha20poly1305_init(void)
ed20078b AB	358	{
	359	if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
	360	WARN_ON(!chacha20poly1305_selftest()))
	361	return -ENODEV;
	362	return 0;
	363	}
	364
f03a3cab	365	static void __exit chacha20poly1305_exit(void)
ac88c322 JD	366	{
	367	}
	368
f03a3cab RD	369	module_init(chacha20poly1305_init);
f03a3cab RD	370	module_exit(chacha20poly1305_exit);
ed20078b AB	371	MODULE_LICENSE("GPL v2");
	372	MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
	373	MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");