[linux-block.git] / lib / crypto / chacha20poly1305.c

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * This is an implementation of the ChaCha20Poly1305 AEAD construction.
 *
 * Information: https://tools.ietf.org/html/rfc8439
 */

#include <crypto/algapi.h>
#include <crypto/chacha20poly1305.h>
#include <crypto/chacha.h>
#include <crypto/poly1305.h>
#include <crypto/scatterwalk.h>

#include <asm/unaligned.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/module.h>

#define CHACHA_KEY_WORDS	(CHACHA_KEY_SIZE / sizeof(u32))

bool __init chacha20poly1305_selftest(void);

static void chacha_load_key(u32 *k, const u8 *in)
{
	k[0] = get_unaligned_le32(in);
	k[1] = get_unaligned_le32(in + 4);
	k[2] = get_unaligned_le32(in + 8);
	k[3] = get_unaligned_le32(in + 12);
	k[4] = get_unaligned_le32(in + 16);
	k[5] = get_unaligned_le32(in + 20);
	k[6] = get_unaligned_le32(in + 24);
	k[7] = get_unaligned_le32(in + 28);
}

static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
{
	u32 k[CHACHA_KEY_WORDS];
	u8 iv[CHACHA_IV_SIZE];

	memset(iv, 0, 8);
	memcpy(iv + 8, nonce + 16, 8);

	chacha_load_key(k, key);

	/* Compute the subkey given the original key and first 128 nonce bits */
	chacha_init(chacha_state, k, nonce);
	hchacha_block(chacha_state, k, 20);

	chacha_init(chacha_state, k, iv);

	memzero_explicit(k, sizeof(k));
	memzero_explicit(iv, sizeof(iv));
}

static void
__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	union {
		u8 block0[POLY1305_KEY_SIZE];
		__le64 lens[2];
	} b;

	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	poly1305_update(&poly1305_state, ad, ad_len);
	if (ad_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));

	chacha20_crypt(chacha_state, dst, src, src_len);

	poly1305_update(&poly1305_state, dst, src_len);
	if (src_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(src_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	poly1305_final(&poly1305_state, dst + src_len);

	memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
	memzero_explicit(&b, sizeof(b));
}

void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			      const u8 *ad, const size_t ad_len,
			      const u64 nonce,
			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];
	u32 k[CHACHA_KEY_WORDS];
	__le64 iv[2];

	chacha_load_key(k, key);

	iv[0] = 0;
	iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, k, (u8 *)iv);
	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);

	memzero_explicit(iv, sizeof(iv));
	memzero_explicit(k, sizeof(k));
}
EXPORT_SYMBOL(chacha20poly1305_encrypt);

void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
			       const u8 *ad, const size_t ad_len,
			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];

	xchacha_init(chacha_state, key, nonce);
	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_encrypt);

static bool
__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	size_t dst_len;
	int ret;
	union {
		u8 block0[POLY1305_KEY_SIZE];
		u8 mac[POLY1305_DIGEST_SIZE];
		__le64 lens[2];
	} b;

	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
		return false;

	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	poly1305_update(&poly1305_state, ad, ad_len);
	if (ad_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));

	dst_len = src_len - POLY1305_DIGEST_SIZE;
	poly1305_update(&poly1305_state, src, dst_len);
	if (dst_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(dst_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	poly1305_final(&poly1305_state, b.mac);

	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
	if (likely(!ret))
		chacha20_crypt(chacha_state, dst, src, dst_len);

	memzero_explicit(&b, sizeof(b));

	return !ret;
}

bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			      const u8 *ad, const size_t ad_len,
			      const u64 nonce,
			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];
	u32 k[CHACHA_KEY_WORDS];
	__le64 iv[2];
	bool ret;

	chacha_load_key(k, key);

	iv[0] = 0;
	iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, k, (u8 *)iv);
	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
					 chacha_state);

	memzero_explicit(chacha_state, sizeof(chacha_state));
	memzero_explicit(iv, sizeof(iv));
	memzero_explicit(k, sizeof(k));
	return ret;
}
EXPORT_SYMBOL(chacha20poly1305_decrypt);

bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
			       const u8 *ad, const size_t ad_len,
			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	u32 chacha_state[CHACHA_STATE_WORDS];

	xchacha_init(chacha_state, key, nonce);
	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
					  chacha_state);
}
EXPORT_SYMBOL(xchacha20poly1305_decrypt);

static
bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
				       const size_t src_len,
				       const u8 *ad, const size_t ad_len,
				       const u64 nonce,
				       const u8 key[CHACHA20POLY1305_KEY_SIZE],
				       int encrypt)
{
	const u8 *pad0 = page_address(ZERO_PAGE(0));
	struct poly1305_desc_ctx poly1305_state;
	u32 chacha_state[CHACHA_STATE_WORDS];
	struct sg_mapping_iter miter;
	size_t partial = 0;
	unsigned int flags;
	bool ret = true;
	int sl;
	union {
		struct {
			u32 k[CHACHA_KEY_WORDS];
			__le64 iv[2];
		};
		u8 block0[POLY1305_KEY_SIZE];
		u8 chacha_stream[CHACHA_BLOCK_SIZE];
		struct {
			u8 mac[2][POLY1305_DIGEST_SIZE];
		};
		__le64 lens[2];
	} b __aligned(16);

	if (WARN_ON(src_len > INT_MAX))
		return false;

	chacha_load_key(b.k, key);

	b.iv[0] = 0;
	b.iv[1] = cpu_to_le64(nonce);

	chacha_init(chacha_state, b.k, (u8 *)b.iv);
	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
	poly1305_init(&poly1305_state, b.block0);

	if (unlikely(ad_len)) {
		poly1305_update(&poly1305_state, ad, ad_len);
		if (ad_len & 0xf)
			poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	}

	flags = SG_MITER_TO_SG;
	if (!preemptible())
		flags |= SG_MITER_ATOMIC;

	sg_miter_start(&miter, src, sg_nents(src), flags);

	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
		u8 *addr = miter.addr;
		size_t length = min_t(size_t, sl, miter.length);

		if (!encrypt)
			poly1305_update(&poly1305_state, addr, length);

		if (unlikely(partial)) {
			size_t l = min(length, CHACHA_BLOCK_SIZE - partial);

			crypto_xor(addr, b.chacha_stream + partial, l);
			partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);

			addr += l;
			length -= l;
		}

		if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
			size_t l = length;

			if (unlikely(length < sl))
				l &= ~(CHACHA_BLOCK_SIZE - 1);
			chacha20_crypt(chacha_state, addr, addr, l);
			addr += l;
			length -= l;
		}

		if (unlikely(length > 0)) {
			chacha20_crypt(chacha_state, b.chacha_stream, pad0,
				       CHACHA_BLOCK_SIZE);
			crypto_xor(addr, b.chacha_stream, length);
			partial = length;
		}

		if (encrypt)
			poly1305_update(&poly1305_state, miter.addr,
					min_t(size_t, sl, miter.length));
	}

	if (src_len & 0xf)
		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));

	b.lens[0] = cpu_to_le64(ad_len);
	b.lens[1] = cpu_to_le64(src_len);
	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));

	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
		if (encrypt) {
			poly1305_final(&poly1305_state,
				       miter.addr + miter.length + sl);
			ret = true;
		} else {
			poly1305_final(&poly1305_state, b.mac[0]);
			ret = !crypto_memneq(b.mac[0],
					     miter.addr + miter.length + sl,
					     POLY1305_DIGEST_SIZE);
		}
	}

	sg_miter_stop(&miter);

	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
		poly1305_final(&poly1305_state, b.mac[1]);
		scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
					 sizeof(b.mac[1]), encrypt);
		ret = encrypt ||
		      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
	}

	memzero_explicit(chacha_state, sizeof(chacha_state));
	memzero_explicit(&b, sizeof(b));

	return ret;
}

bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
					 const u8 *ad, const size_t ad_len,
					 const u64 nonce,
					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
						 nonce, key, 1);
}
EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);

bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
					 const u8 *ad, const size_t ad_len,
					 const u64 nonce,
					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
{
	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
		return false;

	return chacha20poly1305_crypt_sg_inplace(src,
						 src_len - POLY1305_DIGEST_SIZE,
						 ad, ad_len, nonce, key, 0);
}
EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);

static int __init mod_init(void)
{
	if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
	    WARN_ON(!chacha20poly1305_selftest()))
		return -ENODEV;
	return 0;
}

module_init(mod_init);
MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
Commit	Line	Data
ed20078b AB	1	// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	/*
	3	* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	4	*
	5	* This is an implementation of the ChaCha20Poly1305 AEAD construction.
	6	*
	7	* Information: https://tools.ietf.org/html/rfc8439
	8	*/
	9
	10	#include <crypto/algapi.h>
	11	#include <crypto/chacha20poly1305.h>
	12	#include <crypto/chacha.h>
	13	#include <crypto/poly1305.h>
d95312a3	14	#include <crypto/scatterwalk.h>
ed20078b AB	15
	16	#include <asm/unaligned.h>
	17	#include <linux/kernel.h>
	18	#include <linux/init.h>
	19	#include <linux/mm.h>
	20	#include <linux/module.h>
	21
	22	#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))
	23
	24	bool __init chacha20poly1305_selftest(void);
	25
	26	static void chacha_load_key(u32 k, const u8 in)
	27	{
	28	k[0] = get_unaligned_le32(in);
	29	k[1] = get_unaligned_le32(in + 4);
	30	k[2] = get_unaligned_le32(in + 8);
	31	k[3] = get_unaligned_le32(in + 12);
	32	k[4] = get_unaligned_le32(in + 16);
	33	k[5] = get_unaligned_le32(in + 20);
	34	k[6] = get_unaligned_le32(in + 24);
	35	k[7] = get_unaligned_le32(in + 28);
	36	}
	37
	38	static void xchacha_init(u32 chacha_state, const u8 key, const u8 *nonce)
	39	{
	40	u32 k[CHACHA_KEY_WORDS];
	41	u8 iv[CHACHA_IV_SIZE];
	42
	43	memset(iv, 0, 8);
	44	memcpy(iv + 8, nonce + 16, 8);
	45
	46	chacha_load_key(k, key);
	47
	48	/* Compute the subkey given the original key and first 128 nonce bits */
	49	chacha_init(chacha_state, k, nonce);
	50	hchacha_block(chacha_state, k, 20);
	51
	52	chacha_init(chacha_state, k, iv);
	53
	54	memzero_explicit(k, sizeof(k));
	55	memzero_explicit(iv, sizeof(iv));
	56	}
	57
	58	static void
	59	__chacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	60	const u8 ad, const size_t ad_len, u32 chacha_state)
	61	{
	62	const u8 *pad0 = page_address(ZERO_PAGE(0));
	63	struct poly1305_desc_ctx poly1305_state;
	64	union {
	65	u8 block0[POLY1305_KEY_SIZE];
	66	__le64 lens[2];
	67	} b;
	68
413808b7	69	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
ed20078b AB	70	poly1305_init(&poly1305_state, b.block0);
	71
	72	poly1305_update(&poly1305_state, ad, ad_len);
	73	if (ad_len & 0xf)
	74	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	75
413808b7	76	chacha20_crypt(chacha_state, dst, src, src_len);
ed20078b AB	77
	78	poly1305_update(&poly1305_state, dst, src_len);
	79	if (src_len & 0xf)
	80	poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
	81
	82	b.lens[0] = cpu_to_le64(ad_len);
	83	b.lens[1] = cpu_to_le64(src_len);
	84	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	85
	86	poly1305_final(&poly1305_state, dst + src_len);
	87
	88	memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
	89	memzero_explicit(&b, sizeof(b));
	90	}
	91
	92	void chacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	93	const u8 *ad, const size_t ad_len,
	94	const u64 nonce,
	95	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	96	{
	97	u32 chacha_state[CHACHA_STATE_WORDS];
	98	u32 k[CHACHA_KEY_WORDS];
	99	__le64 iv[2];
	100
	101	chacha_load_key(k, key);
	102
	103	iv[0] = 0;
	104	iv[1] = cpu_to_le64(nonce);
	105
	106	chacha_init(chacha_state, k, (u8 *)iv);
	107	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
	108
	109	memzero_explicit(iv, sizeof(iv));
	110	memzero_explicit(k, sizeof(k));
	111	}
	112	EXPORT_SYMBOL(chacha20poly1305_encrypt);
	113
	114	void xchacha20poly1305_encrypt(u8 dst, const u8 src, const size_t src_len,
	115	const u8 *ad, const size_t ad_len,
	116	const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
	117	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	118	{
	119	u32 chacha_state[CHACHA_STATE_WORDS];
	120
	121	xchacha_init(chacha_state, key, nonce);
	122	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
	123	}
	124	EXPORT_SYMBOL(xchacha20poly1305_encrypt);
	125
	126	static bool
	127	__chacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	128	const u8 ad, const size_t ad_len, u32 chacha_state)
	129	{
	130	const u8 *pad0 = page_address(ZERO_PAGE(0));
	131	struct poly1305_desc_ctx poly1305_state;
	132	size_t dst_len;
	133	int ret;
	134	union {
	135	u8 block0[POLY1305_KEY_SIZE];
	136	u8 mac[POLY1305_DIGEST_SIZE];
	137	__le64 lens[2];
	138	} b;
	139
	140	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
141	return false;
142
413808b7	143	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
ed20078b AB	144	poly1305_init(&poly1305_state, b.block0);
	145
	146	poly1305_update(&poly1305_state, ad, ad_len);
	147	if (ad_len & 0xf)
	148	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	149
	150	dst_len = src_len - POLY1305_DIGEST_SIZE;
	151	poly1305_update(&poly1305_state, src, dst_len);
	152	if (dst_len & 0xf)
	153	poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
	154
	155	b.lens[0] = cpu_to_le64(ad_len);
	156	b.lens[1] = cpu_to_le64(dst_len);
	157	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	158
	159	poly1305_final(&poly1305_state, b.mac);
	160
	161	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
	162	if (likely(!ret))
413808b7	163	chacha20_crypt(chacha_state, dst, src, dst_len);
ed20078b AB	164
	165	memzero_explicit(&b, sizeof(b));
	166
	167	return !ret;
	168	}
	169
	170	bool chacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	171	const u8 *ad, const size_t ad_len,
	172	const u64 nonce,
	173	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	174	{
	175	u32 chacha_state[CHACHA_STATE_WORDS];
	176	u32 k[CHACHA_KEY_WORDS];
	177	__le64 iv[2];
	178	bool ret;
	179
	180	chacha_load_key(k, key);
	181
	182	iv[0] = 0;
	183	iv[1] = cpu_to_le64(nonce);
	184
	185	chacha_init(chacha_state, k, (u8 *)iv);
	186	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
	187	chacha_state);
	188
	189	memzero_explicit(chacha_state, sizeof(chacha_state));
	190	memzero_explicit(iv, sizeof(iv));
	191	memzero_explicit(k, sizeof(k));
	192	return ret;
	193	}
	194	EXPORT_SYMBOL(chacha20poly1305_decrypt);
	195
	196	bool xchacha20poly1305_decrypt(u8 dst, const u8 src, const size_t src_len,
	197	const u8 *ad, const size_t ad_len,
	198	const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
	199	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	200	{
	201	u32 chacha_state[CHACHA_STATE_WORDS];
	202
	203	xchacha_init(chacha_state, key, nonce);
	204	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
	205	chacha_state);
	206	}
	207	EXPORT_SYMBOL(xchacha20poly1305_decrypt);
	208
d95312a3 AB	209	static
	210	bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
	211	const size_t src_len,
	212	const u8 *ad, const size_t ad_len,
	213	const u64 nonce,
	214	const u8 key[CHACHA20POLY1305_KEY_SIZE],
	215	int encrypt)
	216	{
	217	const u8 *pad0 = page_address(ZERO_PAGE(0));
	218	struct poly1305_desc_ctx poly1305_state;
	219	u32 chacha_state[CHACHA_STATE_WORDS];
	220	struct sg_mapping_iter miter;
	221	size_t partial = 0;
	222	unsigned int flags;
	223	bool ret = true;
	224	int sl;
	225	union {
	226	struct {
	227	u32 k[CHACHA_KEY_WORDS];
	228	__le64 iv[2];
	229	};
	230	u8 block0[POLY1305_KEY_SIZE];
	231	u8 chacha_stream[CHACHA_BLOCK_SIZE];
	232	struct {
	233	u8 mac[2][POLY1305_DIGEST_SIZE];
	234	};
	235	__le64 lens[2];
	236	} b __aligned(16);
	237
c9cc0517 JD	238	if (WARN_ON(src_len > INT_MAX))
	239	return false;
	240
d95312a3 AB	241	chacha_load_key(b.k, key);
	242
	243	b.iv[0] = 0;
	244	b.iv[1] = cpu_to_le64(nonce);
	245
	246	chacha_init(chacha_state, b.k, (u8 *)b.iv);
413808b7	247	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
d95312a3 AB	248	poly1305_init(&poly1305_state, b.block0);
	249
	250	if (unlikely(ad_len)) {
	251	poly1305_update(&poly1305_state, ad, ad_len);
	252	if (ad_len & 0xf)
	253	poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
	254	}
	255
	256	flags = SG_MITER_TO_SG;
	257	if (!preemptible())
	258	flags \|= SG_MITER_ATOMIC;
	259
	260	sg_miter_start(&miter, src, sg_nents(src), flags);
	261
	262	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
	263	u8 *addr = miter.addr;
	264	size_t length = min_t(size_t, sl, miter.length);
	265
	266	if (!encrypt)
	267	poly1305_update(&poly1305_state, addr, length);
	268
	269	if (unlikely(partial)) {
	270	size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
	271
	272	crypto_xor(addr, b.chacha_stream + partial, l);
	273	partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
	274
	275	addr += l;
	276	length -= l;
	277	}
	278
	279	if (likely(length >= CHACHA_BLOCK_SIZE \|\| length == sl)) {
	280	size_t l = length;
	281
	282	if (unlikely(length < sl))
	283	l &= ~(CHACHA_BLOCK_SIZE - 1);
413808b7	284	chacha20_crypt(chacha_state, addr, addr, l);
d95312a3 AB	285	addr += l;
	286	length -= l;
	287	}
	288
	289	if (unlikely(length > 0)) {
413808b7 EB	290	chacha20_crypt(chacha_state, b.chacha_stream, pad0,
413808b7 EB	291	CHACHA_BLOCK_SIZE);
d95312a3 AB	292	crypto_xor(addr, b.chacha_stream, length);
	293	partial = length;
	294	}
	295
	296	if (encrypt)
	297	poly1305_update(&poly1305_state, miter.addr,
	298	min_t(size_t, sl, miter.length));
	299	}
	300
	301	if (src_len & 0xf)
	302	poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
	303
	304	b.lens[0] = cpu_to_le64(ad_len);
	305	b.lens[1] = cpu_to_le64(src_len);
	306	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
	307
	308	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
	309	if (encrypt) {
	310	poly1305_final(&poly1305_state,
	311	miter.addr + miter.length + sl);
	312	ret = true;
	313	} else {
	314	poly1305_final(&poly1305_state, b.mac[0]);
	315	ret = !crypto_memneq(b.mac[0],
	316	miter.addr + miter.length + sl,
	317	POLY1305_DIGEST_SIZE);
	318	}
	319	}
	320
	321	sg_miter_stop(&miter);
	322
	323	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
	324	poly1305_final(&poly1305_state, b.mac[1]);
	325	scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
	326	sizeof(b.mac[1]), encrypt);
	327	ret = encrypt \|\|
	328	!crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
	329	}
	330
	331	memzero_explicit(chacha_state, sizeof(chacha_state));
	332	memzero_explicit(&b, sizeof(b));
	333
	334	return ret;
	335	}
	336
	337	bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
	338	const u8 *ad, const size_t ad_len,
	339	const u64 nonce,
	340	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	341	{
	342	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
	343	nonce, key, 1);
	344	}
	345	EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
	346
	347	bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
	348	const u8 *ad, const size_t ad_len,
	349	const u64 nonce,
	350	const u8 key[CHACHA20POLY1305_KEY_SIZE])
	351	{
	352	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
	353	return false;
	354
	355	return chacha20poly1305_crypt_sg_inplace(src,
356	src_len - POLY1305_DIGEST_SIZE,
357	ad, ad_len, nonce, key, 0);
358	}
359	EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
360
ed20078b AB	361	static int __init mod_init(void)
	362	{
	363	if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
	364	WARN_ON(!chacha20poly1305_selftest()))
	365	return -ENODEV;
	366	return 0;
	367	}
	368
	369	module_init(mod_init);
	370	MODULE_LICENSE("GPL v2");
	371	MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
	372	MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");