Commit | Line | Data |
---|---|---|
ed20078b AB |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* | |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. | |
4 | * | |
5 | * This is an implementation of the ChaCha20Poly1305 AEAD construction. | |
6 | * | |
7 | * Information: https://tools.ietf.org/html/rfc8439 | |
8 | */ | |
9 | ||
10 | #include <crypto/algapi.h> | |
11 | #include <crypto/chacha20poly1305.h> | |
12 | #include <crypto/chacha.h> | |
13 | #include <crypto/poly1305.h> | |
d95312a3 | 14 | #include <crypto/scatterwalk.h> |
ed20078b AB |
15 | |
16 | #include <asm/unaligned.h> | |
17 | #include <linux/kernel.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/mm.h> | |
20 | #include <linux/module.h> | |
21 | ||
22 | #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) | |
23 | ||
ed20078b AB |
24 | static void chacha_load_key(u32 *k, const u8 *in) |
25 | { | |
26 | k[0] = get_unaligned_le32(in); | |
27 | k[1] = get_unaligned_le32(in + 4); | |
28 | k[2] = get_unaligned_le32(in + 8); | |
29 | k[3] = get_unaligned_le32(in + 12); | |
30 | k[4] = get_unaligned_le32(in + 16); | |
31 | k[5] = get_unaligned_le32(in + 20); | |
32 | k[6] = get_unaligned_le32(in + 24); | |
33 | k[7] = get_unaligned_le32(in + 28); | |
34 | } | |
35 | ||
36 | static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) | |
37 | { | |
38 | u32 k[CHACHA_KEY_WORDS]; | |
39 | u8 iv[CHACHA_IV_SIZE]; | |
40 | ||
41 | memset(iv, 0, 8); | |
42 | memcpy(iv + 8, nonce + 16, 8); | |
43 | ||
44 | chacha_load_key(k, key); | |
45 | ||
46 | /* Compute the subkey given the original key and first 128 nonce bits */ | |
47 | chacha_init(chacha_state, k, nonce); | |
48 | hchacha_block(chacha_state, k, 20); | |
49 | ||
50 | chacha_init(chacha_state, k, iv); | |
51 | ||
52 | memzero_explicit(k, sizeof(k)); | |
53 | memzero_explicit(iv, sizeof(iv)); | |
54 | } | |
55 | ||
56 | static void | |
57 | __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
58 | const u8 *ad, const size_t ad_len, u32 *chacha_state) | |
59 | { | |
60 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
61 | struct poly1305_desc_ctx poly1305_state; | |
62 | union { | |
63 | u8 block0[POLY1305_KEY_SIZE]; | |
64 | __le64 lens[2]; | |
65 | } b; | |
66 | ||
413808b7 | 67 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
ed20078b AB |
68 | poly1305_init(&poly1305_state, b.block0); |
69 | ||
70 | poly1305_update(&poly1305_state, ad, ad_len); | |
71 | if (ad_len & 0xf) | |
72 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
73 | ||
413808b7 | 74 | chacha20_crypt(chacha_state, dst, src, src_len); |
ed20078b AB |
75 | |
76 | poly1305_update(&poly1305_state, dst, src_len); | |
77 | if (src_len & 0xf) | |
78 | poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); | |
79 | ||
80 | b.lens[0] = cpu_to_le64(ad_len); | |
81 | b.lens[1] = cpu_to_le64(src_len); | |
82 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
83 | ||
84 | poly1305_final(&poly1305_state, dst + src_len); | |
85 | ||
86 | memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); | |
87 | memzero_explicit(&b, sizeof(b)); | |
88 | } | |
89 | ||
90 | void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
91 | const u8 *ad, const size_t ad_len, | |
92 | const u64 nonce, | |
93 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
94 | { | |
95 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
96 | u32 k[CHACHA_KEY_WORDS]; | |
97 | __le64 iv[2]; | |
98 | ||
99 | chacha_load_key(k, key); | |
100 | ||
101 | iv[0] = 0; | |
102 | iv[1] = cpu_to_le64(nonce); | |
103 | ||
104 | chacha_init(chacha_state, k, (u8 *)iv); | |
105 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); | |
106 | ||
107 | memzero_explicit(iv, sizeof(iv)); | |
108 | memzero_explicit(k, sizeof(k)); | |
109 | } | |
110 | EXPORT_SYMBOL(chacha20poly1305_encrypt); | |
111 | ||
112 | void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
113 | const u8 *ad, const size_t ad_len, | |
114 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], | |
115 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
116 | { | |
117 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
118 | ||
119 | xchacha_init(chacha_state, key, nonce); | |
120 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); | |
121 | } | |
122 | EXPORT_SYMBOL(xchacha20poly1305_encrypt); | |
123 | ||
124 | static bool | |
125 | __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
126 | const u8 *ad, const size_t ad_len, u32 *chacha_state) | |
127 | { | |
128 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
129 | struct poly1305_desc_ctx poly1305_state; | |
130 | size_t dst_len; | |
131 | int ret; | |
132 | union { | |
133 | u8 block0[POLY1305_KEY_SIZE]; | |
134 | u8 mac[POLY1305_DIGEST_SIZE]; | |
135 | __le64 lens[2]; | |
136 | } b; | |
137 | ||
138 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) | |
139 | return false; | |
140 | ||
413808b7 | 141 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
ed20078b AB |
142 | poly1305_init(&poly1305_state, b.block0); |
143 | ||
144 | poly1305_update(&poly1305_state, ad, ad_len); | |
145 | if (ad_len & 0xf) | |
146 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
147 | ||
148 | dst_len = src_len - POLY1305_DIGEST_SIZE; | |
149 | poly1305_update(&poly1305_state, src, dst_len); | |
150 | if (dst_len & 0xf) | |
151 | poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); | |
152 | ||
153 | b.lens[0] = cpu_to_le64(ad_len); | |
154 | b.lens[1] = cpu_to_le64(dst_len); | |
155 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
156 | ||
157 | poly1305_final(&poly1305_state, b.mac); | |
158 | ||
159 | ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); | |
160 | if (likely(!ret)) | |
413808b7 | 161 | chacha20_crypt(chacha_state, dst, src, dst_len); |
ed20078b AB |
162 | |
163 | memzero_explicit(&b, sizeof(b)); | |
164 | ||
165 | return !ret; | |
166 | } | |
167 | ||
168 | bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
169 | const u8 *ad, const size_t ad_len, | |
170 | const u64 nonce, | |
171 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
172 | { | |
173 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
174 | u32 k[CHACHA_KEY_WORDS]; | |
175 | __le64 iv[2]; | |
176 | bool ret; | |
177 | ||
178 | chacha_load_key(k, key); | |
179 | ||
180 | iv[0] = 0; | |
181 | iv[1] = cpu_to_le64(nonce); | |
182 | ||
183 | chacha_init(chacha_state, k, (u8 *)iv); | |
184 | ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, | |
185 | chacha_state); | |
186 | ||
187 | memzero_explicit(chacha_state, sizeof(chacha_state)); | |
188 | memzero_explicit(iv, sizeof(iv)); | |
189 | memzero_explicit(k, sizeof(k)); | |
190 | return ret; | |
191 | } | |
192 | EXPORT_SYMBOL(chacha20poly1305_decrypt); | |
193 | ||
194 | bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
195 | const u8 *ad, const size_t ad_len, | |
196 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], | |
197 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
198 | { | |
199 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
200 | ||
201 | xchacha_init(chacha_state, key, nonce); | |
202 | return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, | |
203 | chacha_state); | |
204 | } | |
205 | EXPORT_SYMBOL(xchacha20poly1305_decrypt); | |
206 | ||
d95312a3 AB |
207 | static |
208 | bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, | |
209 | const size_t src_len, | |
210 | const u8 *ad, const size_t ad_len, | |
211 | const u64 nonce, | |
212 | const u8 key[CHACHA20POLY1305_KEY_SIZE], | |
213 | int encrypt) | |
214 | { | |
215 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
216 | struct poly1305_desc_ctx poly1305_state; | |
217 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
218 | struct sg_mapping_iter miter; | |
219 | size_t partial = 0; | |
220 | unsigned int flags; | |
221 | bool ret = true; | |
222 | int sl; | |
223 | union { | |
224 | struct { | |
225 | u32 k[CHACHA_KEY_WORDS]; | |
226 | __le64 iv[2]; | |
227 | }; | |
228 | u8 block0[POLY1305_KEY_SIZE]; | |
229 | u8 chacha_stream[CHACHA_BLOCK_SIZE]; | |
230 | struct { | |
231 | u8 mac[2][POLY1305_DIGEST_SIZE]; | |
232 | }; | |
233 | __le64 lens[2]; | |
234 | } b __aligned(16); | |
235 | ||
c9cc0517 JD |
236 | if (WARN_ON(src_len > INT_MAX)) |
237 | return false; | |
238 | ||
d95312a3 AB |
239 | chacha_load_key(b.k, key); |
240 | ||
241 | b.iv[0] = 0; | |
242 | b.iv[1] = cpu_to_le64(nonce); | |
243 | ||
244 | chacha_init(chacha_state, b.k, (u8 *)b.iv); | |
413808b7 | 245 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
d95312a3 AB |
246 | poly1305_init(&poly1305_state, b.block0); |
247 | ||
248 | if (unlikely(ad_len)) { | |
249 | poly1305_update(&poly1305_state, ad, ad_len); | |
250 | if (ad_len & 0xf) | |
251 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
252 | } | |
253 | ||
255f6c2e | 254 | flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; |
d95312a3 AB |
255 | |
256 | sg_miter_start(&miter, src, sg_nents(src), flags); | |
257 | ||
258 | for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { | |
259 | u8 *addr = miter.addr; | |
260 | size_t length = min_t(size_t, sl, miter.length); | |
261 | ||
262 | if (!encrypt) | |
263 | poly1305_update(&poly1305_state, addr, length); | |
264 | ||
265 | if (unlikely(partial)) { | |
266 | size_t l = min(length, CHACHA_BLOCK_SIZE - partial); | |
267 | ||
268 | crypto_xor(addr, b.chacha_stream + partial, l); | |
269 | partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); | |
270 | ||
271 | addr += l; | |
272 | length -= l; | |
273 | } | |
274 | ||
275 | if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { | |
276 | size_t l = length; | |
277 | ||
278 | if (unlikely(length < sl)) | |
279 | l &= ~(CHACHA_BLOCK_SIZE - 1); | |
413808b7 | 280 | chacha20_crypt(chacha_state, addr, addr, l); |
d95312a3 AB |
281 | addr += l; |
282 | length -= l; | |
283 | } | |
284 | ||
285 | if (unlikely(length > 0)) { | |
413808b7 EB |
286 | chacha20_crypt(chacha_state, b.chacha_stream, pad0, |
287 | CHACHA_BLOCK_SIZE); | |
d95312a3 AB |
288 | crypto_xor(addr, b.chacha_stream, length); |
289 | partial = length; | |
290 | } | |
291 | ||
292 | if (encrypt) | |
293 | poly1305_update(&poly1305_state, miter.addr, | |
294 | min_t(size_t, sl, miter.length)); | |
295 | } | |
296 | ||
297 | if (src_len & 0xf) | |
298 | poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); | |
299 | ||
300 | b.lens[0] = cpu_to_le64(ad_len); | |
301 | b.lens[1] = cpu_to_le64(src_len); | |
302 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
303 | ||
304 | if (likely(sl <= -POLY1305_DIGEST_SIZE)) { | |
305 | if (encrypt) { | |
306 | poly1305_final(&poly1305_state, | |
307 | miter.addr + miter.length + sl); | |
308 | ret = true; | |
309 | } else { | |
310 | poly1305_final(&poly1305_state, b.mac[0]); | |
311 | ret = !crypto_memneq(b.mac[0], | |
312 | miter.addr + miter.length + sl, | |
313 | POLY1305_DIGEST_SIZE); | |
314 | } | |
315 | } | |
316 | ||
317 | sg_miter_stop(&miter); | |
318 | ||
319 | if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { | |
320 | poly1305_final(&poly1305_state, b.mac[1]); | |
321 | scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, | |
322 | sizeof(b.mac[1]), encrypt); | |
323 | ret = encrypt || | |
324 | !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); | |
325 | } | |
326 | ||
327 | memzero_explicit(chacha_state, sizeof(chacha_state)); | |
328 | memzero_explicit(&b, sizeof(b)); | |
329 | ||
330 | return ret; | |
331 | } | |
332 | ||
333 | bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, | |
334 | const u8 *ad, const size_t ad_len, | |
335 | const u64 nonce, | |
336 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
337 | { | |
338 | return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, | |
339 | nonce, key, 1); | |
340 | } | |
341 | EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); | |
342 | ||
343 | bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, | |
344 | const u8 *ad, const size_t ad_len, | |
345 | const u64 nonce, | |
346 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
347 | { | |
348 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) | |
349 | return false; | |
350 | ||
351 | return chacha20poly1305_crypt_sg_inplace(src, | |
352 | src_len - POLY1305_DIGEST_SIZE, | |
353 | ad, ad_len, nonce, key, 0); | |
354 | } | |
355 | EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); | |
356 | ||
f03a3cab | 357 | static int __init chacha20poly1305_init(void) |
ed20078b AB |
358 | { |
359 | if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && | |
360 | WARN_ON(!chacha20poly1305_selftest())) | |
361 | return -ENODEV; | |
362 | return 0; | |
363 | } | |
364 | ||
f03a3cab | 365 | static void __exit chacha20poly1305_exit(void) |
ac88c322 JD |
366 | { |
367 | } | |
368 | ||
f03a3cab RD |
369 | module_init(chacha20poly1305_init); |
370 | module_exit(chacha20poly1305_exit); | |
ed20078b AB |
371 | MODULE_LICENSE("GPL v2"); |
372 | MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); | |
373 | MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); |