Commit | Line | Data |
---|---|---|
ed20078b AB |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* | |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. | |
4 | * | |
5 | * This is an implementation of the ChaCha20Poly1305 AEAD construction. | |
6 | * | |
7 | * Information: https://tools.ietf.org/html/rfc8439 | |
8 | */ | |
9 | ||
10 | #include <crypto/algapi.h> | |
11 | #include <crypto/chacha20poly1305.h> | |
12 | #include <crypto/chacha.h> | |
13 | #include <crypto/poly1305.h> | |
d95312a3 | 14 | #include <crypto/scatterwalk.h> |
ed20078b AB |
15 | |
16 | #include <asm/unaligned.h> | |
17 | #include <linux/kernel.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/mm.h> | |
20 | #include <linux/module.h> | |
21 | ||
22 | #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) | |
23 | ||
24 | bool __init chacha20poly1305_selftest(void); | |
25 | ||
26 | static void chacha_load_key(u32 *k, const u8 *in) | |
27 | { | |
28 | k[0] = get_unaligned_le32(in); | |
29 | k[1] = get_unaligned_le32(in + 4); | |
30 | k[2] = get_unaligned_le32(in + 8); | |
31 | k[3] = get_unaligned_le32(in + 12); | |
32 | k[4] = get_unaligned_le32(in + 16); | |
33 | k[5] = get_unaligned_le32(in + 20); | |
34 | k[6] = get_unaligned_le32(in + 24); | |
35 | k[7] = get_unaligned_le32(in + 28); | |
36 | } | |
37 | ||
38 | static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) | |
39 | { | |
40 | u32 k[CHACHA_KEY_WORDS]; | |
41 | u8 iv[CHACHA_IV_SIZE]; | |
42 | ||
43 | memset(iv, 0, 8); | |
44 | memcpy(iv + 8, nonce + 16, 8); | |
45 | ||
46 | chacha_load_key(k, key); | |
47 | ||
48 | /* Compute the subkey given the original key and first 128 nonce bits */ | |
49 | chacha_init(chacha_state, k, nonce); | |
50 | hchacha_block(chacha_state, k, 20); | |
51 | ||
52 | chacha_init(chacha_state, k, iv); | |
53 | ||
54 | memzero_explicit(k, sizeof(k)); | |
55 | memzero_explicit(iv, sizeof(iv)); | |
56 | } | |
57 | ||
58 | static void | |
59 | __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
60 | const u8 *ad, const size_t ad_len, u32 *chacha_state) | |
61 | { | |
62 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
63 | struct poly1305_desc_ctx poly1305_state; | |
64 | union { | |
65 | u8 block0[POLY1305_KEY_SIZE]; | |
66 | __le64 lens[2]; | |
67 | } b; | |
68 | ||
413808b7 | 69 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
ed20078b AB |
70 | poly1305_init(&poly1305_state, b.block0); |
71 | ||
72 | poly1305_update(&poly1305_state, ad, ad_len); | |
73 | if (ad_len & 0xf) | |
74 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
75 | ||
413808b7 | 76 | chacha20_crypt(chacha_state, dst, src, src_len); |
ed20078b AB |
77 | |
78 | poly1305_update(&poly1305_state, dst, src_len); | |
79 | if (src_len & 0xf) | |
80 | poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); | |
81 | ||
82 | b.lens[0] = cpu_to_le64(ad_len); | |
83 | b.lens[1] = cpu_to_le64(src_len); | |
84 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
85 | ||
86 | poly1305_final(&poly1305_state, dst + src_len); | |
87 | ||
88 | memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); | |
89 | memzero_explicit(&b, sizeof(b)); | |
90 | } | |
91 | ||
92 | void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
93 | const u8 *ad, const size_t ad_len, | |
94 | const u64 nonce, | |
95 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
96 | { | |
97 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
98 | u32 k[CHACHA_KEY_WORDS]; | |
99 | __le64 iv[2]; | |
100 | ||
101 | chacha_load_key(k, key); | |
102 | ||
103 | iv[0] = 0; | |
104 | iv[1] = cpu_to_le64(nonce); | |
105 | ||
106 | chacha_init(chacha_state, k, (u8 *)iv); | |
107 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); | |
108 | ||
109 | memzero_explicit(iv, sizeof(iv)); | |
110 | memzero_explicit(k, sizeof(k)); | |
111 | } | |
112 | EXPORT_SYMBOL(chacha20poly1305_encrypt); | |
113 | ||
114 | void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, | |
115 | const u8 *ad, const size_t ad_len, | |
116 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], | |
117 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
118 | { | |
119 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
120 | ||
121 | xchacha_init(chacha_state, key, nonce); | |
122 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); | |
123 | } | |
124 | EXPORT_SYMBOL(xchacha20poly1305_encrypt); | |
125 | ||
126 | static bool | |
127 | __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
128 | const u8 *ad, const size_t ad_len, u32 *chacha_state) | |
129 | { | |
130 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
131 | struct poly1305_desc_ctx poly1305_state; | |
132 | size_t dst_len; | |
133 | int ret; | |
134 | union { | |
135 | u8 block0[POLY1305_KEY_SIZE]; | |
136 | u8 mac[POLY1305_DIGEST_SIZE]; | |
137 | __le64 lens[2]; | |
138 | } b; | |
139 | ||
140 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) | |
141 | return false; | |
142 | ||
413808b7 | 143 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
ed20078b AB |
144 | poly1305_init(&poly1305_state, b.block0); |
145 | ||
146 | poly1305_update(&poly1305_state, ad, ad_len); | |
147 | if (ad_len & 0xf) | |
148 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
149 | ||
150 | dst_len = src_len - POLY1305_DIGEST_SIZE; | |
151 | poly1305_update(&poly1305_state, src, dst_len); | |
152 | if (dst_len & 0xf) | |
153 | poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf)); | |
154 | ||
155 | b.lens[0] = cpu_to_le64(ad_len); | |
156 | b.lens[1] = cpu_to_le64(dst_len); | |
157 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
158 | ||
159 | poly1305_final(&poly1305_state, b.mac); | |
160 | ||
161 | ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE); | |
162 | if (likely(!ret)) | |
413808b7 | 163 | chacha20_crypt(chacha_state, dst, src, dst_len); |
ed20078b AB |
164 | |
165 | memzero_explicit(&b, sizeof(b)); | |
166 | ||
167 | return !ret; | |
168 | } | |
169 | ||
170 | bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
171 | const u8 *ad, const size_t ad_len, | |
172 | const u64 nonce, | |
173 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
174 | { | |
175 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
176 | u32 k[CHACHA_KEY_WORDS]; | |
177 | __le64 iv[2]; | |
178 | bool ret; | |
179 | ||
180 | chacha_load_key(k, key); | |
181 | ||
182 | iv[0] = 0; | |
183 | iv[1] = cpu_to_le64(nonce); | |
184 | ||
185 | chacha_init(chacha_state, k, (u8 *)iv); | |
186 | ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, | |
187 | chacha_state); | |
188 | ||
189 | memzero_explicit(chacha_state, sizeof(chacha_state)); | |
190 | memzero_explicit(iv, sizeof(iv)); | |
191 | memzero_explicit(k, sizeof(k)); | |
192 | return ret; | |
193 | } | |
194 | EXPORT_SYMBOL(chacha20poly1305_decrypt); | |
195 | ||
196 | bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, | |
197 | const u8 *ad, const size_t ad_len, | |
198 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], | |
199 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
200 | { | |
201 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
202 | ||
203 | xchacha_init(chacha_state, key, nonce); | |
204 | return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, | |
205 | chacha_state); | |
206 | } | |
207 | EXPORT_SYMBOL(xchacha20poly1305_decrypt); | |
208 | ||
d95312a3 AB |
209 | static |
210 | bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, | |
211 | const size_t src_len, | |
212 | const u8 *ad, const size_t ad_len, | |
213 | const u64 nonce, | |
214 | const u8 key[CHACHA20POLY1305_KEY_SIZE], | |
215 | int encrypt) | |
216 | { | |
217 | const u8 *pad0 = page_address(ZERO_PAGE(0)); | |
218 | struct poly1305_desc_ctx poly1305_state; | |
219 | u32 chacha_state[CHACHA_STATE_WORDS]; | |
220 | struct sg_mapping_iter miter; | |
221 | size_t partial = 0; | |
222 | unsigned int flags; | |
223 | bool ret = true; | |
224 | int sl; | |
225 | union { | |
226 | struct { | |
227 | u32 k[CHACHA_KEY_WORDS]; | |
228 | __le64 iv[2]; | |
229 | }; | |
230 | u8 block0[POLY1305_KEY_SIZE]; | |
231 | u8 chacha_stream[CHACHA_BLOCK_SIZE]; | |
232 | struct { | |
233 | u8 mac[2][POLY1305_DIGEST_SIZE]; | |
234 | }; | |
235 | __le64 lens[2]; | |
236 | } b __aligned(16); | |
237 | ||
238 | chacha_load_key(b.k, key); | |
239 | ||
240 | b.iv[0] = 0; | |
241 | b.iv[1] = cpu_to_le64(nonce); | |
242 | ||
243 | chacha_init(chacha_state, b.k, (u8 *)b.iv); | |
413808b7 | 244 | chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0)); |
d95312a3 AB |
245 | poly1305_init(&poly1305_state, b.block0); |
246 | ||
247 | if (unlikely(ad_len)) { | |
248 | poly1305_update(&poly1305_state, ad, ad_len); | |
249 | if (ad_len & 0xf) | |
250 | poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf)); | |
251 | } | |
252 | ||
253 | flags = SG_MITER_TO_SG; | |
254 | if (!preemptible()) | |
255 | flags |= SG_MITER_ATOMIC; | |
256 | ||
257 | sg_miter_start(&miter, src, sg_nents(src), flags); | |
258 | ||
259 | for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { | |
260 | u8 *addr = miter.addr; | |
261 | size_t length = min_t(size_t, sl, miter.length); | |
262 | ||
263 | if (!encrypt) | |
264 | poly1305_update(&poly1305_state, addr, length); | |
265 | ||
266 | if (unlikely(partial)) { | |
267 | size_t l = min(length, CHACHA_BLOCK_SIZE - partial); | |
268 | ||
269 | crypto_xor(addr, b.chacha_stream + partial, l); | |
270 | partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); | |
271 | ||
272 | addr += l; | |
273 | length -= l; | |
274 | } | |
275 | ||
276 | if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { | |
277 | size_t l = length; | |
278 | ||
279 | if (unlikely(length < sl)) | |
280 | l &= ~(CHACHA_BLOCK_SIZE - 1); | |
413808b7 | 281 | chacha20_crypt(chacha_state, addr, addr, l); |
d95312a3 AB |
282 | addr += l; |
283 | length -= l; | |
284 | } | |
285 | ||
286 | if (unlikely(length > 0)) { | |
413808b7 EB |
287 | chacha20_crypt(chacha_state, b.chacha_stream, pad0, |
288 | CHACHA_BLOCK_SIZE); | |
d95312a3 AB |
289 | crypto_xor(addr, b.chacha_stream, length); |
290 | partial = length; | |
291 | } | |
292 | ||
293 | if (encrypt) | |
294 | poly1305_update(&poly1305_state, miter.addr, | |
295 | min_t(size_t, sl, miter.length)); | |
296 | } | |
297 | ||
298 | if (src_len & 0xf) | |
299 | poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf)); | |
300 | ||
301 | b.lens[0] = cpu_to_le64(ad_len); | |
302 | b.lens[1] = cpu_to_le64(src_len); | |
303 | poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); | |
304 | ||
305 | if (likely(sl <= -POLY1305_DIGEST_SIZE)) { | |
306 | if (encrypt) { | |
307 | poly1305_final(&poly1305_state, | |
308 | miter.addr + miter.length + sl); | |
309 | ret = true; | |
310 | } else { | |
311 | poly1305_final(&poly1305_state, b.mac[0]); | |
312 | ret = !crypto_memneq(b.mac[0], | |
313 | miter.addr + miter.length + sl, | |
314 | POLY1305_DIGEST_SIZE); | |
315 | } | |
316 | } | |
317 | ||
318 | sg_miter_stop(&miter); | |
319 | ||
320 | if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { | |
321 | poly1305_final(&poly1305_state, b.mac[1]); | |
322 | scatterwalk_map_and_copy(b.mac[encrypt], src, src_len, | |
323 | sizeof(b.mac[1]), encrypt); | |
324 | ret = encrypt || | |
325 | !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE); | |
326 | } | |
327 | ||
328 | memzero_explicit(chacha_state, sizeof(chacha_state)); | |
329 | memzero_explicit(&b, sizeof(b)); | |
330 | ||
331 | return ret; | |
332 | } | |
333 | ||
334 | bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, | |
335 | const u8 *ad, const size_t ad_len, | |
336 | const u64 nonce, | |
337 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
338 | { | |
339 | return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, | |
340 | nonce, key, 1); | |
341 | } | |
342 | EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); | |
343 | ||
344 | bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, | |
345 | const u8 *ad, const size_t ad_len, | |
346 | const u64 nonce, | |
347 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) | |
348 | { | |
349 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) | |
350 | return false; | |
351 | ||
352 | return chacha20poly1305_crypt_sg_inplace(src, | |
353 | src_len - POLY1305_DIGEST_SIZE, | |
354 | ad, ad_len, nonce, key, 0); | |
355 | } | |
356 | EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); | |
357 | ||
ed20078b AB |
358 | static int __init mod_init(void) |
359 | { | |
360 | if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && | |
361 | WARN_ON(!chacha20poly1305_selftest())) | |
362 | return -ENODEV; | |
363 | return 0; | |
364 | } | |
365 | ||
366 | module_init(mod_init); | |
367 | MODULE_LICENSE("GPL v2"); | |
368 | MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); | |
369 | MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); |