Commit | Line | Data |
---|---|---|
1c08a104 JD |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* | |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. | |
4 | * | |
5 | * This is based in part on Andrew Moon's poly1305-donna, which is in the | |
6 | * public domain. | |
7 | */ | |
8 | ||
9 | #include <linux/kernel.h> | |
10 | #include <asm/unaligned.h> | |
11 | #include <crypto/internal/poly1305.h> | |
12 | ||
8d195e7a AB |
13 | void poly1305_core_setkey(struct poly1305_core_key *key, |
14 | const u8 raw_key[POLY1305_BLOCK_SIZE]) | |
1c08a104 JD |
15 | { |
16 | u64 t0, t1; | |
17 | ||
18 | /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ | |
19 | t0 = get_unaligned_le64(&raw_key[0]); | |
20 | t1 = get_unaligned_le64(&raw_key[8]); | |
21 | ||
22 | key->key.r64[0] = t0 & 0xffc0fffffffULL; | |
23 | key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; | |
24 | key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; | |
25 | ||
26 | /* s = 20*r */ | |
27 | key->precomputed_s.r64[0] = key->key.r64[1] * 20; | |
28 | key->precomputed_s.r64[1] = key->key.r64[2] * 20; | |
29 | } | |
30 | EXPORT_SYMBOL(poly1305_core_setkey); | |
31 | ||
32 | void poly1305_core_blocks(struct poly1305_state *state, | |
33 | const struct poly1305_core_key *key, const void *src, | |
34 | unsigned int nblocks, u32 hibit) | |
35 | { | |
36 | const u8 *input = src; | |
37 | u64 hibit64; | |
38 | u64 r0, r1, r2; | |
39 | u64 s1, s2; | |
40 | u64 h0, h1, h2; | |
41 | u64 c; | |
42 | u128 d0, d1, d2, d; | |
43 | ||
44 | if (!nblocks) | |
45 | return; | |
46 | ||
47 | hibit64 = ((u64)hibit) << 40; | |
48 | ||
49 | r0 = key->key.r64[0]; | |
50 | r1 = key->key.r64[1]; | |
51 | r2 = key->key.r64[2]; | |
52 | ||
53 | h0 = state->h64[0]; | |
54 | h1 = state->h64[1]; | |
55 | h2 = state->h64[2]; | |
56 | ||
57 | s1 = key->precomputed_s.r64[0]; | |
58 | s2 = key->precomputed_s.r64[1]; | |
59 | ||
60 | do { | |
61 | u64 t0, t1; | |
62 | ||
63 | /* h += m[i] */ | |
64 | t0 = get_unaligned_le64(&input[0]); | |
65 | t1 = get_unaligned_le64(&input[8]); | |
66 | ||
67 | h0 += t0 & 0xfffffffffffULL; | |
68 | h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; | |
69 | h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; | |
70 | ||
71 | /* h *= r */ | |
72 | d0 = (u128)h0 * r0; | |
73 | d = (u128)h1 * s2; | |
74 | d0 += d; | |
75 | d = (u128)h2 * s1; | |
76 | d0 += d; | |
77 | d1 = (u128)h0 * r1; | |
78 | d = (u128)h1 * r0; | |
79 | d1 += d; | |
80 | d = (u128)h2 * s2; | |
81 | d1 += d; | |
82 | d2 = (u128)h0 * r2; | |
83 | d = (u128)h1 * r1; | |
84 | d2 += d; | |
85 | d = (u128)h2 * r0; | |
86 | d2 += d; | |
87 | ||
88 | /* (partial) h %= p */ | |
89 | c = (u64)(d0 >> 44); | |
90 | h0 = (u64)d0 & 0xfffffffffffULL; | |
91 | d1 += c; | |
92 | c = (u64)(d1 >> 44); | |
93 | h1 = (u64)d1 & 0xfffffffffffULL; | |
94 | d2 += c; | |
95 | c = (u64)(d2 >> 42); | |
96 | h2 = (u64)d2 & 0x3ffffffffffULL; | |
97 | h0 += c * 5; | |
98 | c = h0 >> 44; | |
99 | h0 = h0 & 0xfffffffffffULL; | |
100 | h1 += c; | |
101 | ||
102 | input += POLY1305_BLOCK_SIZE; | |
103 | } while (--nblocks); | |
104 | ||
105 | state->h64[0] = h0; | |
106 | state->h64[1] = h1; | |
107 | state->h64[2] = h2; | |
108 | } | |
109 | EXPORT_SYMBOL(poly1305_core_blocks); | |
110 | ||
111 | void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], | |
112 | void *dst) | |
113 | { | |
114 | u8 *mac = dst; | |
115 | u64 h0, h1, h2, c; | |
116 | u64 g0, g1, g2; | |
117 | u64 t0, t1; | |
118 | ||
119 | /* fully carry h */ | |
120 | h0 = state->h64[0]; | |
121 | h1 = state->h64[1]; | |
122 | h2 = state->h64[2]; | |
123 | ||
124 | c = h1 >> 44; | |
125 | h1 &= 0xfffffffffffULL; | |
126 | h2 += c; | |
127 | c = h2 >> 42; | |
128 | h2 &= 0x3ffffffffffULL; | |
129 | h0 += c * 5; | |
130 | c = h0 >> 44; | |
131 | h0 &= 0xfffffffffffULL; | |
132 | h1 += c; | |
133 | c = h1 >> 44; | |
134 | h1 &= 0xfffffffffffULL; | |
135 | h2 += c; | |
136 | c = h2 >> 42; | |
137 | h2 &= 0x3ffffffffffULL; | |
138 | h0 += c * 5; | |
139 | c = h0 >> 44; | |
140 | h0 &= 0xfffffffffffULL; | |
141 | h1 += c; | |
142 | ||
143 | /* compute h + -p */ | |
144 | g0 = h0 + 5; | |
145 | c = g0 >> 44; | |
146 | g0 &= 0xfffffffffffULL; | |
147 | g1 = h1 + c; | |
148 | c = g1 >> 44; | |
149 | g1 &= 0xfffffffffffULL; | |
150 | g2 = h2 + c - (1ULL << 42); | |
151 | ||
152 | /* select h if h < p, or h + -p if h >= p */ | |
153 | c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; | |
154 | g0 &= c; | |
155 | g1 &= c; | |
156 | g2 &= c; | |
157 | c = ~c; | |
158 | h0 = (h0 & c) | g0; | |
159 | h1 = (h1 & c) | g1; | |
160 | h2 = (h2 & c) | g2; | |
161 | ||
162 | if (likely(nonce)) { | |
163 | /* h = (h + nonce) */ | |
164 | t0 = ((u64)nonce[1] << 32) | nonce[0]; | |
165 | t1 = ((u64)nonce[3] << 32) | nonce[2]; | |
166 | ||
167 | h0 += t0 & 0xfffffffffffULL; | |
168 | c = h0 >> 44; | |
169 | h0 &= 0xfffffffffffULL; | |
170 | h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; | |
171 | c = h1 >> 44; | |
172 | h1 &= 0xfffffffffffULL; | |
173 | h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; | |
174 | h2 &= 0x3ffffffffffULL; | |
175 | } | |
176 | ||
177 | /* mac = h % (2^128) */ | |
178 | h0 = h0 | (h1 << 44); | |
179 | h1 = (h1 >> 20) | (h2 << 24); | |
180 | ||
181 | put_unaligned_le64(h0, &mac[0]); | |
182 | put_unaligned_le64(h1, &mac[8]); | |
183 | } | |
184 | EXPORT_SYMBOL(poly1305_core_emit); |