Commit | Line | Data |
---|---|---|
58790314 | 1 | /* |
84c61fe1 | 2 | * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> |
58790314 JK |
3 | * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. |
4 | * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. | |
5 | * | |
6 | * This software is available to you under a choice of one of two | |
7 | * licenses. You may choose to be licensed under the terms of the GNU | |
8 | * General Public License (GPL) Version 2, available from the file | |
9 | * COPYING in the main directory of this source tree, or the | |
10 | * OpenIB.org BSD license below: | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or | |
13 | * without modification, are permitted provided that the following | |
14 | * conditions are met: | |
15 | * | |
16 | * - Redistributions of source code must retain the above | |
17 | * copyright notice, this list of conditions and the following | |
18 | * disclaimer. | |
19 | * | |
20 | * - Redistributions in binary form must reproduce the above | |
21 | * copyright notice, this list of conditions and the following | |
22 | * disclaimer in the documentation and/or other materials | |
23 | * provided with the distribution. | |
24 | * | |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
32 | * SOFTWARE. | |
33 | */ | |
34 | ||
35 | #ifndef _TLS_INT_H | |
36 | #define _TLS_INT_H | |
37 | ||
38 | #include <asm/byteorder.h> | |
39 | #include <linux/types.h> | |
40 | #include <linux/skmsg.h> | |
41 | #include <net/tls.h> | |
6a7eccef | 42 | #include <net/tls_prot.h> |
58790314 | 43 | |
fd31f399 JK |
44 | #define TLS_PAGE_ORDER (min_t(unsigned int, PAGE_ALLOC_COSTLY_ORDER, \ |
45 | TLS_MAX_PAYLOAD_SIZE >> PAGE_SHIFT)) | |
46 | ||
58790314 JK |
47 | #define __TLS_INC_STATS(net, field) \ |
48 | __SNMP_INC_STATS((net)->mib.tls_statistics, field) | |
49 | #define TLS_INC_STATS(net, field) \ | |
50 | SNMP_INC_STATS((net)->mib.tls_statistics, field) | |
51 | #define TLS_DEC_STATS(net, field) \ | |
52 | SNMP_DEC_STATS((net)->mib.tls_statistics, field) | |
53 | ||
8db44ab2 | 54 | struct tls_cipher_desc { |
176a3f50 | 55 | unsigned int nonce; |
fd0fc6fd SD |
56 | unsigned int iv; |
57 | unsigned int key; | |
58 | unsigned int salt; | |
59 | unsigned int tag; | |
60 | unsigned int rec_seq; | |
176a3f50 SD |
61 | unsigned int iv_offset; |
62 | unsigned int key_offset; | |
63 | unsigned int salt_offset; | |
64 | unsigned int rec_seq_offset; | |
65 | char *cipher_name; | |
66 | bool offloadable; | |
67 | size_t crypto_info; | |
fd0fc6fd SD |
68 | }; |
69 | ||
037303d6 SD |
70 | #define TLS_CIPHER_MIN TLS_CIPHER_AES_GCM_128 |
71 | #define TLS_CIPHER_MAX TLS_CIPHER_ARIA_GCM_256 | |
8db44ab2 | 72 | extern const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN]; |
037303d6 | 73 | |
8db44ab2 | 74 | static inline const struct tls_cipher_desc *get_cipher_desc(u16 cipher_type) |
037303d6 SD |
75 | { |
76 | if (cipher_type < TLS_CIPHER_MIN || cipher_type > TLS_CIPHER_MAX) | |
77 | return NULL; | |
78 | ||
8db44ab2 | 79 | return &tls_cipher_desc[cipher_type - TLS_CIPHER_MIN]; |
037303d6 SD |
80 | } |
81 | ||
176a3f50 SD |
82 | static inline char *crypto_info_iv(struct tls_crypto_info *crypto_info, |
83 | const struct tls_cipher_desc *cipher_desc) | |
84 | { | |
85 | return (char *)crypto_info + cipher_desc->iv_offset; | |
86 | } | |
87 | ||
88 | static inline char *crypto_info_key(struct tls_crypto_info *crypto_info, | |
89 | const struct tls_cipher_desc *cipher_desc) | |
90 | { | |
91 | return (char *)crypto_info + cipher_desc->key_offset; | |
92 | } | |
93 | ||
94 | static inline char *crypto_info_salt(struct tls_crypto_info *crypto_info, | |
95 | const struct tls_cipher_desc *cipher_desc) | |
96 | { | |
97 | return (char *)crypto_info + cipher_desc->salt_offset; | |
98 | } | |
99 | ||
100 | static inline char *crypto_info_rec_seq(struct tls_crypto_info *crypto_info, | |
101 | const struct tls_cipher_desc *cipher_desc) | |
102 | { | |
103 | return (char *)crypto_info + cipher_desc->rec_seq_offset; | |
104 | } | |
105 | ||
fd0fc6fd | 106 | |
58790314 JK |
107 | /* TLS records are maintained in 'struct tls_rec'. It stores the memory pages |
108 | * allocated or mapped for each TLS record. After encryption, the records are | |
109 | * stores in a linked list. | |
110 | */ | |
111 | struct tls_rec { | |
112 | struct list_head list; | |
113 | int tx_ready; | |
114 | int tx_flags; | |
115 | ||
116 | struct sk_msg msg_plaintext; | |
117 | struct sk_msg msg_encrypted; | |
118 | ||
119 | /* AAD | msg_plaintext.sg.data | sg_tag */ | |
120 | struct scatterlist sg_aead_in[2]; | |
121 | /* AAD | msg_encrypted.sg.data (data contains overhead for hdr & iv & tag) */ | |
122 | struct scatterlist sg_aead_out[2]; | |
123 | ||
124 | char content_type; | |
125 | struct scatterlist sg_content_type; | |
126 | ||
8d338c76 HX |
127 | struct sock *sk; |
128 | ||
58790314 | 129 | char aad_space[TLS_AAD_SPACE_SIZE]; |
bee6b7b3 | 130 | u8 iv_data[TLS_MAX_IV_SIZE]; |
58790314 JK |
131 | struct aead_request aead_req; |
132 | u8 aead_req_ctx[]; | |
133 | }; | |
134 | ||
135 | int __net_init tls_proc_init(struct net *net); | |
136 | void __net_exit tls_proc_fini(struct net *net); | |
137 | ||
138 | struct tls_context *tls_ctx_create(struct sock *sk); | |
139 | void tls_ctx_free(struct sock *sk, struct tls_context *ctx); | |
140 | void update_sk_prot(struct sock *sk, struct tls_context *ctx); | |
141 | ||
142 | int wait_on_pending_writer(struct sock *sk, long *timeo); | |
58790314 JK |
143 | void tls_err_abort(struct sock *sk, int err); |
144 | ||
1a074f76 SD |
145 | int init_prot_info(struct tls_prot_info *prot, |
146 | const struct tls_crypto_info *crypto_info, | |
b7c4f573 | 147 | const struct tls_cipher_desc *cipher_desc); |
b6a30ec9 | 148 | int tls_set_sw_offload(struct sock *sk, int tx); |
58790314 JK |
149 | void tls_update_rx_zc_capable(struct tls_context *tls_ctx); |
150 | void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx); | |
151 | void tls_sw_strparser_done(struct tls_context *tls_ctx); | |
152 | int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); | |
df720d28 | 153 | void tls_sw_splice_eof(struct socket *sock); |
58790314 JK |
154 | void tls_sw_cancel_work_tx(struct tls_context *tls_ctx); |
155 | void tls_sw_release_resources_tx(struct sock *sk); | |
156 | void tls_sw_free_ctx_tx(struct tls_context *tls_ctx); | |
157 | void tls_sw_free_resources_rx(struct sock *sk); | |
158 | void tls_sw_release_resources_rx(struct sock *sk); | |
159 | void tls_sw_free_ctx_rx(struct tls_context *tls_ctx); | |
160 | int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, | |
161 | int flags, int *addr_len); | |
162 | bool tls_sw_sock_is_readable(struct sock *sk); | |
163 | ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, | |
164 | struct pipe_inode_info *pipe, | |
165 | size_t len, unsigned int flags); | |
662fbcec HR |
166 | int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, |
167 | sk_read_actor_t read_actor); | |
58790314 JK |
168 | |
169 | int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); | |
d4c1e80b | 170 | void tls_device_splice_eof(struct socket *sock); |
58790314 JK |
171 | int tls_tx_records(struct sock *sk, int flags); |
172 | ||
173 | void tls_sw_write_space(struct sock *sk, struct tls_context *ctx); | |
174 | void tls_device_write_space(struct sock *sk, struct tls_context *ctx); | |
175 | ||
176 | int tls_process_cmsg(struct sock *sk, struct msghdr *msg, | |
177 | unsigned char *record_type); | |
541cc48b | 178 | int decrypt_skb(struct sock *sk, struct scatterlist *sgout); |
58790314 JK |
179 | |
180 | int tls_sw_fallback_init(struct sock *sk, | |
181 | struct tls_offload_context_tx *offload_ctx, | |
182 | struct tls_crypto_info *crypto_info); | |
183 | ||
84c61fe1 JK |
184 | int tls_strp_dev_init(void); |
185 | void tls_strp_dev_exit(void); | |
186 | ||
187 | void tls_strp_done(struct tls_strparser *strp); | |
188 | void tls_strp_stop(struct tls_strparser *strp); | |
189 | int tls_strp_init(struct tls_strparser *strp, struct sock *sk); | |
190 | void tls_strp_data_ready(struct tls_strparser *strp); | |
191 | ||
192 | void tls_strp_check_rcv(struct tls_strparser *strp); | |
193 | void tls_strp_msg_done(struct tls_strparser *strp); | |
194 | ||
195 | int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb); | |
196 | void tls_rx_msg_ready(struct tls_strparser *strp); | |
197 | ||
198 | void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh); | |
8b3c59a7 | 199 | int tls_strp_msg_cow(struct tls_sw_context_rx *ctx); |
d4e5db64 | 200 | struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx); |
84c61fe1 | 201 | int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst); |
c618db2a | 202 | |
58790314 JK |
203 | static inline struct tls_msg *tls_msg(struct sk_buff *skb) |
204 | { | |
205 | struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; | |
206 | ||
207 | return &scb->tls; | |
208 | } | |
209 | ||
541cc48b JK |
210 | static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx) |
211 | { | |
84c61fe1 JK |
212 | DEBUG_NET_WARN_ON_ONCE(!ctx->strp.msg_ready || !ctx->strp.anchor->len); |
213 | return ctx->strp.anchor; | |
541cc48b JK |
214 | } |
215 | ||
b92a13d4 JK |
216 | static inline bool tls_strp_msg_ready(struct tls_sw_context_rx *ctx) |
217 | { | |
84c61fe1 | 218 | return ctx->strp.msg_ready; |
b92a13d4 JK |
219 | } |
220 | ||
eca9bfaf JK |
221 | static inline bool tls_strp_msg_mixed_decrypted(struct tls_sw_context_rx *ctx) |
222 | { | |
223 | return ctx->strp.mixed_decrypted; | |
224 | } | |
225 | ||
58790314 | 226 | #ifdef CONFIG_TLS_DEVICE |
816cd168 | 227 | int tls_device_init(void); |
58790314 | 228 | void tls_device_cleanup(void); |
4f486699 | 229 | int tls_set_device_offload(struct sock *sk); |
58790314 JK |
230 | void tls_device_free_resources_tx(struct sock *sk); |
231 | int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); | |
232 | void tls_device_offload_cleanup_rx(struct sock *sk); | |
233 | void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq); | |
541cc48b | 234 | int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx); |
58790314 | 235 | #else |
816cd168 | 236 | static inline int tls_device_init(void) { return 0; } |
58790314 JK |
237 | static inline void tls_device_cleanup(void) {} |
238 | ||
239 | static inline int | |
4f486699 | 240 | tls_set_device_offload(struct sock *sk) |
58790314 JK |
241 | { |
242 | return -EOPNOTSUPP; | |
243 | } | |
244 | ||
245 | static inline void tls_device_free_resources_tx(struct sock *sk) {} | |
246 | ||
247 | static inline int | |
248 | tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx) | |
249 | { | |
250 | return -EOPNOTSUPP; | |
251 | } | |
252 | ||
253 | static inline void tls_device_offload_cleanup_rx(struct sock *sk) {} | |
254 | static inline void | |
255 | tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {} | |
256 | ||
257 | static inline int | |
541cc48b | 258 | tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) |
58790314 JK |
259 | { |
260 | return 0; | |
261 | } | |
262 | #endif | |
263 | ||
264 | int tls_push_sg(struct sock *sk, struct tls_context *ctx, | |
265 | struct scatterlist *sg, u16 first_offset, | |
266 | int flags); | |
267 | int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, | |
268 | int flags); | |
269 | void tls_free_partial_record(struct sock *sk, struct tls_context *ctx); | |
270 | ||
271 | static inline bool tls_is_partially_sent_record(struct tls_context *ctx) | |
272 | { | |
273 | return !!ctx->partially_sent_record; | |
274 | } | |
275 | ||
276 | static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx) | |
277 | { | |
278 | return tls_ctx->pending_open_record_frags; | |
279 | } | |
280 | ||
281 | static inline bool tls_bigint_increment(unsigned char *seq, int len) | |
282 | { | |
283 | int i; | |
284 | ||
285 | for (i = len - 1; i >= 0; i--) { | |
286 | ++seq[i]; | |
287 | if (seq[i] != 0) | |
288 | break; | |
289 | } | |
290 | ||
291 | return (i == -1); | |
292 | } | |
293 | ||
294 | static inline void tls_bigint_subtract(unsigned char *seq, int n) | |
295 | { | |
296 | u64 rcd_sn; | |
297 | __be64 *p; | |
298 | ||
299 | BUILD_BUG_ON(TLS_MAX_REC_SEQ_SIZE != 8); | |
300 | ||
301 | p = (__be64 *)seq; | |
302 | rcd_sn = be64_to_cpu(*p); | |
303 | *p = cpu_to_be64(rcd_sn - n); | |
304 | } | |
305 | ||
306 | static inline void | |
307 | tls_advance_record_sn(struct sock *sk, struct tls_prot_info *prot, | |
308 | struct cipher_context *ctx) | |
309 | { | |
310 | if (tls_bigint_increment(ctx->rec_seq, prot->rec_seq_size)) | |
311 | tls_err_abort(sk, -EBADMSG); | |
312 | ||
313 | if (prot->version != TLS_1_3_VERSION && | |
314 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) | |
315 | tls_bigint_increment(ctx->iv + prot->salt_size, | |
316 | prot->iv_size); | |
317 | } | |
318 | ||
319 | static inline void | |
320 | tls_xor_iv_with_seq(struct tls_prot_info *prot, char *iv, char *seq) | |
321 | { | |
322 | int i; | |
323 | ||
324 | if (prot->version == TLS_1_3_VERSION || | |
325 | prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) { | |
326 | for (i = 0; i < 8; i++) | |
327 | iv[i + 4] ^= seq[i]; | |
328 | } | |
329 | } | |
330 | ||
331 | static inline void | |
332 | tls_fill_prepend(struct tls_context *ctx, char *buf, size_t plaintext_len, | |
333 | unsigned char record_type) | |
334 | { | |
335 | struct tls_prot_info *prot = &ctx->prot_info; | |
336 | size_t pkt_len, iv_size = prot->iv_size; | |
337 | ||
338 | pkt_len = plaintext_len + prot->tag_size; | |
339 | if (prot->version != TLS_1_3_VERSION && | |
340 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) { | |
341 | pkt_len += iv_size; | |
342 | ||
343 | memcpy(buf + TLS_NONCE_OFFSET, | |
344 | ctx->tx.iv + prot->salt_size, iv_size); | |
345 | } | |
346 | ||
347 | /* we cover nonce explicit here as well, so buf should be of | |
348 | * size KTLS_DTLS_HEADER_SIZE + KTLS_DTLS_NONCE_EXPLICIT_SIZE | |
349 | */ | |
350 | buf[0] = prot->version == TLS_1_3_VERSION ? | |
351 | TLS_RECORD_TYPE_DATA : record_type; | |
352 | /* Note that VERSION must be TLS_1_2 for both TLS1.2 and TLS1.3 */ | |
353 | buf[1] = TLS_1_2_VERSION_MINOR; | |
354 | buf[2] = TLS_1_2_VERSION_MAJOR; | |
355 | /* we can use IV for nonce explicit according to spec */ | |
356 | buf[3] = pkt_len >> 8; | |
357 | buf[4] = pkt_len & 0xFF; | |
358 | } | |
359 | ||
360 | static inline | |
361 | void tls_make_aad(char *buf, size_t size, char *record_sequence, | |
362 | unsigned char record_type, struct tls_prot_info *prot) | |
363 | { | |
364 | if (prot->version != TLS_1_3_VERSION) { | |
365 | memcpy(buf, record_sequence, prot->rec_seq_size); | |
366 | buf += 8; | |
367 | } else { | |
368 | size += prot->tag_size; | |
369 | } | |
370 | ||
371 | buf[0] = prot->version == TLS_1_3_VERSION ? | |
372 | TLS_RECORD_TYPE_DATA : record_type; | |
373 | buf[1] = TLS_1_2_VERSION_MAJOR; | |
374 | buf[2] = TLS_1_2_VERSION_MINOR; | |
375 | buf[3] = size >> 8; | |
376 | buf[4] = size & 0xFF; | |
377 | } | |
378 | ||
379 | #endif |