Commit | Line | Data |
---|---|---|
58790314 | 1 | /* |
84c61fe1 | 2 | * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> |
58790314 JK |
3 | * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. |
4 | * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. | |
5 | * | |
6 | * This software is available to you under a choice of one of two | |
7 | * licenses. You may choose to be licensed under the terms of the GNU | |
8 | * General Public License (GPL) Version 2, available from the file | |
9 | * COPYING in the main directory of this source tree, or the | |
10 | * OpenIB.org BSD license below: | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or | |
13 | * without modification, are permitted provided that the following | |
14 | * conditions are met: | |
15 | * | |
16 | * - Redistributions of source code must retain the above | |
17 | * copyright notice, this list of conditions and the following | |
18 | * disclaimer. | |
19 | * | |
20 | * - Redistributions in binary form must reproduce the above | |
21 | * copyright notice, this list of conditions and the following | |
22 | * disclaimer in the documentation and/or other materials | |
23 | * provided with the distribution. | |
24 | * | |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
32 | * SOFTWARE. | |
33 | */ | |
34 | ||
35 | #ifndef _TLS_INT_H | |
36 | #define _TLS_INT_H | |
37 | ||
38 | #include <asm/byteorder.h> | |
39 | #include <linux/types.h> | |
40 | #include <linux/skmsg.h> | |
41 | #include <net/tls.h> | |
42 | ||
fd31f399 JK |
43 | #define TLS_PAGE_ORDER (min_t(unsigned int, PAGE_ALLOC_COSTLY_ORDER, \ |
44 | TLS_MAX_PAYLOAD_SIZE >> PAGE_SHIFT)) | |
45 | ||
58790314 JK |
46 | #define __TLS_INC_STATS(net, field) \ |
47 | __SNMP_INC_STATS((net)->mib.tls_statistics, field) | |
48 | #define TLS_INC_STATS(net, field) \ | |
49 | SNMP_INC_STATS((net)->mib.tls_statistics, field) | |
50 | #define TLS_DEC_STATS(net, field) \ | |
51 | SNMP_DEC_STATS((net)->mib.tls_statistics, field) | |
52 | ||
53 | /* TLS records are maintained in 'struct tls_rec'. It stores the memory pages | |
54 | * allocated or mapped for each TLS record. After encryption, the records are | |
55 | * stores in a linked list. | |
56 | */ | |
57 | struct tls_rec { | |
58 | struct list_head list; | |
59 | int tx_ready; | |
60 | int tx_flags; | |
61 | ||
62 | struct sk_msg msg_plaintext; | |
63 | struct sk_msg msg_encrypted; | |
64 | ||
65 | /* AAD | msg_plaintext.sg.data | sg_tag */ | |
66 | struct scatterlist sg_aead_in[2]; | |
67 | /* AAD | msg_encrypted.sg.data (data contains overhead for hdr & iv & tag) */ | |
68 | struct scatterlist sg_aead_out[2]; | |
69 | ||
70 | char content_type; | |
71 | struct scatterlist sg_content_type; | |
72 | ||
8d338c76 HX |
73 | struct sock *sk; |
74 | ||
58790314 JK |
75 | char aad_space[TLS_AAD_SPACE_SIZE]; |
76 | u8 iv_data[MAX_IV_SIZE]; | |
77 | struct aead_request aead_req; | |
78 | u8 aead_req_ctx[]; | |
79 | }; | |
80 | ||
81 | int __net_init tls_proc_init(struct net *net); | |
82 | void __net_exit tls_proc_fini(struct net *net); | |
83 | ||
84 | struct tls_context *tls_ctx_create(struct sock *sk); | |
85 | void tls_ctx_free(struct sock *sk, struct tls_context *ctx); | |
86 | void update_sk_prot(struct sock *sk, struct tls_context *ctx); | |
87 | ||
88 | int wait_on_pending_writer(struct sock *sk, long *timeo); | |
89 | int tls_sk_query(struct sock *sk, int optname, char __user *optval, | |
90 | int __user *optlen); | |
91 | int tls_sk_attach(struct sock *sk, int optname, char __user *optval, | |
92 | unsigned int optlen); | |
93 | void tls_err_abort(struct sock *sk, int err); | |
94 | ||
95 | int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx); | |
96 | void tls_update_rx_zc_capable(struct tls_context *tls_ctx); | |
97 | void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx); | |
98 | void tls_sw_strparser_done(struct tls_context *tls_ctx); | |
99 | int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); | |
100 | int tls_sw_sendpage_locked(struct sock *sk, struct page *page, | |
101 | int offset, size_t size, int flags); | |
102 | int tls_sw_sendpage(struct sock *sk, struct page *page, | |
103 | int offset, size_t size, int flags); | |
104 | void tls_sw_cancel_work_tx(struct tls_context *tls_ctx); | |
105 | void tls_sw_release_resources_tx(struct sock *sk); | |
106 | void tls_sw_free_ctx_tx(struct tls_context *tls_ctx); | |
107 | void tls_sw_free_resources_rx(struct sock *sk); | |
108 | void tls_sw_release_resources_rx(struct sock *sk); | |
109 | void tls_sw_free_ctx_rx(struct tls_context *tls_ctx); | |
110 | int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, | |
111 | int flags, int *addr_len); | |
112 | bool tls_sw_sock_is_readable(struct sock *sk); | |
113 | ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, | |
114 | struct pipe_inode_info *pipe, | |
115 | size_t len, unsigned int flags); | |
116 | ||
117 | int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); | |
118 | int tls_device_sendpage(struct sock *sk, struct page *page, | |
119 | int offset, size_t size, int flags); | |
120 | int tls_tx_records(struct sock *sk, int flags); | |
121 | ||
122 | void tls_sw_write_space(struct sock *sk, struct tls_context *ctx); | |
123 | void tls_device_write_space(struct sock *sk, struct tls_context *ctx); | |
124 | ||
125 | int tls_process_cmsg(struct sock *sk, struct msghdr *msg, | |
126 | unsigned char *record_type); | |
541cc48b | 127 | int decrypt_skb(struct sock *sk, struct scatterlist *sgout); |
58790314 JK |
128 | |
129 | int tls_sw_fallback_init(struct sock *sk, | |
130 | struct tls_offload_context_tx *offload_ctx, | |
131 | struct tls_crypto_info *crypto_info); | |
132 | ||
84c61fe1 JK |
133 | int tls_strp_dev_init(void); |
134 | void tls_strp_dev_exit(void); | |
135 | ||
136 | void tls_strp_done(struct tls_strparser *strp); | |
137 | void tls_strp_stop(struct tls_strparser *strp); | |
138 | int tls_strp_init(struct tls_strparser *strp, struct sock *sk); | |
139 | void tls_strp_data_ready(struct tls_strparser *strp); | |
140 | ||
141 | void tls_strp_check_rcv(struct tls_strparser *strp); | |
142 | void tls_strp_msg_done(struct tls_strparser *strp); | |
143 | ||
144 | int tls_rx_msg_size(struct tls_strparser *strp, struct sk_buff *skb); | |
145 | void tls_rx_msg_ready(struct tls_strparser *strp); | |
146 | ||
147 | void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh); | |
8b3c59a7 | 148 | int tls_strp_msg_cow(struct tls_sw_context_rx *ctx); |
d4e5db64 | 149 | struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx); |
84c61fe1 | 150 | int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst); |
c618db2a | 151 | |
58790314 JK |
152 | static inline struct tls_msg *tls_msg(struct sk_buff *skb) |
153 | { | |
154 | struct sk_skb_cb *scb = (struct sk_skb_cb *)skb->cb; | |
155 | ||
156 | return &scb->tls; | |
157 | } | |
158 | ||
541cc48b JK |
159 | static inline struct sk_buff *tls_strp_msg(struct tls_sw_context_rx *ctx) |
160 | { | |
84c61fe1 JK |
161 | DEBUG_NET_WARN_ON_ONCE(!ctx->strp.msg_ready || !ctx->strp.anchor->len); |
162 | return ctx->strp.anchor; | |
541cc48b JK |
163 | } |
164 | ||
b92a13d4 JK |
165 | static inline bool tls_strp_msg_ready(struct tls_sw_context_rx *ctx) |
166 | { | |
84c61fe1 | 167 | return ctx->strp.msg_ready; |
b92a13d4 JK |
168 | } |
169 | ||
58790314 | 170 | #ifdef CONFIG_TLS_DEVICE |
816cd168 | 171 | int tls_device_init(void); |
58790314 JK |
172 | void tls_device_cleanup(void); |
173 | int tls_set_device_offload(struct sock *sk, struct tls_context *ctx); | |
174 | void tls_device_free_resources_tx(struct sock *sk); | |
175 | int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); | |
176 | void tls_device_offload_cleanup_rx(struct sock *sk); | |
177 | void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq); | |
541cc48b | 178 | int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx); |
58790314 | 179 | #else |
816cd168 | 180 | static inline int tls_device_init(void) { return 0; } |
58790314 JK |
181 | static inline void tls_device_cleanup(void) {} |
182 | ||
183 | static inline int | |
184 | tls_set_device_offload(struct sock *sk, struct tls_context *ctx) | |
185 | { | |
186 | return -EOPNOTSUPP; | |
187 | } | |
188 | ||
189 | static inline void tls_device_free_resources_tx(struct sock *sk) {} | |
190 | ||
191 | static inline int | |
192 | tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx) | |
193 | { | |
194 | return -EOPNOTSUPP; | |
195 | } | |
196 | ||
197 | static inline void tls_device_offload_cleanup_rx(struct sock *sk) {} | |
198 | static inline void | |
199 | tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {} | |
200 | ||
201 | static inline int | |
541cc48b | 202 | tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) |
58790314 JK |
203 | { |
204 | return 0; | |
205 | } | |
206 | #endif | |
207 | ||
208 | int tls_push_sg(struct sock *sk, struct tls_context *ctx, | |
209 | struct scatterlist *sg, u16 first_offset, | |
210 | int flags); | |
211 | int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, | |
212 | int flags); | |
213 | void tls_free_partial_record(struct sock *sk, struct tls_context *ctx); | |
214 | ||
215 | static inline bool tls_is_partially_sent_record(struct tls_context *ctx) | |
216 | { | |
217 | return !!ctx->partially_sent_record; | |
218 | } | |
219 | ||
220 | static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx) | |
221 | { | |
222 | return tls_ctx->pending_open_record_frags; | |
223 | } | |
224 | ||
225 | static inline bool tls_bigint_increment(unsigned char *seq, int len) | |
226 | { | |
227 | int i; | |
228 | ||
229 | for (i = len - 1; i >= 0; i--) { | |
230 | ++seq[i]; | |
231 | if (seq[i] != 0) | |
232 | break; | |
233 | } | |
234 | ||
235 | return (i == -1); | |
236 | } | |
237 | ||
238 | static inline void tls_bigint_subtract(unsigned char *seq, int n) | |
239 | { | |
240 | u64 rcd_sn; | |
241 | __be64 *p; | |
242 | ||
243 | BUILD_BUG_ON(TLS_MAX_REC_SEQ_SIZE != 8); | |
244 | ||
245 | p = (__be64 *)seq; | |
246 | rcd_sn = be64_to_cpu(*p); | |
247 | *p = cpu_to_be64(rcd_sn - n); | |
248 | } | |
249 | ||
250 | static inline void | |
251 | tls_advance_record_sn(struct sock *sk, struct tls_prot_info *prot, | |
252 | struct cipher_context *ctx) | |
253 | { | |
254 | if (tls_bigint_increment(ctx->rec_seq, prot->rec_seq_size)) | |
255 | tls_err_abort(sk, -EBADMSG); | |
256 | ||
257 | if (prot->version != TLS_1_3_VERSION && | |
258 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) | |
259 | tls_bigint_increment(ctx->iv + prot->salt_size, | |
260 | prot->iv_size); | |
261 | } | |
262 | ||
263 | static inline void | |
264 | tls_xor_iv_with_seq(struct tls_prot_info *prot, char *iv, char *seq) | |
265 | { | |
266 | int i; | |
267 | ||
268 | if (prot->version == TLS_1_3_VERSION || | |
269 | prot->cipher_type == TLS_CIPHER_CHACHA20_POLY1305) { | |
270 | for (i = 0; i < 8; i++) | |
271 | iv[i + 4] ^= seq[i]; | |
272 | } | |
273 | } | |
274 | ||
275 | static inline void | |
276 | tls_fill_prepend(struct tls_context *ctx, char *buf, size_t plaintext_len, | |
277 | unsigned char record_type) | |
278 | { | |
279 | struct tls_prot_info *prot = &ctx->prot_info; | |
280 | size_t pkt_len, iv_size = prot->iv_size; | |
281 | ||
282 | pkt_len = plaintext_len + prot->tag_size; | |
283 | if (prot->version != TLS_1_3_VERSION && | |
284 | prot->cipher_type != TLS_CIPHER_CHACHA20_POLY1305) { | |
285 | pkt_len += iv_size; | |
286 | ||
287 | memcpy(buf + TLS_NONCE_OFFSET, | |
288 | ctx->tx.iv + prot->salt_size, iv_size); | |
289 | } | |
290 | ||
291 | /* we cover nonce explicit here as well, so buf should be of | |
292 | * size KTLS_DTLS_HEADER_SIZE + KTLS_DTLS_NONCE_EXPLICIT_SIZE | |
293 | */ | |
294 | buf[0] = prot->version == TLS_1_3_VERSION ? | |
295 | TLS_RECORD_TYPE_DATA : record_type; | |
296 | /* Note that VERSION must be TLS_1_2 for both TLS1.2 and TLS1.3 */ | |
297 | buf[1] = TLS_1_2_VERSION_MINOR; | |
298 | buf[2] = TLS_1_2_VERSION_MAJOR; | |
299 | /* we can use IV for nonce explicit according to spec */ | |
300 | buf[3] = pkt_len >> 8; | |
301 | buf[4] = pkt_len & 0xFF; | |
302 | } | |
303 | ||
304 | static inline | |
305 | void tls_make_aad(char *buf, size_t size, char *record_sequence, | |
306 | unsigned char record_type, struct tls_prot_info *prot) | |
307 | { | |
308 | if (prot->version != TLS_1_3_VERSION) { | |
309 | memcpy(buf, record_sequence, prot->rec_seq_size); | |
310 | buf += 8; | |
311 | } else { | |
312 | size += prot->tag_size; | |
313 | } | |
314 | ||
315 | buf[0] = prot->version == TLS_1_3_VERSION ? | |
316 | TLS_RECORD_TYPE_DATA : record_type; | |
317 | buf[1] = TLS_1_2_VERSION_MAJOR; | |
318 | buf[2] = TLS_1_2_VERSION_MINOR; | |
319 | buf[3] = size >> 8; | |
320 | buf[4] = size & 0xFF; | |
321 | } | |
322 | ||
323 | #endif |