projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: freescale: fix return type of ndo_start_xmit function
[linux-2.6-block.git] / net / tls / tls_sw.c
CommitLineData
3c4d7559
DW		 1/*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 */
36
c46234eb 37#include <linux/sched/signal.h>
3c4d7559
DW		 38#include <linux/module.h>
39#include <crypto/aead.h>
40
c46234eb 41#include <net/strparser.h>
3c4d7559
DW		 42#include <net/tls.h>
43
b16520f7
KC		 44#define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
45
0927f71d
DRK		 46static int __skb_nsg(struct sk_buff *skb, int offset, int len,
47 unsigned int recursion_level)
48{
49 int start = skb_headlen(skb);
50 int i, chunk = start - offset;
51 struct sk_buff *frag_iter;
52 int elt = 0;
53
54 if (unlikely(recursion_level >= 24))
55 return -EMSGSIZE;
56
57 if (chunk > 0) {
58 if (chunk > len)
59 chunk = len;
60 elt++;
61 len -= chunk;
62 if (len == 0)
63 return elt;
64 offset += chunk;
65 }
66
67 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
68 int end;
69
70 WARN_ON(start > offset + len);
71
72 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
73 chunk = end - offset;
74 if (chunk > 0) {
75 if (chunk > len)
76 chunk = len;
77 elt++;
78 len -= chunk;
79 if (len == 0)
80 return elt;
81 offset += chunk;
82 }
83 start = end;
84 }
85
86 if (unlikely(skb_has_frag_list(skb))) {
87 skb_walk_frags(skb, frag_iter) {
88 int end, ret;
89
90 WARN_ON(start > offset + len);
91
92 end = start + frag_iter->len;
93 chunk = end - offset;
94 if (chunk > 0) {
95 if (chunk > len)
96 chunk = len;
97 ret = __skb_nsg(frag_iter, offset - start, chunk,
98 recursion_level + 1);
99 if (unlikely(ret < 0))
100 return ret;
101 elt += ret;
102 len -= chunk;
103 if (len == 0)
104 return elt;
105 offset += chunk;
106 }
107 start = end;
108 }
109 }
110 BUG_ON(len);
111 return elt;
112}
113
114/* Return the number of scatterlist elements required to completely map the
115 * skb, or -EMSGSIZE if the recursion depth is exceeded.
116 */
117static int skb_nsg(struct sk_buff *skb, int offset, int len)
118{
119 return __skb_nsg(skb, offset, len, 0);
120}
121
94524d8f
VG		 122static void tls_decrypt_done(struct crypto_async_request *req, int err)
123{
124 struct aead_request *aead_req = (struct aead_request *)req;
94524d8f 125 struct scatterlist *sgout = aead_req->dst;
7a3dd8c8
JF		 126 struct tls_sw_context_rx *ctx;
127 struct tls_context *tls_ctx;
94524d8f 128 struct scatterlist *sg;
7a3dd8c8 129 struct sk_buff *skb;
94524d8f 130 unsigned int pages;
7a3dd8c8
JF		 131 int pending;
132
133 skb = (struct sk_buff *)req->data;
134 tls_ctx = tls_get_ctx(skb->sk);
135 ctx = tls_sw_ctx_rx(tls_ctx);
136 pending = atomic_dec_return(&ctx->decrypt_pending);
94524d8f
VG		 137
138 /* Propagate if there was an err */
139 if (err) {
140 ctx->async_wait.err = err;
7a3dd8c8 141 tls_err_abort(skb->sk, err);
94524d8f
VG		 142 }
143
7a3dd8c8
JF		 144 /* After using skb->sk to propagate sk through crypto async callback
145 * we need to NULL it again.
146 */
147 skb->sk = NULL;
148
94524d8f 149 /* Release the skb, pages and memory allocated for crypto req */
7a3dd8c8 150 kfree_skb(skb);
94524d8f
VG		 151
152 /* Skip the first S/G entry as it points to AAD */
153 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) {
154 if (!sg)
155 break;
156 put_page(sg_page(sg));
157 }
158
159 kfree(aead_req);
160
161 if (!pending && READ_ONCE(ctx->async_notify))
162 complete(&ctx->async_wait.completion);
163}
164
c46234eb 165static int tls_do_decryption(struct sock *sk,
94524d8f 166 struct sk_buff *skb,
c46234eb
DW		 167 struct scatterlist *sgin,
168 struct scatterlist *sgout,
169 char *iv_recv,
170 size_t data_len,
94524d8f
VG		 171 struct aead_request *aead_req,
172 bool async)
c46234eb
DW		 173{
174 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 175 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 176 int ret;
c46234eb 177
0b243d00 178 aead_request_set_tfm(aead_req, ctx->aead_recv);
c46234eb
DW		 179 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
180 aead_request_set_crypt(aead_req, sgin, sgout,
181 data_len + tls_ctx->rx.tag_size,
182 (u8 *)iv_recv);
c46234eb 183
94524d8f 184 if (async) {
7a3dd8c8
JF		 185 /* Using skb->sk to push sk through to crypto async callback
186 * handler. This allows propagating errors up to the socket
187 * if needed. It _must_ be cleared in the async handler
188 * before kfree_skb is called. We _know_ skb->sk is NULL
189 * because it is a clone from strparser.
190 */
191 skb->sk = sk;
94524d8f
VG		 192 aead_request_set_callback(aead_req,
193 CRYPTO_TFM_REQ_MAY_BACKLOG,
194 tls_decrypt_done, skb);
195 atomic_inc(&ctx->decrypt_pending);
196 } else {
197 aead_request_set_callback(aead_req,
198 CRYPTO_TFM_REQ_MAY_BACKLOG,
199 crypto_req_done, &ctx->async_wait);
200 }
201
202 ret = crypto_aead_decrypt(aead_req);
203 if (ret == -EINPROGRESS) {
204 if (async)
205 return ret;
206
207 ret = crypto_wait_req(ret, &ctx->async_wait);
208 }
209
210 if (async)
211 atomic_dec(&ctx->decrypt_pending);
212
c46234eb
DW		 213 return ret;
214}
215
3c4d7559
DW		 216static void trim_sg(struct sock *sk, struct scatterlist *sg,
217 int *sg_num_elem, unsigned int *sg_size, int target_size)
218{
219 int i = *sg_num_elem - 1;
220 int trim = *sg_size - target_size;
221
222 if (trim <= 0) {
223 WARN_ON(trim < 0);
224 return;
225 }
226
227 *sg_size = target_size;
228 while (trim >= sg[i].length) {
229 trim -= sg[i].length;
230 sk_mem_uncharge(sk, sg[i].length);
231 put_page(sg_page(&sg[i]));
232 i--;
233
234 if (i < 0)
235 goto out;
236 }
237
238 sg[i].length -= trim;
239 sk_mem_uncharge(sk, trim);
240
241out:
242 *sg_num_elem = i + 1;
243}
244
245static void trim_both_sgl(struct sock *sk, int target_size)
246{
247 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 248 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 249
250 trim_sg(sk, ctx->sg_plaintext_data,
251 &ctx->sg_plaintext_num_elem,
252 &ctx->sg_plaintext_size,
253 target_size);
254
255 if (target_size > 0)
dbe42559 256 target_size += tls_ctx->tx.overhead_size;
3c4d7559
DW		 257
258 trim_sg(sk, ctx->sg_encrypted_data,
259 &ctx->sg_encrypted_num_elem,
260 &ctx->sg_encrypted_size,
261 target_size);
262}
263
3c4d7559
DW		 264static int alloc_encrypted_sg(struct sock *sk, int len)
265{
266 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 267 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 268 int rc = 0;
269
2c3682f0 270 rc = sk_alloc_sg(sk, len,
8c05dbf0 271 ctx->sg_encrypted_data, 0,
2c3682f0
JF		 272 &ctx->sg_encrypted_num_elem,
273 &ctx->sg_encrypted_size, 0);
3c4d7559 274
52ea992c
VG		 275 if (rc == -ENOSPC)
276 ctx->sg_encrypted_num_elem = ARRAY_SIZE(ctx->sg_encrypted_data);
277
3c4d7559
DW		 278 return rc;
279}
280
281static int alloc_plaintext_sg(struct sock *sk, int len)
282{
283 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 284 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 285 int rc = 0;
286
8c05dbf0 287 rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0,
2c3682f0
JF		 288 &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
289 tls_ctx->pending_open_record_frags);
3c4d7559 290
52ea992c
VG		 291 if (rc == -ENOSPC)
292 ctx->sg_plaintext_num_elem = ARRAY_SIZE(ctx->sg_plaintext_data);
293
3c4d7559
DW		 294 return rc;
295}
296
297static void free_sg(struct sock *sk, struct scatterlist *sg,
298 int *sg_num_elem, unsigned int *sg_size)
299{
300 int i, n = *sg_num_elem;
301
302 for (i = 0; i < n; ++i) {
303 sk_mem_uncharge(sk, sg[i].length);
304 put_page(sg_page(&sg[i]));
305 }
306 *sg_num_elem = 0;
307 *sg_size = 0;
308}
309
310static void tls_free_both_sg(struct sock *sk)
311{
312 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 313 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 314
315 free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
316 &ctx->sg_encrypted_size);
317
318 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
319 &ctx->sg_plaintext_size);
320}
321
322static int tls_do_encryption(struct tls_context *tls_ctx,
a447da7d
DB		 323 struct tls_sw_context_tx *ctx,
324 struct aead_request *aead_req,
325 size_t data_len)
3c4d7559 326{
3c4d7559
DW		 327 int rc;
328
dbe42559
DW		 329 ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size;
330 ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size;
3c4d7559
DW		 331
332 aead_request_set_tfm(aead_req, ctx->aead_send);
333 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
334 aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
dbe42559 335 data_len, tls_ctx->tx.iv);
a54667f6
VG		 336
337 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
338 crypto_req_done, &ctx->async_wait);
339
340 rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait);
3c4d7559 341
dbe42559
DW		 342 ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
343 ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
3c4d7559 344
3c4d7559
DW		 345 return rc;
346}
347
348static int tls_push_record(struct sock *sk, int flags,
349 unsigned char record_type)
350{
351 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 352 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a447da7d 353 struct aead_request *req;
3c4d7559
DW		 354 int rc;
355
d2bdd268 356 req = aead_request_alloc(ctx->aead_send, sk->sk_allocation);
a447da7d
DB		 357 if (!req)
358 return -ENOMEM;
359
3c4d7559
DW		 360 sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
361 sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
362
213ef6e7 363 tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size,
dbe42559 364 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
3c4d7559
DW		 365 record_type);
366
367 tls_fill_prepend(tls_ctx,
368 page_address(sg_page(&ctx->sg_encrypted_data[0])) +
369 ctx->sg_encrypted_data[0].offset,
370 ctx->sg_plaintext_size, record_type);
371
372 tls_ctx->pending_open_record_frags = 0;
373 set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
374
a447da7d 375 rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
3c4d7559
DW		 376 if (rc < 0) {
377 /* If we are called from write_space and
378 * we fail, we need to set this SOCK_NOSPACE
379 * to trigger another write_space in the future.
380 */
381 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
a447da7d 382 goto out_req;
3c4d7559
DW		 383 }
384
385 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
386 &ctx->sg_plaintext_size);
387
388 ctx->sg_encrypted_num_elem = 0;
389 ctx->sg_encrypted_size = 0;
390
391 /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
392 rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
393 if (rc < 0 && rc != -EAGAIN)
f4a8e43f 394 tls_err_abort(sk, EBADMSG);
3c4d7559 395
dbe42559 396 tls_advance_record_sn(sk, &tls_ctx->tx);
a447da7d 397out_req:
d2bdd268 398 aead_request_free(req);
3c4d7559
DW		 399 return rc;
400}
401
402static int tls_sw_push_pending_record(struct sock *sk, int flags)
403{
404 return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
405}
406
407static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
69ca9293
DW		 408 int length, int *pages_used,
409 unsigned int *size_used,
410 struct scatterlist *to, int to_max_pages,
2da19ed3 411 bool charge)
3c4d7559 412{
3c4d7559
DW		 413 struct page *pages[MAX_SKB_FRAGS];
414
415 size_t offset;
416 ssize_t copied, use;
417 int i = 0;
69ca9293
DW		 418 unsigned int size = *size_used;
419 int num_elem = *pages_used;
3c4d7559
DW		 420 int rc = 0;
421 int maxpages;
422
423 while (length > 0) {
424 i = 0;
69ca9293 425 maxpages = to_max_pages - num_elem;
3c4d7559
DW		 426 if (maxpages == 0) {
427 rc = -EFAULT;
428 goto out;
429 }
430 copied = iov_iter_get_pages(from, pages,
431 length,
432 maxpages, &offset);
433 if (copied <= 0) {
434 rc = -EFAULT;
435 goto out;
436 }
437
438 iov_iter_advance(from, copied);
439
440 length -= copied;
441 size += copied;
442 while (copied) {
443 use = min_t(int, copied, PAGE_SIZE - offset);
444
69ca9293 445 sg_set_page(&to[num_elem],
3c4d7559 446 pages[i], use, offset);
69ca9293
DW		 447 sg_unmark_end(&to[num_elem]);
448 if (charge)
449 sk_mem_charge(sk, use);
3c4d7559
DW		 450
451 offset = 0;
452 copied -= use;
453
454 ++i;
455 ++num_elem;
456 }
457 }
458
cfb4099f
VG		 459 /* Mark the end in the last sg entry if newly added */
460 if (num_elem > *pages_used)
461 sg_mark_end(&to[num_elem - 1]);
3c4d7559 462out:
2da19ed3
DRK		 463 if (rc)
464 iov_iter_revert(from, size - *size_used);
69ca9293
DW		 465 *size_used = size;
466 *pages_used = num_elem;
467
3c4d7559
DW		 468 return rc;
469}
470
471static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
472 int bytes)
473{
474 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 475 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 476 struct scatterlist *sg = ctx->sg_plaintext_data;
477 int copy, i, rc = 0;
478
479 for (i = tls_ctx->pending_open_record_frags;
480 i < ctx->sg_plaintext_num_elem; ++i) {
481 copy = sg[i].length;
482 if (copy_from_iter(
483 page_address(sg_page(&sg[i])) + sg[i].offset,
484 copy, from) != copy) {
485 rc = -EFAULT;
486 goto out;
487 }
488 bytes -= copy;
489
490 ++tls_ctx->pending_open_record_frags;
491
492 if (!bytes)
493 break;
494 }
495
496out:
497 return rc;
498}
499
500int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
501{
502 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 503 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
15008579 504 int ret;
3c4d7559
DW		 505 int required_size;
506 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
507 bool eor = !(msg->msg_flags & MSG_MORE);
508 size_t try_to_copy, copied = 0;
509 unsigned char record_type = TLS_RECORD_TYPE_DATA;
510 int record_room;
511 bool full_record;
512 int orig_size;
0a26cf3f 513 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
3c4d7559
DW		 514
515 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
516 return -ENOTSUPP;
517
518 lock_sock(sk);
519
15008579
VG		 520 ret = tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo);
521 if (ret)
3c4d7559
DW		 522 goto send_end;
523
524 if (unlikely(msg->msg_controllen)) {
525 ret = tls_proccess_cmsg(sk, msg, &record_type);
526 if (ret)
527 goto send_end;
528 }
529
530 while (msg_data_left(msg)) {
531 if (sk->sk_err) {
30be8f8d 532 ret = -sk->sk_err;
3c4d7559
DW		 533 goto send_end;
534 }
535
536 orig_size = ctx->sg_plaintext_size;
537 full_record = false;
538 try_to_copy = msg_data_left(msg);
539 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
540 if (try_to_copy >= record_room) {
541 try_to_copy = record_room;
542 full_record = true;
543 }
544
545 required_size = ctx->sg_plaintext_size + try_to_copy +
dbe42559 546 tls_ctx->tx.overhead_size;
3c4d7559
DW		 547
548 if (!sk_stream_memory_free(sk))
549 goto wait_for_sndbuf;
550alloc_encrypted:
551 ret = alloc_encrypted_sg(sk, required_size);
552 if (ret) {
553 if (ret != -ENOSPC)
554 goto wait_for_memory;
555
556 /* Adjust try_to_copy according to the amount that was
557 * actually allocated. The difference is due
558 * to max sg elements limit
559 */
560 try_to_copy -= required_size - ctx->sg_encrypted_size;
561 full_record = true;
562 }
0a26cf3f 563 if (!is_kvec && (full_record || eor)) {
3c4d7559 564 ret = zerocopy_from_iter(sk, &msg->msg_iter,
69ca9293
DW		 565 try_to_copy, &ctx->sg_plaintext_num_elem,
566 &ctx->sg_plaintext_size,
567 ctx->sg_plaintext_data,
568 ARRAY_SIZE(ctx->sg_plaintext_data),
2da19ed3 569 true);
3c4d7559
DW		 570 if (ret)
571 goto fallback_to_reg_send;
572
573 copied += try_to_copy;
574 ret = tls_push_record(sk, msg->msg_flags, record_type);
5a3611ef 575 if (ret)
3c4d7559 576 goto send_end;
5a3611ef 577 continue;
3c4d7559 578
3c4d7559 579fallback_to_reg_send:
3c4d7559
DW		 580 trim_sg(sk, ctx->sg_plaintext_data,
581 &ctx->sg_plaintext_num_elem,
582 &ctx->sg_plaintext_size,
583 orig_size);
584 }
585
586 required_size = ctx->sg_plaintext_size + try_to_copy;
587alloc_plaintext:
588 ret = alloc_plaintext_sg(sk, required_size);
589 if (ret) {
590 if (ret != -ENOSPC)
591 goto wait_for_memory;
592
593 /* Adjust try_to_copy according to the amount that was
594 * actually allocated. The difference is due
595 * to max sg elements limit
596 */
597 try_to_copy -= required_size - ctx->sg_plaintext_size;
598 full_record = true;
599
600 trim_sg(sk, ctx->sg_encrypted_data,
601 &ctx->sg_encrypted_num_elem,
602 &ctx->sg_encrypted_size,
603 ctx->sg_plaintext_size +
dbe42559 604 tls_ctx->tx.overhead_size);
3c4d7559
DW		 605 }
606
607 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
608 if (ret)
609 goto trim_sgl;
610
611 copied += try_to_copy;
612 if (full_record || eor) {
613push_record:
614 ret = tls_push_record(sk, msg->msg_flags, record_type);
615 if (ret) {
616 if (ret == -ENOMEM)
617 goto wait_for_memory;
618
619 goto send_end;
620 }
621 }
622
623 continue;
624
625wait_for_sndbuf:
626 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
627wait_for_memory:
628 ret = sk_stream_wait_memory(sk, &timeo);
629 if (ret) {
630trim_sgl:
631 trim_both_sgl(sk, orig_size);
632 goto send_end;
633 }
634
635 if (tls_is_pending_closed_record(tls_ctx))
636 goto push_record;
637
638 if (ctx->sg_encrypted_size < required_size)
639 goto alloc_encrypted;
640
641 goto alloc_plaintext;
642 }
643
644send_end:
645 ret = sk_stream_error(sk, msg->msg_flags, ret);
646
647 release_sock(sk);
648 return copied ? copied : ret;
649}
650
651int tls_sw_sendpage(struct sock *sk, struct page *page,
652 int offset, size_t size, int flags)
653{
654 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 655 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
15008579 656 int ret;
3c4d7559
DW		 657 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
658 bool eor;
659 size_t orig_size = size;
660 unsigned char record_type = TLS_RECORD_TYPE_DATA;
661 struct scatterlist *sg;
662 bool full_record;
663 int record_room;
664
665 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
666 MSG_SENDPAGE_NOTLAST))
667 return -ENOTSUPP;
668
669 /* No MSG_EOR from splice, only look at MSG_MORE */
670 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
671
672 lock_sock(sk);
673
674 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
675
15008579
VG		 676 ret = tls_complete_pending_work(sk, tls_ctx, flags, &timeo);
677 if (ret)
3c4d7559
DW		 678 goto sendpage_end;
679
680 /* Call the sk_stream functions to manage the sndbuf mem. */
681 while (size > 0) {
682 size_t copy, required_size;
683
684 if (sk->sk_err) {
30be8f8d 685 ret = -sk->sk_err;
3c4d7559
DW		 686 goto sendpage_end;
687 }
688
689 full_record = false;
690 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
691 copy = size;
692 if (copy >= record_room) {
693 copy = record_room;
694 full_record = true;
695 }
696 required_size = ctx->sg_plaintext_size + copy +
dbe42559 697 tls_ctx->tx.overhead_size;
3c4d7559
DW		 698
699 if (!sk_stream_memory_free(sk))
700 goto wait_for_sndbuf;
701alloc_payload:
702 ret = alloc_encrypted_sg(sk, required_size);
703 if (ret) {
704 if (ret != -ENOSPC)
705 goto wait_for_memory;
706
707 /* Adjust copy according to the amount that was
708 * actually allocated. The difference is due
709 * to max sg elements limit
710 */
711 copy -= required_size - ctx->sg_plaintext_size;
712 full_record = true;
713 }
714
715 get_page(page);
716 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
717 sg_set_page(sg, page, copy, offset);
7a8c4dd9
DW		 718 sg_unmark_end(sg);
719
3c4d7559
DW		 720 ctx->sg_plaintext_num_elem++;
721
722 sk_mem_charge(sk, copy);
723 offset += copy;
724 size -= copy;
725 ctx->sg_plaintext_size += copy;
726 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
727
728 if (full_record || eor ||
729 ctx->sg_plaintext_num_elem ==
730 ARRAY_SIZE(ctx->sg_plaintext_data)) {
731push_record:
732 ret = tls_push_record(sk, flags, record_type);
733 if (ret) {
734 if (ret == -ENOMEM)
735 goto wait_for_memory;
736
737 goto sendpage_end;
738 }
739 }
740 continue;
741wait_for_sndbuf:
742 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
743wait_for_memory:
744 ret = sk_stream_wait_memory(sk, &timeo);
745 if (ret) {
746 trim_both_sgl(sk, ctx->sg_plaintext_size);
747 goto sendpage_end;
748 }
749
750 if (tls_is_pending_closed_record(tls_ctx))
751 goto push_record;
752
753 goto alloc_payload;
754 }
755
756sendpage_end:
757 if (orig_size > size)
758 ret = orig_size - size;
759 else
760 ret = sk_stream_error(sk, flags, ret);
761
762 release_sock(sk);
763 return ret;
764}
765
c46234eb
DW		 766static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
767 long timeo, int *err)
768{
769 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 770 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 771 struct sk_buff *skb;
772 DEFINE_WAIT_FUNC(wait, woken_wake_function);
773
774 while (!(skb = ctx->recv_pkt)) {
775 if (sk->sk_err) {
776 *err = sock_error(sk);
777 return NULL;
778 }
779
fcf4793e
DRK		 780 if (sk->sk_shutdown & RCV_SHUTDOWN)
781 return NULL;
782
c46234eb
DW		 783 if (sock_flag(sk, SOCK_DONE))
784 return NULL;
785
786 if ((flags & MSG_DONTWAIT) || !timeo) {
787 *err = -EAGAIN;
788 return NULL;
789 }
790
791 add_wait_queue(sk_sleep(sk), &wait);
792 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
793 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait);
794 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
795 remove_wait_queue(sk_sleep(sk), &wait);
796
797 /* Handle signals */
798 if (signal_pending(current)) {
799 *err = sock_intr_errno(timeo);
800 return NULL;
801 }
802 }
803
804 return skb;
805}
806
0b243d00
VG		 807/* This function decrypts the input skb into either out_iov or in out_sg
808 * or in skb buffers itself. The input parameter 'zc' indicates if
809 * zero-copy mode needs to be tried or not. With zero-copy mode, either
810 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
811 * NULL, then the decryption happens inside skb buffers itself, i.e.
812 * zero-copy gets disabled and 'zc' is updated.
813 */
814
815static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
816 struct iov_iter *out_iov,
817 struct scatterlist *out_sg,
818 int *chunk, bool *zc)
819{
820 struct tls_context *tls_ctx = tls_get_ctx(sk);
821 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
822 struct strp_msg *rxm = strp_msg(skb);
823 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0;
824 struct aead_request *aead_req;
825 struct sk_buff *unused;
826 u8 *aad, *iv, *mem = NULL;
827 struct scatterlist *sgin = NULL;
828 struct scatterlist *sgout = NULL;
829 const int data_len = rxm->full_len - tls_ctx->rx.overhead_size;
830
831 if (*zc && (out_iov || out_sg)) {
832 if (out_iov)
833 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
834 else
835 n_sgout = sg_nents(out_sg);
0927f71d
DRK		 836 n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size,
837 rxm->full_len - tls_ctx->rx.prepend_size);
0b243d00
VG		 838 } else {
839 n_sgout = 0;
840 *zc = false;
0927f71d 841 n_sgin = skb_cow_data(skb, 0, &unused);
0b243d00
VG		 842 }
843
0b243d00
VG		 844 if (n_sgin < 1)
845 return -EBADMSG;
846
847 /* Increment to accommodate AAD */
848 n_sgin = n_sgin + 1;
849
850 nsg = n_sgin + n_sgout;
851
852 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
853 mem_size = aead_size + (nsg * sizeof(struct scatterlist));
854 mem_size = mem_size + TLS_AAD_SPACE_SIZE;
855 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv);
856
857 /* Allocate a single block of memory which contains
858 * aead_req || sgin[] || sgout[] || aad || iv.
859 * This order achieves correct alignment for aead_req, sgin, sgout.
860 */
861 mem = kmalloc(mem_size, sk->sk_allocation);
862 if (!mem)
863 return -ENOMEM;
864
865 /* Segment the allocated memory */
866 aead_req = (struct aead_request *)mem;
867 sgin = (struct scatterlist *)(mem + aead_size);
868 sgout = sgin + n_sgin;
869 aad = (u8 *)(sgout + n_sgout);
870 iv = aad + TLS_AAD_SPACE_SIZE;
871
872 /* Prepare IV */
873 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
874 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
875 tls_ctx->rx.iv_size);
876 if (err < 0) {
877 kfree(mem);
878 return err;
879 }
880 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
881
882 /* Prepare AAD */
883 tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size,
884 tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size,
885 ctx->control);
886
887 /* Prepare sgin */
888 sg_init_table(sgin, n_sgin);
889 sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE);
890 err = skb_to_sgvec(skb, &sgin[1],
891 rxm->offset + tls_ctx->rx.prepend_size,
892 rxm->full_len - tls_ctx->rx.prepend_size);
893 if (err < 0) {
894 kfree(mem);
895 return err;
896 }
897
898 if (n_sgout) {
899 if (out_iov) {
900 sg_init_table(sgout, n_sgout);
901 sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE);
902
903 *chunk = 0;
904 err = zerocopy_from_iter(sk, out_iov, data_len, &pages,
905 chunk, &sgout[1],
906 (n_sgout - 1), false);
907 if (err < 0)
908 goto fallback_to_reg_recv;
909 } else if (out_sg) {
910 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
911 } else {
912 goto fallback_to_reg_recv;
913 }
914 } else {
915fallback_to_reg_recv:
916 sgout = sgin;
917 pages = 0;
918 *chunk = 0;
919 *zc = false;
920 }
921
922 /* Prepare and submit AEAD request */
94524d8f
VG		 923 err = tls_do_decryption(sk, skb, sgin, sgout, iv,
924 data_len, aead_req, *zc);
925 if (err == -EINPROGRESS)
926 return err;
0b243d00
VG		 927
928 /* Release the pages in case iov was mapped to pages */
929 for (; pages > 0; pages--)
930 put_page(sg_page(&sgout[pages]));
931
932 kfree(mem);
933 return err;
934}
935
dafb67f3 936static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
0b243d00 937 struct iov_iter *dest, int *chunk, bool *zc)
dafb67f3
BP		 938{
939 struct tls_context *tls_ctx = tls_get_ctx(sk);
940 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
941 struct strp_msg *rxm = strp_msg(skb);
942 int err = 0;
943
4799ac81
BP		 944#ifdef CONFIG_TLS_DEVICE
945 err = tls_device_decrypted(sk, skb);
dafb67f3
BP		 946 if (err < 0)
947 return err;
4799ac81
BP		 948#endif
949 if (!ctx->decrypted) {
0b243d00 950 err = decrypt_internal(sk, skb, dest, NULL, chunk, zc);
94524d8f
VG		 951 if (err < 0) {
952 if (err == -EINPROGRESS)
953 tls_advance_record_sn(sk, &tls_ctx->rx);
954
4799ac81 955 return err;
94524d8f 956 }
4799ac81
BP		 957 } else {
958 *zc = false;
959 }
dafb67f3
BP		 960
961 rxm->offset += tls_ctx->rx.prepend_size;
962 rxm->full_len -= tls_ctx->rx.overhead_size;
963 tls_advance_record_sn(sk, &tls_ctx->rx);
964 ctx->decrypted = true;
965 ctx->saved_data_ready(sk);
966
967 return err;
968}
969
970int decrypt_skb(struct sock *sk, struct sk_buff *skb,
971 struct scatterlist *sgout)
c46234eb 972{
0b243d00
VG		 973 bool zc = true;
974 int chunk;
c46234eb 975
0b243d00 976 return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc);
c46234eb
DW		 977}
978
979static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
980 unsigned int len)
981{
982 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 983 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 984
94524d8f
VG		 985 if (skb) {
986 struct strp_msg *rxm = strp_msg(skb);
c46234eb 987
94524d8f
VG		 988 if (len < rxm->full_len) {
989 rxm->offset += len;
990 rxm->full_len -= len;
991 return false;
992 }
993 kfree_skb(skb);
c46234eb
DW		 994 }
995
996 /* Finished with message */
997 ctx->recv_pkt = NULL;
7170e604 998 __strp_unpause(&ctx->strp);
c46234eb
DW		 999
1000 return true;
1001}
1002
1003int tls_sw_recvmsg(struct sock *sk,
1004 struct msghdr *msg,
1005 size_t len,
1006 int nonblock,
1007 int flags,
1008 int *addr_len)
1009{
1010 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1011 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1012 unsigned char control;
1013 struct strp_msg *rxm;
1014 struct sk_buff *skb;
1015 ssize_t copied = 0;
1016 bool cmsg = false;
06030dba 1017 int target, err = 0;
c46234eb 1018 long timeo;
0a26cf3f 1019 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
94524d8f 1020 int num_async = 0;
c46234eb
DW		 1021
1022 flags |= nonblock;
1023
1024 if (unlikely(flags & MSG_ERRQUEUE))
1025 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
1026
1027 lock_sock(sk);
1028
06030dba 1029 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
c46234eb
DW		 1030 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1031 do {
1032 bool zc = false;
94524d8f 1033 bool async = false;
c46234eb
DW		 1034 int chunk = 0;
1035
1036 skb = tls_wait_data(sk, flags, timeo, &err);
1037 if (!skb)
1038 goto recv_end;
1039
1040 rxm = strp_msg(skb);
94524d8f 1041
c46234eb
DW		 1042 if (!cmsg) {
1043 int cerr;
1044
1045 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
1046 sizeof(ctx->control), &ctx->control);
1047 cmsg = true;
1048 control = ctx->control;
1049 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1050 if (cerr || msg->msg_flags & MSG_CTRUNC) {
1051 err = -EIO;
1052 goto recv_end;
1053 }
1054 }
1055 } else if (control != ctx->control) {
1056 goto recv_end;
1057 }
1058
1059 if (!ctx->decrypted) {
0b243d00 1060 int to_copy = rxm->full_len - tls_ctx->rx.overhead_size;
c46234eb 1061
0b243d00
VG		 1062 if (!is_kvec && to_copy <= len &&
1063 likely(!(flags & MSG_PEEK)))
c46234eb 1064 zc = true;
0b243d00
VG		 1065
1066 err = decrypt_skb_update(sk, skb, &msg->msg_iter,
1067 &chunk, &zc);
94524d8f 1068 if (err < 0 && err != -EINPROGRESS) {
0b243d00
VG		 1069 tls_err_abort(sk, EBADMSG);
1070 goto recv_end;
c46234eb 1071 }
94524d8f
VG		 1072
1073 if (err == -EINPROGRESS) {
1074 async = true;
1075 num_async++;
1076 goto pick_next_record;
1077 }
1078
c46234eb
DW		 1079 ctx->decrypted = true;
1080 }
1081
1082 if (!zc) {
1083 chunk = min_t(unsigned int, rxm->full_len, len);
94524d8f 1084
c46234eb
DW		 1085 err = skb_copy_datagram_msg(skb, rxm->offset, msg,
1086 chunk);
1087 if (err < 0)
1088 goto recv_end;
1089 }
1090
94524d8f 1091pick_next_record:
c46234eb
DW		 1092 copied += chunk;
1093 len -= chunk;
1094 if (likely(!(flags & MSG_PEEK))) {
1095 u8 control = ctx->control;
1096
94524d8f
VG		 1097 /* For async, drop current skb reference */
1098 if (async)
1099 skb = NULL;
1100
c46234eb
DW		 1101 if (tls_sw_advance_skb(sk, skb, chunk)) {
1102 /* Return full control message to
1103 * userspace before trying to parse
1104 * another message type
1105 */
1106 msg->msg_flags |= MSG_EOR;
1107 if (control != TLS_RECORD_TYPE_DATA)
1108 goto recv_end;
94524d8f
VG		 1109 } else {
1110 break;
c46234eb 1111 }
50c6b58a
DB		 1112 } else {
1113 /* MSG_PEEK right now cannot look beyond current skb
1114 * from strparser, meaning we cannot advance skb here
1115 * and thus unpause strparser since we'd loose original
1116 * one.
1117 */
1118 break;
c46234eb 1119 }
94524d8f 1120
06030dba
DB		 1121 /* If we have a new message from strparser, continue now. */
1122 if (copied >= target && !ctx->recv_pkt)
1123 break;
c46234eb
DW		 1124 } while (len);
1125
1126recv_end:
94524d8f
VG		 1127 if (num_async) {
1128 /* Wait for all previously submitted records to be decrypted */
1129 smp_store_mb(ctx->async_notify, true);
1130 if (atomic_read(&ctx->decrypt_pending)) {
1131 err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1132 if (err) {
1133 /* one of async decrypt failed */
1134 tls_err_abort(sk, err);
1135 copied = 0;
1136 }
1137 } else {
1138 reinit_completion(&ctx->async_wait.completion);
1139 }
1140 WRITE_ONCE(ctx->async_notify, false);
1141 }
1142
c46234eb
DW		 1143 release_sock(sk);
1144 return copied ? : err;
1145}
1146
1147ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
1148 struct pipe_inode_info *pipe,
1149 size_t len, unsigned int flags)
1150{
1151 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
f66de3ee 1152 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1153 struct strp_msg *rxm = NULL;
1154 struct sock *sk = sock->sk;
1155 struct sk_buff *skb;
1156 ssize_t copied = 0;
1157 int err = 0;
1158 long timeo;
1159 int chunk;
0b243d00 1160 bool zc = false;
c46234eb
DW		 1161
1162 lock_sock(sk);
1163
1164 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1165
1166 skb = tls_wait_data(sk, flags, timeo, &err);
1167 if (!skb)
1168 goto splice_read_end;
1169
1170 /* splice does not support reading control messages */
1171 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1172 err = -ENOTSUPP;
1173 goto splice_read_end;
1174 }
1175
1176 if (!ctx->decrypted) {
0b243d00 1177 err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc);
c46234eb
DW		 1178
1179 if (err < 0) {
1180 tls_err_abort(sk, EBADMSG);
1181 goto splice_read_end;
1182 }
1183 ctx->decrypted = true;
1184 }
1185 rxm = strp_msg(skb);
1186
1187 chunk = min_t(unsigned int, rxm->full_len, len);
1188 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
1189 if (copied < 0)
1190 goto splice_read_end;
1191
1192 if (likely(!(flags & MSG_PEEK)))
1193 tls_sw_advance_skb(sk, skb, copied);
1194
1195splice_read_end:
1196 release_sock(sk);
1197 return copied ? : err;
1198}
1199
a11e1d43
LT		 1200unsigned int tls_sw_poll(struct file *file, struct socket *sock,
1201 struct poll_table_struct *wait)
c46234eb 1202{
a11e1d43 1203 unsigned int ret;
c46234eb
DW		 1204 struct sock *sk = sock->sk;
1205 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1206 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 1207
a11e1d43
LT		 1208 /* Grab POLLOUT and POLLHUP from the underlying socket */
1209 ret = ctx->sk_poll(file, sock, wait);
c46234eb 1210
a11e1d43
LT		 1211 /* Clear POLLIN bits, and set based on recv_pkt */
1212 ret &= ~(POLLIN | POLLRDNORM);
c46234eb 1213 if (ctx->recv_pkt)
a11e1d43 1214 ret |= POLLIN | POLLRDNORM;
c46234eb 1215
a11e1d43 1216 return ret;
c46234eb
DW		 1217}
1218
1219static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
1220{
1221 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 1222 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
3463e51d 1223 char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
c46234eb
DW		 1224 struct strp_msg *rxm = strp_msg(skb);
1225 size_t cipher_overhead;
1226 size_t data_len = 0;
1227 int ret;
1228
1229 /* Verify that we have a full TLS header, or wait for more data */
1230 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
1231 return 0;
1232
3463e51d
KC		 1233 /* Sanity-check size of on-stack buffer. */
1234 if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) {
1235 ret = -EINVAL;
1236 goto read_failure;
1237 }
1238
c46234eb
DW		 1239 /* Linearize header to local buffer */
1240 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
1241
1242 if (ret < 0)
1243 goto read_failure;
1244
1245 ctx->control = header[0];
1246
1247 data_len = ((header[4] & 0xFF) | (header[3] << 8));
1248
1249 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
1250
1251 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
1252 ret = -EMSGSIZE;
1253 goto read_failure;
1254 }
1255 if (data_len < cipher_overhead) {
1256 ret = -EBADMSG;
1257 goto read_failure;
1258 }
1259
86029d10
SD		 1260 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.info.version) ||
1261 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.info.version)) {
c46234eb
DW		 1262 ret = -EINVAL;
1263 goto read_failure;
1264 }
1265
4799ac81
BP		 1266#ifdef CONFIG_TLS_DEVICE
1267 handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset,
1268 *(u64*)tls_ctx->rx.rec_seq);
1269#endif
c46234eb
DW		 1270 return data_len + TLS_HEADER_SIZE;
1271
1272read_failure:
1273 tls_err_abort(strp->sk, ret);
1274
1275 return ret;
1276}
1277
1278static void tls_queue(struct strparser *strp, struct sk_buff *skb)
1279{
1280 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 1281 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1282
1283 ctx->decrypted = false;
1284
1285 ctx->recv_pkt = skb;
1286 strp_pause(strp);
1287
ad13acce 1288 ctx->saved_data_ready(strp->sk);
c46234eb
DW		 1289}
1290
1291static void tls_data_ready(struct sock *sk)
1292{
1293 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1294 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1295
1296 strp_data_ready(&ctx->strp);
1297}
1298
f66de3ee 1299void tls_sw_free_resources_tx(struct sock *sk)
3c4d7559
DW		 1300{
1301 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1302 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559 1303
201876b3 1304 crypto_free_aead(ctx->aead_send);
f66de3ee
BP		 1305 tls_free_both_sg(sk);
1306
1307 kfree(ctx);
1308}
1309
39f56e1a 1310void tls_sw_release_resources_rx(struct sock *sk)
f66de3ee
BP		 1311{
1312 struct tls_context *tls_ctx = tls_get_ctx(sk);
1313 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1314
c46234eb 1315 if (ctx->aead_recv) {
201876b3
VG		 1316 kfree_skb(ctx->recv_pkt);
1317 ctx->recv_pkt = NULL;
c46234eb
DW		 1318 crypto_free_aead(ctx->aead_recv);
1319 strp_stop(&ctx->strp);
1320 write_lock_bh(&sk->sk_callback_lock);
1321 sk->sk_data_ready = ctx->saved_data_ready;
1322 write_unlock_bh(&sk->sk_callback_lock);
1323 release_sock(sk);
1324 strp_done(&ctx->strp);
1325 lock_sock(sk);
1326 }
39f56e1a
BP		 1327}
1328
1329void tls_sw_free_resources_rx(struct sock *sk)
1330{
1331 struct tls_context *tls_ctx = tls_get_ctx(sk);
1332 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1333
1334 tls_sw_release_resources_rx(sk);
3c4d7559 1335
3c4d7559
DW		 1336 kfree(ctx);
1337}
1338
c46234eb 1339int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
3c4d7559 1340{
3c4d7559
DW		 1341 struct tls_crypto_info *crypto_info;
1342 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
f66de3ee
BP		 1343 struct tls_sw_context_tx *sw_ctx_tx = NULL;
1344 struct tls_sw_context_rx *sw_ctx_rx = NULL;
c46234eb
DW		 1345 struct cipher_context *cctx;
1346 struct crypto_aead **aead;
1347 struct strp_callbacks cb;
3c4d7559
DW		 1348 u16 nonce_size, tag_size, iv_size, rec_seq_size;
1349 char *iv, *rec_seq;
1350 int rc = 0;
1351
1352 if (!ctx) {
1353 rc = -EINVAL;
1354 goto out;
1355 }
1356
f66de3ee 1357 if (tx) {
b190a587
BP		 1358 if (!ctx->priv_ctx_tx) {
1359 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
1360 if (!sw_ctx_tx) {
1361 rc = -ENOMEM;
1362 goto out;
1363 }
1364 ctx->priv_ctx_tx = sw_ctx_tx;
1365 } else {
1366 sw_ctx_tx =
1367 (struct tls_sw_context_tx *)ctx->priv_ctx_tx;
c46234eb 1368 }
c46234eb 1369 } else {
b190a587
BP		 1370 if (!ctx->priv_ctx_rx) {
1371 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
1372 if (!sw_ctx_rx) {
1373 rc = -ENOMEM;
1374 goto out;
1375 }
1376 ctx->priv_ctx_rx = sw_ctx_rx;
1377 } else {
1378 sw_ctx_rx =
1379 (struct tls_sw_context_rx *)ctx->priv_ctx_rx;
f66de3ee 1380 }
3c4d7559
DW		 1381 }
1382
c46234eb 1383 if (tx) {
b190a587 1384 crypto_init_wait(&sw_ctx_tx->async_wait);
86029d10 1385 crypto_info = &ctx->crypto_send.info;
c46234eb 1386 cctx = &ctx->tx;
f66de3ee 1387 aead = &sw_ctx_tx->aead_send;
c46234eb 1388 } else {
b190a587 1389 crypto_init_wait(&sw_ctx_rx->async_wait);
86029d10 1390 crypto_info = &ctx->crypto_recv.info;
c46234eb 1391 cctx = &ctx->rx;
f66de3ee 1392 aead = &sw_ctx_rx->aead_recv;
c46234eb
DW		 1393 }
1394
3c4d7559
DW		 1395 switch (crypto_info->cipher_type) {
1396 case TLS_CIPHER_AES_GCM_128: {
1397 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1398 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1399 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1400 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
1401 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
1402 rec_seq =
1403 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
1404 gcm_128_info =
1405 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
1406 break;
1407 }
1408 default:
1409 rc = -EINVAL;
cf6d43ef 1410 goto free_priv;
3c4d7559
DW		 1411 }
1412
b16520f7 1413 /* Sanity-check the IV size for stack allocations. */
3463e51d 1414 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) {
b16520f7
KC		 1415 rc = -EINVAL;
1416 goto free_priv;
1417 }
1418
c46234eb
DW		 1419 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
1420 cctx->tag_size = tag_size;
1421 cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
1422 cctx->iv_size = iv_size;
1423 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1424 GFP_KERNEL);
1425 if (!cctx->iv) {
3c4d7559 1426 rc = -ENOMEM;
cf6d43ef 1427 goto free_priv;
3c4d7559 1428 }
c46234eb
DW		 1429 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1430 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
1431 cctx->rec_seq_size = rec_seq_size;
969d5090 1432 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
c46234eb 1433 if (!cctx->rec_seq) {
3c4d7559
DW		 1434 rc = -ENOMEM;
1435 goto free_iv;
1436 }
c46234eb 1437
f66de3ee
BP		 1438 if (sw_ctx_tx) {
1439 sg_init_table(sw_ctx_tx->sg_encrypted_data,
1440 ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data));
1441 sg_init_table(sw_ctx_tx->sg_plaintext_data,
1442 ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data));
1443
1444 sg_init_table(sw_ctx_tx->sg_aead_in, 2);
1445 sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space,
1446 sizeof(sw_ctx_tx->aad_space));
1447 sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]);
1448 sg_chain(sw_ctx_tx->sg_aead_in, 2,
1449 sw_ctx_tx->sg_plaintext_data);
1450 sg_init_table(sw_ctx_tx->sg_aead_out, 2);
1451 sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space,
1452 sizeof(sw_ctx_tx->aad_space));
1453 sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]);
1454 sg_chain(sw_ctx_tx->sg_aead_out, 2,
1455 sw_ctx_tx->sg_encrypted_data);
c46234eb
DW		 1456 }
1457
1458 if (!*aead) {
1459 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
1460 if (IS_ERR(*aead)) {
1461 rc = PTR_ERR(*aead);
1462 *aead = NULL;
3c4d7559
DW		 1463 goto free_rec_seq;
1464 }
1465 }
1466
1467 ctx->push_pending_record = tls_sw_push_pending_record;
1468
7cba09c6 1469 rc = crypto_aead_setkey(*aead, gcm_128_info->key,
3c4d7559
DW		 1470 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1471 if (rc)
1472 goto free_aead;
1473
c46234eb
DW		 1474 rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
1475 if (rc)
1476 goto free_aead;
1477
f66de3ee 1478 if (sw_ctx_rx) {
c46234eb
DW		 1479 /* Set up strparser */
1480 memset(&cb, 0, sizeof(cb));
1481 cb.rcv_msg = tls_queue;
1482 cb.parse_msg = tls_read_size;
1483
f66de3ee 1484 strp_init(&sw_ctx_rx->strp, sk, &cb);
c46234eb
DW		 1485
1486 write_lock_bh(&sk->sk_callback_lock);
f66de3ee 1487 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
c46234eb
DW		 1488 sk->sk_data_ready = tls_data_ready;
1489 write_unlock_bh(&sk->sk_callback_lock);
1490
a11e1d43 1491 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
c46234eb 1492
f66de3ee 1493 strp_check_rcv(&sw_ctx_rx->strp);
c46234eb
DW		 1494 }
1495
1496 goto out;
3c4d7559
DW		 1497
1498free_aead:
c46234eb
DW		 1499 crypto_free_aead(*aead);
1500 *aead = NULL;
3c4d7559 1501free_rec_seq:
c46234eb
DW		 1502 kfree(cctx->rec_seq);
1503 cctx->rec_seq = NULL;
3c4d7559 1504free_iv:
f66de3ee
BP		 1505 kfree(cctx->iv);
1506 cctx->iv = NULL;
cf6d43ef 1507free_priv:
f66de3ee
BP		 1508 if (tx) {
1509 kfree(ctx->priv_ctx_tx);
1510 ctx->priv_ctx_tx = NULL;
1511 } else {
1512 kfree(ctx->priv_ctx_rx);
1513 ctx->priv_ctx_rx = NULL;
1514 }
3c4d7559
DW		 1515out:
1516 return rc;
1517}
Linux 6.x block layer and io_uring tree(s)