projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'few-NTF_ROUTER-related-updates'
[linux-block.git] / net / tls / tls_sw.c
CommitLineData
3c4d7559
DW		 1/*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 */
36
c46234eb 37#include <linux/sched/signal.h>
3c4d7559
DW		 38#include <linux/module.h>
39#include <crypto/aead.h>
40
c46234eb 41#include <net/strparser.h>
3c4d7559
DW		 42#include <net/tls.h>
43
b16520f7
KC		 44#define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
45
0927f71d
DRK		 46static int __skb_nsg(struct sk_buff *skb, int offset, int len,
47 unsigned int recursion_level)
48{
49 int start = skb_headlen(skb);
50 int i, chunk = start - offset;
51 struct sk_buff *frag_iter;
52 int elt = 0;
53
54 if (unlikely(recursion_level >= 24))
55 return -EMSGSIZE;
56
57 if (chunk > 0) {
58 if (chunk > len)
59 chunk = len;
60 elt++;
61 len -= chunk;
62 if (len == 0)
63 return elt;
64 offset += chunk;
65 }
66
67 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
68 int end;
69
70 WARN_ON(start > offset + len);
71
72 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
73 chunk = end - offset;
74 if (chunk > 0) {
75 if (chunk > len)
76 chunk = len;
77 elt++;
78 len -= chunk;
79 if (len == 0)
80 return elt;
81 offset += chunk;
82 }
83 start = end;
84 }
85
86 if (unlikely(skb_has_frag_list(skb))) {
87 skb_walk_frags(skb, frag_iter) {
88 int end, ret;
89
90 WARN_ON(start > offset + len);
91
92 end = start + frag_iter->len;
93 chunk = end - offset;
94 if (chunk > 0) {
95 if (chunk > len)
96 chunk = len;
97 ret = __skb_nsg(frag_iter, offset - start, chunk,
98 recursion_level + 1);
99 if (unlikely(ret < 0))
100 return ret;
101 elt += ret;
102 len -= chunk;
103 if (len == 0)
104 return elt;
105 offset += chunk;
106 }
107 start = end;
108 }
109 }
110 BUG_ON(len);
111 return elt;
112}
113
114/* Return the number of scatterlist elements required to completely map the
115 * skb, or -EMSGSIZE if the recursion depth is exceeded.
116 */
117static int skb_nsg(struct sk_buff *skb, int offset, int len)
118{
119 return __skb_nsg(skb, offset, len, 0);
120}
121
94524d8f
VG		 122static void tls_decrypt_done(struct crypto_async_request *req, int err)
123{
124 struct aead_request *aead_req = (struct aead_request *)req;
94524d8f 125 struct scatterlist *sgout = aead_req->dst;
7a3dd8c8
JF		 126 struct tls_sw_context_rx *ctx;
127 struct tls_context *tls_ctx;
94524d8f 128 struct scatterlist *sg;
7a3dd8c8 129 struct sk_buff *skb;
94524d8f 130 unsigned int pages;
7a3dd8c8
JF		 131 int pending;
132
133 skb = (struct sk_buff *)req->data;
134 tls_ctx = tls_get_ctx(skb->sk);
135 ctx = tls_sw_ctx_rx(tls_ctx);
136 pending = atomic_dec_return(&ctx->decrypt_pending);
94524d8f
VG		 137
138 /* Propagate if there was an err */
139 if (err) {
140 ctx->async_wait.err = err;
7a3dd8c8 141 tls_err_abort(skb->sk, err);
94524d8f
VG		 142 }
143
7a3dd8c8
JF		 144 /* After using skb->sk to propagate sk through crypto async callback
145 * we need to NULL it again.
146 */
147 skb->sk = NULL;
148
94524d8f 149 /* Release the skb, pages and memory allocated for crypto req */
7a3dd8c8 150 kfree_skb(skb);
94524d8f
VG		 151
152 /* Skip the first S/G entry as it points to AAD */
153 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) {
154 if (!sg)
155 break;
156 put_page(sg_page(sg));
157 }
158
159 kfree(aead_req);
160
161 if (!pending && READ_ONCE(ctx->async_notify))
162 complete(&ctx->async_wait.completion);
163}
164
c46234eb 165static int tls_do_decryption(struct sock *sk,
94524d8f 166 struct sk_buff *skb,
c46234eb
DW		 167 struct scatterlist *sgin,
168 struct scatterlist *sgout,
169 char *iv_recv,
170 size_t data_len,
94524d8f
VG		 171 struct aead_request *aead_req,
172 bool async)
c46234eb
DW		 173{
174 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 175 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 176 int ret;
c46234eb 177
0b243d00 178 aead_request_set_tfm(aead_req, ctx->aead_recv);
c46234eb
DW		 179 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
180 aead_request_set_crypt(aead_req, sgin, sgout,
181 data_len + tls_ctx->rx.tag_size,
182 (u8 *)iv_recv);
c46234eb 183
94524d8f 184 if (async) {
7a3dd8c8
JF		 185 /* Using skb->sk to push sk through to crypto async callback
186 * handler. This allows propagating errors up to the socket
187 * if needed. It _must_ be cleared in the async handler
188 * before kfree_skb is called. We _know_ skb->sk is NULL
189 * because it is a clone from strparser.
190 */
191 skb->sk = sk;
94524d8f
VG		 192 aead_request_set_callback(aead_req,
193 CRYPTO_TFM_REQ_MAY_BACKLOG,
194 tls_decrypt_done, skb);
195 atomic_inc(&ctx->decrypt_pending);
196 } else {
197 aead_request_set_callback(aead_req,
198 CRYPTO_TFM_REQ_MAY_BACKLOG,
199 crypto_req_done, &ctx->async_wait);
200 }
201
202 ret = crypto_aead_decrypt(aead_req);
203 if (ret == -EINPROGRESS) {
204 if (async)
205 return ret;
206
207 ret = crypto_wait_req(ret, &ctx->async_wait);
208 }
209
210 if (async)
211 atomic_dec(&ctx->decrypt_pending);
212
c46234eb
DW		 213 return ret;
214}
215
3c4d7559
DW		 216static void trim_sg(struct sock *sk, struct scatterlist *sg,
217 int *sg_num_elem, unsigned int *sg_size, int target_size)
218{
219 int i = *sg_num_elem - 1;
220 int trim = *sg_size - target_size;
221
222 if (trim <= 0) {
223 WARN_ON(trim < 0);
224 return;
225 }
226
227 *sg_size = target_size;
228 while (trim >= sg[i].length) {
229 trim -= sg[i].length;
230 sk_mem_uncharge(sk, sg[i].length);
231 put_page(sg_page(&sg[i]));
232 i--;
233
234 if (i < 0)
235 goto out;
236 }
237
238 sg[i].length -= trim;
239 sk_mem_uncharge(sk, trim);
240
241out:
242 *sg_num_elem = i + 1;
243}
244
245static void trim_both_sgl(struct sock *sk, int target_size)
246{
247 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 248 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8 249 struct tls_rec *rec = ctx->open_rec;
3c4d7559 250
a42055e8
VG		 251 trim_sg(sk, rec->sg_plaintext_data,
252 &rec->sg_plaintext_num_elem,
253 &rec->sg_plaintext_size,
3c4d7559
DW		 254 target_size);
255
256 if (target_size > 0)
dbe42559 257 target_size += tls_ctx->tx.overhead_size;
3c4d7559 258
a42055e8
VG		 259 trim_sg(sk, rec->sg_encrypted_data,
260 &rec->sg_encrypted_num_elem,
261 &rec->sg_encrypted_size,
3c4d7559
DW		 262 target_size);
263}
264
3c4d7559
DW		 265static int alloc_encrypted_sg(struct sock *sk, int len)
266{
267 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 268 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8 269 struct tls_rec *rec = ctx->open_rec;
3c4d7559
DW		 270 int rc = 0;
271
2c3682f0 272 rc = sk_alloc_sg(sk, len,
a42055e8
VG		 273 rec->sg_encrypted_data, 0,
274 &rec->sg_encrypted_num_elem,
275 &rec->sg_encrypted_size, 0);
3c4d7559 276
52ea992c 277 if (rc == -ENOSPC)
a42055e8 278 rec->sg_encrypted_num_elem = ARRAY_SIZE(rec->sg_encrypted_data);
52ea992c 279
3c4d7559
DW		 280 return rc;
281}
282
283static int alloc_plaintext_sg(struct sock *sk, int len)
284{
285 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 286 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8 287 struct tls_rec *rec = ctx->open_rec;
3c4d7559
DW		 288 int rc = 0;
289
a42055e8
VG		 290 rc = sk_alloc_sg(sk, len, rec->sg_plaintext_data, 0,
291 &rec->sg_plaintext_num_elem, &rec->sg_plaintext_size,
2c3682f0 292 tls_ctx->pending_open_record_frags);
3c4d7559 293
52ea992c 294 if (rc == -ENOSPC)
a42055e8 295 rec->sg_plaintext_num_elem = ARRAY_SIZE(rec->sg_plaintext_data);
52ea992c 296
3c4d7559
DW		 297 return rc;
298}
299
300static void free_sg(struct sock *sk, struct scatterlist *sg,
301 int *sg_num_elem, unsigned int *sg_size)
302{
303 int i, n = *sg_num_elem;
304
305 for (i = 0; i < n; ++i) {
306 sk_mem_uncharge(sk, sg[i].length);
307 put_page(sg_page(&sg[i]));
308 }
309 *sg_num_elem = 0;
310 *sg_size = 0;
311}
312
313static void tls_free_both_sg(struct sock *sk)
314{
315 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 316 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8 317 struct tls_rec *rec = ctx->open_rec;
3c4d7559 318
a42055e8
VG		 319 /* Return if there is no open record */
320 if (!rec)
321 return;
322
323 free_sg(sk, rec->sg_encrypted_data,
324 &rec->sg_encrypted_num_elem,
325 &rec->sg_encrypted_size);
326
327 free_sg(sk, rec->sg_plaintext_data,
328 &rec->sg_plaintext_num_elem,
329 &rec->sg_plaintext_size);
330}
331
332static bool append_tx_ready_list(struct tls_context *tls_ctx,
333 struct tls_sw_context_tx *ctx,
334 struct tls_rec *enc_rec)
335{
336 u64 new_seq = be64_to_cpup((const __be64 *)&enc_rec->aad_space);
337 struct list_head *pos;
338
339 /* Need to insert encrypted record in tx_ready_list sorted
340 * as per sequence number. Traverse linked list from tail.
341 */
342 list_for_each_prev(pos, &ctx->tx_ready_list) {
343 struct tls_rec *rec = (struct tls_rec *)pos;
344 u64 seq = be64_to_cpup((const __be64 *)&rec->aad_space);
345
346 if (new_seq > seq)
347 break;
348 }
349
350 list_add((struct list_head *)&enc_rec->list, pos);
351
352 return is_tx_ready(tls_ctx, ctx);
353}
354
355int tls_tx_records(struct sock *sk, int flags)
356{
357 struct tls_context *tls_ctx = tls_get_ctx(sk);
358 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
359 struct tls_rec *rec, *tmp;
360 int tx_flags, rc = 0;
361
362 if (tls_is_partially_sent_record(tls_ctx)) {
363 rec = list_first_entry(&ctx->tx_ready_list,
364 struct tls_rec, list);
365
366 if (flags == -1)
367 tx_flags = rec->tx_flags;
368 else
369 tx_flags = flags;
370
371 rc = tls_push_partial_record(sk, tls_ctx, tx_flags);
372 if (rc)
373 goto tx_err;
374
375 /* Full record has been transmitted.
376 * Remove the head of tx_ready_list
377 */
378 tls_ctx->tx_seq_number++;
379 list_del(&rec->list);
380 kfree(rec);
381 }
382
383 /* Tx all ready records which have expected sequence number */
384 list_for_each_entry_safe(rec, tmp, &ctx->tx_ready_list, list) {
385 u64 seq = be64_to_cpup((const __be64 *)&rec->aad_space);
386
387 if (seq == tls_ctx->tx_seq_number) {
388 if (flags == -1)
389 tx_flags = rec->tx_flags;
390 else
391 tx_flags = flags;
392
393 rc = tls_push_sg(sk, tls_ctx,
394 &rec->sg_encrypted_data[0],
395 0, tx_flags);
396 if (rc)
397 goto tx_err;
398
399 tls_ctx->tx_seq_number++;
400 list_del(&rec->list);
401 kfree(rec);
402 } else {
403 break;
404 }
405 }
406
407tx_err:
408 if (rc < 0 && rc != -EAGAIN)
409 tls_err_abort(sk, EBADMSG);
410
411 return rc;
412}
413
414static void tls_encrypt_done(struct crypto_async_request *req, int err)
415{
416 struct aead_request *aead_req = (struct aead_request *)req;
417 struct sock *sk = req->data;
418 struct tls_context *tls_ctx = tls_get_ctx(sk);
419 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
420 struct tls_rec *rec;
421 bool ready = false;
422 int pending;
423
424 rec = container_of(aead_req, struct tls_rec, aead_req);
425
426 rec->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
427 rec->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
428
429 free_sg(sk, rec->sg_plaintext_data,
430 &rec->sg_plaintext_num_elem, &rec->sg_plaintext_size);
431
432 /* Free the record if error is previously set on socket */
433 if (err || sk->sk_err) {
434 free_sg(sk, rec->sg_encrypted_data,
435 &rec->sg_encrypted_num_elem, &rec->sg_encrypted_size);
3c4d7559 436
a42055e8
VG		 437 kfree(rec);
438 rec = NULL;
439
440 /* If err is already set on socket, return the same code */
441 if (sk->sk_err) {
442 ctx->async_wait.err = sk->sk_err;
443 } else {
444 ctx->async_wait.err = err;
445 tls_err_abort(sk, err);
446 }
447 }
448
449 /* Append the record in tx queue */
450 if (rec)
451 ready = append_tx_ready_list(tls_ctx, ctx, rec);
452
453 pending = atomic_dec_return(&ctx->encrypt_pending);
454
455 if (!pending && READ_ONCE(ctx->async_notify))
456 complete(&ctx->async_wait.completion);
457
458 if (!ready)
459 return;
460
461 /* Schedule the transmission */
462 if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
463 schedule_delayed_work(&ctx->tx_work.work, 1);
3c4d7559
DW		 464}
465
a42055e8
VG		 466static int tls_do_encryption(struct sock *sk,
467 struct tls_context *tls_ctx,
a447da7d
DB		 468 struct tls_sw_context_tx *ctx,
469 struct aead_request *aead_req,
470 size_t data_len)
3c4d7559 471{
a42055e8 472 struct tls_rec *rec = ctx->open_rec;
3c4d7559
DW		 473 int rc;
474
a42055e8
VG		 475 rec->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size;
476 rec->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size;
3c4d7559
DW		 477
478 aead_request_set_tfm(aead_req, ctx->aead_send);
479 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
a42055e8
VG		 480 aead_request_set_crypt(aead_req, rec->sg_aead_in,
481 rec->sg_aead_out,
dbe42559 482 data_len, tls_ctx->tx.iv);
a54667f6
VG		 483
484 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
a42055e8
VG		 485 tls_encrypt_done, sk);
486
487 atomic_inc(&ctx->encrypt_pending);
a54667f6 488
a42055e8
VG		 489 rc = crypto_aead_encrypt(aead_req);
490 if (!rc || rc != -EINPROGRESS) {
491 atomic_dec(&ctx->encrypt_pending);
492 rec->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
493 rec->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
494 }
3c4d7559 495
a42055e8
VG		 496 /* Case of encryption failure */
497 if (rc && rc != -EINPROGRESS)
498 return rc;
3c4d7559 499
a42055e8
VG		 500 /* Unhook the record from context if encryption is not failure */
501 ctx->open_rec = NULL;
502 tls_advance_record_sn(sk, &tls_ctx->tx);
3c4d7559
DW		 503 return rc;
504}
505
506static int tls_push_record(struct sock *sk, int flags,
507 unsigned char record_type)
508{
509 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 510 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8 511 struct tls_rec *rec = ctx->open_rec;
a447da7d 512 struct aead_request *req;
3c4d7559
DW		 513 int rc;
514
a42055e8
VG		 515 if (!rec)
516 return 0;
a447da7d 517
a42055e8
VG		 518 rec->tx_flags = flags;
519 req = &rec->aead_req;
3c4d7559 520
a42055e8
VG		 521 sg_mark_end(rec->sg_plaintext_data + rec->sg_plaintext_num_elem - 1);
522 sg_mark_end(rec->sg_encrypted_data + rec->sg_encrypted_num_elem - 1);
523
524 tls_make_aad(rec->aad_space, rec->sg_plaintext_size,
dbe42559 525 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
3c4d7559
DW		 526 record_type);
527
528 tls_fill_prepend(tls_ctx,
a42055e8
VG		 529 page_address(sg_page(&rec->sg_encrypted_data[0])) +
530 rec->sg_encrypted_data[0].offset,
531 rec->sg_plaintext_size, record_type);
3c4d7559
DW		 532
533 tls_ctx->pending_open_record_frags = 0;
3c4d7559 534
a42055e8
VG		 535 rc = tls_do_encryption(sk, tls_ctx, ctx, req, rec->sg_plaintext_size);
536 if (rc == -EINPROGRESS)
537 return -EINPROGRESS;
3c4d7559 538
a42055e8
VG		 539 free_sg(sk, rec->sg_plaintext_data, &rec->sg_plaintext_num_elem,
540 &rec->sg_plaintext_size);
3c4d7559 541
a42055e8 542 if (rc < 0) {
f4a8e43f 543 tls_err_abort(sk, EBADMSG);
a42055e8
VG		 544 return rc;
545 }
3c4d7559 546
a42055e8
VG		 547 /* Put the record in tx_ready_list and start tx if permitted.
548 * This happens only when encryption is not asynchronous.
549 */
550 if (append_tx_ready_list(tls_ctx, ctx, rec))
551 return tls_tx_records(sk, flags);
552
553 return 0;
3c4d7559
DW		 554}
555
556static int tls_sw_push_pending_record(struct sock *sk, int flags)
557{
558 return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
559}
560
561static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
69ca9293
DW		 562 int length, int *pages_used,
563 unsigned int *size_used,
564 struct scatterlist *to, int to_max_pages,
2da19ed3 565 bool charge)
3c4d7559 566{
3c4d7559
DW		 567 struct page *pages[MAX_SKB_FRAGS];
568
569 size_t offset;
570 ssize_t copied, use;
571 int i = 0;
69ca9293
DW		 572 unsigned int size = *size_used;
573 int num_elem = *pages_used;
3c4d7559
DW		 574 int rc = 0;
575 int maxpages;
576
577 while (length > 0) {
578 i = 0;
69ca9293 579 maxpages = to_max_pages - num_elem;
3c4d7559
DW		 580 if (maxpages == 0) {
581 rc = -EFAULT;
582 goto out;
583 }
584 copied = iov_iter_get_pages(from, pages,
585 length,
586 maxpages, &offset);
587 if (copied <= 0) {
588 rc = -EFAULT;
589 goto out;
590 }
591
592 iov_iter_advance(from, copied);
593
594 length -= copied;
595 size += copied;
596 while (copied) {
597 use = min_t(int, copied, PAGE_SIZE - offset);
598
69ca9293 599 sg_set_page(&to[num_elem],
3c4d7559 600 pages[i], use, offset);
69ca9293
DW		 601 sg_unmark_end(&to[num_elem]);
602 if (charge)
603 sk_mem_charge(sk, use);
3c4d7559
DW		 604
605 offset = 0;
606 copied -= use;
607
608 ++i;
609 ++num_elem;
610 }
611 }
612
cfb4099f
VG		 613 /* Mark the end in the last sg entry if newly added */
614 if (num_elem > *pages_used)
615 sg_mark_end(&to[num_elem - 1]);
3c4d7559 616out:
2da19ed3
DRK		 617 if (rc)
618 iov_iter_revert(from, size - *size_used);
69ca9293
DW		 619 *size_used = size;
620 *pages_used = num_elem;
621
3c4d7559
DW		 622 return rc;
623}
624
625static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
626 int bytes)
627{
628 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 629 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8
VG		 630 struct tls_rec *rec = ctx->open_rec;
631 struct scatterlist *sg = rec->sg_plaintext_data;
3c4d7559
DW		 632 int copy, i, rc = 0;
633
634 for (i = tls_ctx->pending_open_record_frags;
a42055e8 635 i < rec->sg_plaintext_num_elem; ++i) {
3c4d7559
DW		 636 copy = sg[i].length;
637 if (copy_from_iter(
638 page_address(sg_page(&sg[i])) + sg[i].offset,
639 copy, from) != copy) {
640 rc = -EFAULT;
641 goto out;
642 }
643 bytes -= copy;
644
645 ++tls_ctx->pending_open_record_frags;
646
647 if (!bytes)
648 break;
649 }
650
651out:
652 return rc;
653}
654
a42055e8 655struct tls_rec *get_rec(struct sock *sk)
3c4d7559
DW		 656{
657 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 658 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8
VG		 659 struct tls_rec *rec;
660 int mem_size;
661
662 /* Return if we already have an open record */
663 if (ctx->open_rec)
664 return ctx->open_rec;
665
666 mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send);
667
668 rec = kzalloc(mem_size, sk->sk_allocation);
669 if (!rec)
670 return NULL;
671
672 sg_init_table(&rec->sg_plaintext_data[0],
673 ARRAY_SIZE(rec->sg_plaintext_data));
674 sg_init_table(&rec->sg_encrypted_data[0],
675 ARRAY_SIZE(rec->sg_encrypted_data));
676
677 sg_init_table(rec->sg_aead_in, 2);
678 sg_set_buf(&rec->sg_aead_in[0], rec->aad_space,
679 sizeof(rec->aad_space));
680 sg_unmark_end(&rec->sg_aead_in[1]);
681 sg_chain(rec->sg_aead_in, 2, rec->sg_plaintext_data);
682
683 sg_init_table(rec->sg_aead_out, 2);
684 sg_set_buf(&rec->sg_aead_out[0], rec->aad_space,
685 sizeof(rec->aad_space));
686 sg_unmark_end(&rec->sg_aead_out[1]);
687 sg_chain(rec->sg_aead_out, 2, rec->sg_encrypted_data);
688
689 ctx->open_rec = rec;
690
691 return rec;
692}
693
694int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
695{
3c4d7559 696 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
a42055e8
VG		 697 struct tls_context *tls_ctx = tls_get_ctx(sk);
698 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
699 struct crypto_tfm *tfm = crypto_aead_tfm(ctx->aead_send);
700 bool async_capable = tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC;
701 unsigned char record_type = TLS_RECORD_TYPE_DATA;
702 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
3c4d7559
DW		 703 bool eor = !(msg->msg_flags & MSG_MORE);
704 size_t try_to_copy, copied = 0;
a42055e8
VG		 705 struct tls_rec *rec;
706 int required_size;
707 int num_async = 0;
3c4d7559 708 bool full_record;
a42055e8
VG		 709 int record_room;
710 int num_zc = 0;
3c4d7559 711 int orig_size;
a42055e8 712 int ret;
3c4d7559
DW		 713
714 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
715 return -ENOTSUPP;
716
717 lock_sock(sk);
718
a42055e8
VG		 719 /* Wait till there is any pending write on socket */
720 if (unlikely(sk->sk_write_pending)) {
721 ret = wait_on_pending_writer(sk, &timeo);
722 if (unlikely(ret))
723 goto send_end;
724 }
3c4d7559
DW		 725
726 if (unlikely(msg->msg_controllen)) {
727 ret = tls_proccess_cmsg(sk, msg, &record_type);
a42055e8
VG		 728 if (ret) {
729 if (ret == -EINPROGRESS)
730 num_async++;
731 else if (ret != -EAGAIN)
732 goto send_end;
733 }
3c4d7559
DW		 734 }
735
736 while (msg_data_left(msg)) {
737 if (sk->sk_err) {
30be8f8d 738 ret = -sk->sk_err;
3c4d7559
DW		 739 goto send_end;
740 }
741
a42055e8
VG		 742 rec = get_rec(sk);
743 if (!rec) {
744 ret = -ENOMEM;
745 goto send_end;
746 }
747
748 orig_size = rec->sg_plaintext_size;
3c4d7559
DW		 749 full_record = false;
750 try_to_copy = msg_data_left(msg);
a42055e8 751 record_room = TLS_MAX_PAYLOAD_SIZE - rec->sg_plaintext_size;
3c4d7559
DW		 752 if (try_to_copy >= record_room) {
753 try_to_copy = record_room;
754 full_record = true;
755 }
756
a42055e8 757 required_size = rec->sg_plaintext_size + try_to_copy +
dbe42559 758 tls_ctx->tx.overhead_size;
3c4d7559
DW		 759
760 if (!sk_stream_memory_free(sk))
761 goto wait_for_sndbuf;
a42055e8 762
3c4d7559
DW		 763alloc_encrypted:
764 ret = alloc_encrypted_sg(sk, required_size);
765 if (ret) {
766 if (ret != -ENOSPC)
767 goto wait_for_memory;
768
769 /* Adjust try_to_copy according to the amount that was
770 * actually allocated. The difference is due
771 * to max sg elements limit
772 */
a42055e8 773 try_to_copy -= required_size - rec->sg_encrypted_size;
3c4d7559
DW		 774 full_record = true;
775 }
a42055e8
VG		 776
777 if (!is_kvec && (full_record || eor) && !async_capable) {
3c4d7559 778 ret = zerocopy_from_iter(sk, &msg->msg_iter,
a42055e8
VG		 779 try_to_copy, &rec->sg_plaintext_num_elem,
780 &rec->sg_plaintext_size,
781 rec->sg_plaintext_data,
782 ARRAY_SIZE(rec->sg_plaintext_data),
2da19ed3 783 true);
3c4d7559
DW		 784 if (ret)
785 goto fallback_to_reg_send;
786
a42055e8 787 num_zc++;
3c4d7559
DW		 788 copied += try_to_copy;
789 ret = tls_push_record(sk, msg->msg_flags, record_type);
a42055e8
VG		 790 if (ret) {
791 if (ret == -EINPROGRESS)
792 num_async++;
793 else if (ret != -EAGAIN)
794 goto send_end;
795 }
5a3611ef 796 continue;
3c4d7559 797
3c4d7559 798fallback_to_reg_send:
a42055e8
VG		 799 trim_sg(sk, rec->sg_plaintext_data,
800 &rec->sg_plaintext_num_elem,
801 &rec->sg_plaintext_size,
3c4d7559
DW		 802 orig_size);
803 }
804
a42055e8 805 required_size = rec->sg_plaintext_size + try_to_copy;
3c4d7559
DW		 806alloc_plaintext:
807 ret = alloc_plaintext_sg(sk, required_size);
808 if (ret) {
809 if (ret != -ENOSPC)
810 goto wait_for_memory;
811
812 /* Adjust try_to_copy according to the amount that was
813 * actually allocated. The difference is due
814 * to max sg elements limit
815 */
a42055e8 816 try_to_copy -= required_size - rec->sg_plaintext_size;
3c4d7559
DW		 817 full_record = true;
818
a42055e8
VG		 819 trim_sg(sk, rec->sg_encrypted_data,
820 &rec->sg_encrypted_num_elem,
821 &rec->sg_encrypted_size,
822 rec->sg_plaintext_size +
dbe42559 823 tls_ctx->tx.overhead_size);
3c4d7559
DW		 824 }
825
826 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
827 if (ret)
828 goto trim_sgl;
829
830 copied += try_to_copy;
831 if (full_record || eor) {
3c4d7559
DW		 832 ret = tls_push_record(sk, msg->msg_flags, record_type);
833 if (ret) {
a42055e8
VG		 834 if (ret == -EINPROGRESS)
835 num_async++;
836 else if (ret != -EAGAIN)
837 goto send_end;
3c4d7559
DW		 838 }
839 }
840
841 continue;
842
843wait_for_sndbuf:
844 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
845wait_for_memory:
846 ret = sk_stream_wait_memory(sk, &timeo);
847 if (ret) {
848trim_sgl:
849 trim_both_sgl(sk, orig_size);
850 goto send_end;
851 }
852
a42055e8 853 if (rec->sg_encrypted_size < required_size)
3c4d7559
DW		 854 goto alloc_encrypted;
855
856 goto alloc_plaintext;
857 }
858
a42055e8
VG		 859 if (!num_async) {
860 goto send_end;
861 } else if (num_zc) {
862 /* Wait for pending encryptions to get completed */
863 smp_store_mb(ctx->async_notify, true);
864
865 if (atomic_read(&ctx->encrypt_pending))
866 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
867 else
868 reinit_completion(&ctx->async_wait.completion);
869
870 WRITE_ONCE(ctx->async_notify, false);
871
872 if (ctx->async_wait.err) {
873 ret = ctx->async_wait.err;
874 copied = 0;
875 }
876 }
877
878 /* Transmit if any encryptions have completed */
879 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
880 cancel_delayed_work(&ctx->tx_work.work);
881 tls_tx_records(sk, msg->msg_flags);
882 }
883
3c4d7559
DW		 884send_end:
885 ret = sk_stream_error(sk, msg->msg_flags, ret);
886
887 release_sock(sk);
888 return copied ? copied : ret;
889}
890
891int tls_sw_sendpage(struct sock *sk, struct page *page,
892 int offset, size_t size, int flags)
893{
a42055e8 894 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
3c4d7559 895 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 896 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559 897 unsigned char record_type = TLS_RECORD_TYPE_DATA;
a42055e8 898 size_t orig_size = size;
3c4d7559 899 struct scatterlist *sg;
a42055e8
VG		 900 struct tls_rec *rec;
901 int num_async = 0;
3c4d7559
DW		 902 bool full_record;
903 int record_room;
a42055e8
VG		 904 bool eor;
905 int ret;
3c4d7559
DW		 906
907 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
908 MSG_SENDPAGE_NOTLAST))
909 return -ENOTSUPP;
910
911 /* No MSG_EOR from splice, only look at MSG_MORE */
912 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
913
914 lock_sock(sk);
915
916 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
917
a42055e8
VG		 918 /* Wait till there is any pending write on socket */
919 if (unlikely(sk->sk_write_pending)) {
920 ret = wait_on_pending_writer(sk, &timeo);
921 if (unlikely(ret))
922 goto sendpage_end;
923 }
3c4d7559
DW		 924
925 /* Call the sk_stream functions to manage the sndbuf mem. */
926 while (size > 0) {
927 size_t copy, required_size;
928
929 if (sk->sk_err) {
30be8f8d 930 ret = -sk->sk_err;
3c4d7559
DW		 931 goto sendpage_end;
932 }
933
a42055e8
VG		 934 rec = get_rec(sk);
935 if (!rec) {
936 ret = -ENOMEM;
937 goto sendpage_end;
938 }
939
3c4d7559 940 full_record = false;
a42055e8 941 record_room = TLS_MAX_PAYLOAD_SIZE - rec->sg_plaintext_size;
3c4d7559
DW		 942 copy = size;
943 if (copy >= record_room) {
944 copy = record_room;
945 full_record = true;
946 }
a42055e8 947 required_size = rec->sg_plaintext_size + copy +
dbe42559 948 tls_ctx->tx.overhead_size;
3c4d7559
DW		 949
950 if (!sk_stream_memory_free(sk))
951 goto wait_for_sndbuf;
952alloc_payload:
953 ret = alloc_encrypted_sg(sk, required_size);
954 if (ret) {
955 if (ret != -ENOSPC)
956 goto wait_for_memory;
957
958 /* Adjust copy according to the amount that was
959 * actually allocated. The difference is due
960 * to max sg elements limit
961 */
a42055e8 962 copy -= required_size - rec->sg_plaintext_size;
3c4d7559
DW		 963 full_record = true;
964 }
965
966 get_page(page);
a42055e8 967 sg = rec->sg_plaintext_data + rec->sg_plaintext_num_elem;
3c4d7559 968 sg_set_page(sg, page, copy, offset);
7a8c4dd9
DW		 969 sg_unmark_end(sg);
970
a42055e8 971 rec->sg_plaintext_num_elem++;
3c4d7559
DW		 972
973 sk_mem_charge(sk, copy);
974 offset += copy;
975 size -= copy;
a42055e8
VG		 976 rec->sg_plaintext_size += copy;
977 tls_ctx->pending_open_record_frags = rec->sg_plaintext_num_elem;
3c4d7559
DW		 978
979 if (full_record || eor ||
a42055e8
VG		 980 rec->sg_plaintext_num_elem ==
981 ARRAY_SIZE(rec->sg_plaintext_data)) {
3c4d7559
DW		 982 ret = tls_push_record(sk, flags, record_type);
983 if (ret) {
a42055e8
VG		 984 if (ret == -EINPROGRESS)
985 num_async++;
986 else if (ret != -EAGAIN)
987 goto sendpage_end;
3c4d7559
DW		 988 }
989 }
990 continue;
991wait_for_sndbuf:
992 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
993wait_for_memory:
994 ret = sk_stream_wait_memory(sk, &timeo);
995 if (ret) {
a42055e8 996 trim_both_sgl(sk, rec->sg_plaintext_size);
3c4d7559
DW		 997 goto sendpage_end;
998 }
999
3c4d7559
DW		 1000 goto alloc_payload;
1001 }
1002
a42055e8
VG		 1003 if (num_async) {
1004 /* Transmit if any encryptions have completed */
1005 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1006 cancel_delayed_work(&ctx->tx_work.work);
1007 tls_tx_records(sk, flags);
1008 }
1009 }
3c4d7559
DW		 1010sendpage_end:
1011 if (orig_size > size)
1012 ret = orig_size - size;
1013 else
1014 ret = sk_stream_error(sk, flags, ret);
1015
1016 release_sock(sk);
1017 return ret;
1018}
1019
c46234eb
DW		 1020static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
1021 long timeo, int *err)
1022{
1023 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1024 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1025 struct sk_buff *skb;
1026 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1027
1028 while (!(skb = ctx->recv_pkt)) {
1029 if (sk->sk_err) {
1030 *err = sock_error(sk);
1031 return NULL;
1032 }
1033
fcf4793e
DRK		 1034 if (sk->sk_shutdown & RCV_SHUTDOWN)
1035 return NULL;
1036
c46234eb
DW		 1037 if (sock_flag(sk, SOCK_DONE))
1038 return NULL;
1039
1040 if ((flags & MSG_DONTWAIT) || !timeo) {
1041 *err = -EAGAIN;
1042 return NULL;
1043 }
1044
1045 add_wait_queue(sk_sleep(sk), &wait);
1046 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1047 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait);
1048 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1049 remove_wait_queue(sk_sleep(sk), &wait);
1050
1051 /* Handle signals */
1052 if (signal_pending(current)) {
1053 *err = sock_intr_errno(timeo);
1054 return NULL;
1055 }
1056 }
1057
1058 return skb;
1059}
1060
0b243d00
VG		 1061/* This function decrypts the input skb into either out_iov or in out_sg
1062 * or in skb buffers itself. The input parameter 'zc' indicates if
1063 * zero-copy mode needs to be tried or not. With zero-copy mode, either
1064 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
1065 * NULL, then the decryption happens inside skb buffers itself, i.e.
1066 * zero-copy gets disabled and 'zc' is updated.
1067 */
1068
1069static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
1070 struct iov_iter *out_iov,
1071 struct scatterlist *out_sg,
1072 int *chunk, bool *zc)
1073{
1074 struct tls_context *tls_ctx = tls_get_ctx(sk);
1075 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1076 struct strp_msg *rxm = strp_msg(skb);
1077 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0;
1078 struct aead_request *aead_req;
1079 struct sk_buff *unused;
1080 u8 *aad, *iv, *mem = NULL;
1081 struct scatterlist *sgin = NULL;
1082 struct scatterlist *sgout = NULL;
1083 const int data_len = rxm->full_len - tls_ctx->rx.overhead_size;
1084
1085 if (*zc && (out_iov || out_sg)) {
1086 if (out_iov)
1087 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
1088 else
1089 n_sgout = sg_nents(out_sg);
0927f71d
DRK		 1090 n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size,
1091 rxm->full_len - tls_ctx->rx.prepend_size);
0b243d00
VG		 1092 } else {
1093 n_sgout = 0;
1094 *zc = false;
0927f71d 1095 n_sgin = skb_cow_data(skb, 0, &unused);
0b243d00
VG		 1096 }
1097
0b243d00
VG		 1098 if (n_sgin < 1)
1099 return -EBADMSG;
1100
1101 /* Increment to accommodate AAD */
1102 n_sgin = n_sgin + 1;
1103
1104 nsg = n_sgin + n_sgout;
1105
1106 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
1107 mem_size = aead_size + (nsg * sizeof(struct scatterlist));
1108 mem_size = mem_size + TLS_AAD_SPACE_SIZE;
1109 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv);
1110
1111 /* Allocate a single block of memory which contains
1112 * aead_req || sgin[] || sgout[] || aad || iv.
1113 * This order achieves correct alignment for aead_req, sgin, sgout.
1114 */
1115 mem = kmalloc(mem_size, sk->sk_allocation);
1116 if (!mem)
1117 return -ENOMEM;
1118
1119 /* Segment the allocated memory */
1120 aead_req = (struct aead_request *)mem;
1121 sgin = (struct scatterlist *)(mem + aead_size);
1122 sgout = sgin + n_sgin;
1123 aad = (u8 *)(sgout + n_sgout);
1124 iv = aad + TLS_AAD_SPACE_SIZE;
1125
1126 /* Prepare IV */
1127 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
1128 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1129 tls_ctx->rx.iv_size);
1130 if (err < 0) {
1131 kfree(mem);
1132 return err;
1133 }
1134 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1135
1136 /* Prepare AAD */
1137 tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size,
1138 tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size,
1139 ctx->control);
1140
1141 /* Prepare sgin */
1142 sg_init_table(sgin, n_sgin);
1143 sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE);
1144 err = skb_to_sgvec(skb, &sgin[1],
1145 rxm->offset + tls_ctx->rx.prepend_size,
1146 rxm->full_len - tls_ctx->rx.prepend_size);
1147 if (err < 0) {
1148 kfree(mem);
1149 return err;
1150 }
1151
1152 if (n_sgout) {
1153 if (out_iov) {
1154 sg_init_table(sgout, n_sgout);
1155 sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE);
1156
1157 *chunk = 0;
1158 err = zerocopy_from_iter(sk, out_iov, data_len, &pages,
1159 chunk, &sgout[1],
1160 (n_sgout - 1), false);
1161 if (err < 0)
1162 goto fallback_to_reg_recv;
1163 } else if (out_sg) {
1164 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
1165 } else {
1166 goto fallback_to_reg_recv;
1167 }
1168 } else {
1169fallback_to_reg_recv:
1170 sgout = sgin;
1171 pages = 0;
1172 *chunk = 0;
1173 *zc = false;
1174 }
1175
1176 /* Prepare and submit AEAD request */
94524d8f
VG		 1177 err = tls_do_decryption(sk, skb, sgin, sgout, iv,
1178 data_len, aead_req, *zc);
1179 if (err == -EINPROGRESS)
1180 return err;
0b243d00
VG		 1181
1182 /* Release the pages in case iov was mapped to pages */
1183 for (; pages > 0; pages--)
1184 put_page(sg_page(&sgout[pages]));
1185
1186 kfree(mem);
1187 return err;
1188}
1189
dafb67f3 1190static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
0b243d00 1191 struct iov_iter *dest, int *chunk, bool *zc)
dafb67f3
BP		 1192{
1193 struct tls_context *tls_ctx = tls_get_ctx(sk);
1194 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1195 struct strp_msg *rxm = strp_msg(skb);
1196 int err = 0;
1197
4799ac81
BP		 1198#ifdef CONFIG_TLS_DEVICE
1199 err = tls_device_decrypted(sk, skb);
dafb67f3
BP		 1200 if (err < 0)
1201 return err;
4799ac81
BP		 1202#endif
1203 if (!ctx->decrypted) {
0b243d00 1204 err = decrypt_internal(sk, skb, dest, NULL, chunk, zc);
94524d8f
VG		 1205 if (err < 0) {
1206 if (err == -EINPROGRESS)
1207 tls_advance_record_sn(sk, &tls_ctx->rx);
1208
4799ac81 1209 return err;
94524d8f 1210 }
4799ac81
BP		 1211 } else {
1212 *zc = false;
1213 }
dafb67f3
BP		 1214
1215 rxm->offset += tls_ctx->rx.prepend_size;
1216 rxm->full_len -= tls_ctx->rx.overhead_size;
1217 tls_advance_record_sn(sk, &tls_ctx->rx);
1218 ctx->decrypted = true;
1219 ctx->saved_data_ready(sk);
1220
1221 return err;
1222}
1223
1224int decrypt_skb(struct sock *sk, struct sk_buff *skb,
1225 struct scatterlist *sgout)
c46234eb 1226{
0b243d00
VG		 1227 bool zc = true;
1228 int chunk;
c46234eb 1229
0b243d00 1230 return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc);
c46234eb
DW		 1231}
1232
1233static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
1234 unsigned int len)
1235{
1236 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1237 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 1238
94524d8f
VG		 1239 if (skb) {
1240 struct strp_msg *rxm = strp_msg(skb);
c46234eb 1241
94524d8f
VG		 1242 if (len < rxm->full_len) {
1243 rxm->offset += len;
1244 rxm->full_len -= len;
1245 return false;
1246 }
1247 kfree_skb(skb);
c46234eb
DW		 1248 }
1249
1250 /* Finished with message */
1251 ctx->recv_pkt = NULL;
7170e604 1252 __strp_unpause(&ctx->strp);
c46234eb
DW		 1253
1254 return true;
1255}
1256
1257int tls_sw_recvmsg(struct sock *sk,
1258 struct msghdr *msg,
1259 size_t len,
1260 int nonblock,
1261 int flags,
1262 int *addr_len)
1263{
1264 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1265 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1266 unsigned char control;
1267 struct strp_msg *rxm;
1268 struct sk_buff *skb;
1269 ssize_t copied = 0;
1270 bool cmsg = false;
06030dba 1271 int target, err = 0;
c46234eb 1272 long timeo;
0a26cf3f 1273 bool is_kvec = msg->msg_iter.type & ITER_KVEC;
94524d8f 1274 int num_async = 0;
c46234eb
DW		 1275
1276 flags |= nonblock;
1277
1278 if (unlikely(flags & MSG_ERRQUEUE))
1279 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
1280
1281 lock_sock(sk);
1282
06030dba 1283 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
c46234eb
DW		 1284 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1285 do {
1286 bool zc = false;
94524d8f 1287 bool async = false;
c46234eb
DW		 1288 int chunk = 0;
1289
1290 skb = tls_wait_data(sk, flags, timeo, &err);
1291 if (!skb)
1292 goto recv_end;
1293
1294 rxm = strp_msg(skb);
94524d8f 1295
c46234eb
DW		 1296 if (!cmsg) {
1297 int cerr;
1298
1299 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
1300 sizeof(ctx->control), &ctx->control);
1301 cmsg = true;
1302 control = ctx->control;
1303 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1304 if (cerr || msg->msg_flags & MSG_CTRUNC) {
1305 err = -EIO;
1306 goto recv_end;
1307 }
1308 }
1309 } else if (control != ctx->control) {
1310 goto recv_end;
1311 }
1312
1313 if (!ctx->decrypted) {
0b243d00 1314 int to_copy = rxm->full_len - tls_ctx->rx.overhead_size;
c46234eb 1315
0b243d00
VG		 1316 if (!is_kvec && to_copy <= len &&
1317 likely(!(flags & MSG_PEEK)))
c46234eb 1318 zc = true;
0b243d00
VG		 1319
1320 err = decrypt_skb_update(sk, skb, &msg->msg_iter,
1321 &chunk, &zc);
94524d8f 1322 if (err < 0 && err != -EINPROGRESS) {
0b243d00
VG		 1323 tls_err_abort(sk, EBADMSG);
1324 goto recv_end;
c46234eb 1325 }
94524d8f
VG		 1326
1327 if (err == -EINPROGRESS) {
1328 async = true;
1329 num_async++;
1330 goto pick_next_record;
1331 }
1332
c46234eb
DW		 1333 ctx->decrypted = true;
1334 }
1335
1336 if (!zc) {
1337 chunk = min_t(unsigned int, rxm->full_len, len);
94524d8f 1338
c46234eb
DW		 1339 err = skb_copy_datagram_msg(skb, rxm->offset, msg,
1340 chunk);
1341 if (err < 0)
1342 goto recv_end;
1343 }
1344
94524d8f 1345pick_next_record:
c46234eb
DW		 1346 copied += chunk;
1347 len -= chunk;
1348 if (likely(!(flags & MSG_PEEK))) {
1349 u8 control = ctx->control;
1350
94524d8f
VG		 1351 /* For async, drop current skb reference */
1352 if (async)
1353 skb = NULL;
1354
c46234eb
DW		 1355 if (tls_sw_advance_skb(sk, skb, chunk)) {
1356 /* Return full control message to
1357 * userspace before trying to parse
1358 * another message type
1359 */
1360 msg->msg_flags |= MSG_EOR;
1361 if (control != TLS_RECORD_TYPE_DATA)
1362 goto recv_end;
94524d8f
VG		 1363 } else {
1364 break;
c46234eb 1365 }
50c6b58a
DB		 1366 } else {
1367 /* MSG_PEEK right now cannot look beyond current skb
1368 * from strparser, meaning we cannot advance skb here
1369 * and thus unpause strparser since we'd loose original
1370 * one.
1371 */
1372 break;
c46234eb 1373 }
94524d8f 1374
06030dba
DB		 1375 /* If we have a new message from strparser, continue now. */
1376 if (copied >= target && !ctx->recv_pkt)
1377 break;
c46234eb
DW		 1378 } while (len);
1379
1380recv_end:
94524d8f
VG		 1381 if (num_async) {
1382 /* Wait for all previously submitted records to be decrypted */
1383 smp_store_mb(ctx->async_notify, true);
1384 if (atomic_read(&ctx->decrypt_pending)) {
1385 err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1386 if (err) {
1387 /* one of async decrypt failed */
1388 tls_err_abort(sk, err);
1389 copied = 0;
1390 }
1391 } else {
1392 reinit_completion(&ctx->async_wait.completion);
1393 }
1394 WRITE_ONCE(ctx->async_notify, false);
1395 }
1396
c46234eb
DW		 1397 release_sock(sk);
1398 return copied ? : err;
1399}
1400
1401ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
1402 struct pipe_inode_info *pipe,
1403 size_t len, unsigned int flags)
1404{
1405 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
f66de3ee 1406 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1407 struct strp_msg *rxm = NULL;
1408 struct sock *sk = sock->sk;
1409 struct sk_buff *skb;
1410 ssize_t copied = 0;
1411 int err = 0;
1412 long timeo;
1413 int chunk;
0b243d00 1414 bool zc = false;
c46234eb
DW		 1415
1416 lock_sock(sk);
1417
1418 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1419
1420 skb = tls_wait_data(sk, flags, timeo, &err);
1421 if (!skb)
1422 goto splice_read_end;
1423
1424 /* splice does not support reading control messages */
1425 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1426 err = -ENOTSUPP;
1427 goto splice_read_end;
1428 }
1429
1430 if (!ctx->decrypted) {
0b243d00 1431 err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc);
c46234eb
DW		 1432
1433 if (err < 0) {
1434 tls_err_abort(sk, EBADMSG);
1435 goto splice_read_end;
1436 }
1437 ctx->decrypted = true;
1438 }
1439 rxm = strp_msg(skb);
1440
1441 chunk = min_t(unsigned int, rxm->full_len, len);
1442 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
1443 if (copied < 0)
1444 goto splice_read_end;
1445
1446 if (likely(!(flags & MSG_PEEK)))
1447 tls_sw_advance_skb(sk, skb, copied);
1448
1449splice_read_end:
1450 release_sock(sk);
1451 return copied ? : err;
1452}
1453
a11e1d43
LT		 1454unsigned int tls_sw_poll(struct file *file, struct socket *sock,
1455 struct poll_table_struct *wait)
c46234eb 1456{
a11e1d43 1457 unsigned int ret;
c46234eb
DW		 1458 struct sock *sk = sock->sk;
1459 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1460 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 1461
a11e1d43
LT		 1462 /* Grab POLLOUT and POLLHUP from the underlying socket */
1463 ret = ctx->sk_poll(file, sock, wait);
c46234eb 1464
a11e1d43
LT		 1465 /* Clear POLLIN bits, and set based on recv_pkt */
1466 ret &= ~(POLLIN | POLLRDNORM);
c46234eb 1467 if (ctx->recv_pkt)
a11e1d43 1468 ret |= POLLIN | POLLRDNORM;
c46234eb 1469
a11e1d43 1470 return ret;
c46234eb
DW		 1471}
1472
1473static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
1474{
1475 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 1476 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
3463e51d 1477 char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
c46234eb
DW		 1478 struct strp_msg *rxm = strp_msg(skb);
1479 size_t cipher_overhead;
1480 size_t data_len = 0;
1481 int ret;
1482
1483 /* Verify that we have a full TLS header, or wait for more data */
1484 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
1485 return 0;
1486
3463e51d
KC		 1487 /* Sanity-check size of on-stack buffer. */
1488 if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) {
1489 ret = -EINVAL;
1490 goto read_failure;
1491 }
1492
c46234eb
DW		 1493 /* Linearize header to local buffer */
1494 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
1495
1496 if (ret < 0)
1497 goto read_failure;
1498
1499 ctx->control = header[0];
1500
1501 data_len = ((header[4] & 0xFF) | (header[3] << 8));
1502
1503 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
1504
1505 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
1506 ret = -EMSGSIZE;
1507 goto read_failure;
1508 }
1509 if (data_len < cipher_overhead) {
1510 ret = -EBADMSG;
1511 goto read_failure;
1512 }
1513
86029d10
SD		 1514 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.info.version) ||
1515 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.info.version)) {
c46234eb
DW		 1516 ret = -EINVAL;
1517 goto read_failure;
1518 }
1519
4799ac81
BP		 1520#ifdef CONFIG_TLS_DEVICE
1521 handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset,
1522 *(u64*)tls_ctx->rx.rec_seq);
1523#endif
c46234eb
DW		 1524 return data_len + TLS_HEADER_SIZE;
1525
1526read_failure:
1527 tls_err_abort(strp->sk, ret);
1528
1529 return ret;
1530}
1531
1532static void tls_queue(struct strparser *strp, struct sk_buff *skb)
1533{
1534 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 1535 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1536
1537 ctx->decrypted = false;
1538
1539 ctx->recv_pkt = skb;
1540 strp_pause(strp);
1541
ad13acce 1542 ctx->saved_data_ready(strp->sk);
c46234eb
DW		 1543}
1544
1545static void tls_data_ready(struct sock *sk)
1546{
1547 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1548 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1549
1550 strp_data_ready(&ctx->strp);
1551}
1552
f66de3ee 1553void tls_sw_free_resources_tx(struct sock *sk)
3c4d7559
DW		 1554{
1555 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1556 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a42055e8
VG		 1557 struct tls_rec *rec, *tmp;
1558
1559 /* Wait for any pending async encryptions to complete */
1560 smp_store_mb(ctx->async_notify, true);
1561 if (atomic_read(&ctx->encrypt_pending))
1562 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1563
1564 cancel_delayed_work_sync(&ctx->tx_work.work);
1565
1566 /* Tx whatever records we can transmit and abandon the rest */
1567 tls_tx_records(sk, -1);
1568
1569 /* Free up un-sent records in tx_ready_list. First, free
1570 * the partially sent record if any at head of tx_list.
1571 */
1572 if (tls_ctx->partially_sent_record) {
1573 struct scatterlist *sg = tls_ctx->partially_sent_record;
1574
1575 while (1) {
1576 put_page(sg_page(sg));
1577 sk_mem_uncharge(sk, sg->length);
1578
1579 if (sg_is_last(sg))
1580 break;
1581 sg++;
1582 }
1583
1584 tls_ctx->partially_sent_record = NULL;
1585
1586 rec = list_first_entry(&ctx->tx_ready_list,
1587 struct tls_rec, list);
1588 list_del(&rec->list);
1589 kfree(rec);
1590 }
1591
1592 list_for_each_entry_safe(rec, tmp, &ctx->tx_ready_list, list) {
1593 free_sg(sk, rec->sg_encrypted_data,
1594 &rec->sg_encrypted_num_elem,
1595 &rec->sg_encrypted_size);
1596
1597 list_del(&rec->list);
1598 kfree(rec);
1599 }
3c4d7559 1600
201876b3 1601 crypto_free_aead(ctx->aead_send);
f66de3ee
BP		 1602 tls_free_both_sg(sk);
1603
1604 kfree(ctx);
1605}
1606
39f56e1a 1607void tls_sw_release_resources_rx(struct sock *sk)
f66de3ee
BP		 1608{
1609 struct tls_context *tls_ctx = tls_get_ctx(sk);
1610 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1611
c46234eb 1612 if (ctx->aead_recv) {
201876b3
VG		 1613 kfree_skb(ctx->recv_pkt);
1614 ctx->recv_pkt = NULL;
c46234eb
DW		 1615 crypto_free_aead(ctx->aead_recv);
1616 strp_stop(&ctx->strp);
1617 write_lock_bh(&sk->sk_callback_lock);
1618 sk->sk_data_ready = ctx->saved_data_ready;
1619 write_unlock_bh(&sk->sk_callback_lock);
1620 release_sock(sk);
1621 strp_done(&ctx->strp);
1622 lock_sock(sk);
1623 }
39f56e1a
BP		 1624}
1625
1626void tls_sw_free_resources_rx(struct sock *sk)
1627{
1628 struct tls_context *tls_ctx = tls_get_ctx(sk);
1629 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1630
1631 tls_sw_release_resources_rx(sk);
3c4d7559 1632
3c4d7559
DW		 1633 kfree(ctx);
1634}
1635
a42055e8
VG		 1636/* The work handler to transmitt the encrypted records in tx_ready_list */
1637static void tx_work_handler(struct work_struct *work)
1638{
1639 struct delayed_work *delayed_work = to_delayed_work(work);
1640 struct tx_work *tx_work = container_of(delayed_work,
1641 struct tx_work, work);
1642 struct sock *sk = tx_work->sk;
1643 struct tls_context *tls_ctx = tls_get_ctx(sk);
1644 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1645
1646 if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
1647 return;
1648
1649 lock_sock(sk);
1650 tls_tx_records(sk, -1);
1651 release_sock(sk);
1652}
1653
c46234eb 1654int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
3c4d7559 1655{
3c4d7559
DW		 1656 struct tls_crypto_info *crypto_info;
1657 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
f66de3ee
BP		 1658 struct tls_sw_context_tx *sw_ctx_tx = NULL;
1659 struct tls_sw_context_rx *sw_ctx_rx = NULL;
c46234eb
DW		 1660 struct cipher_context *cctx;
1661 struct crypto_aead **aead;
1662 struct strp_callbacks cb;
3c4d7559
DW		 1663 u16 nonce_size, tag_size, iv_size, rec_seq_size;
1664 char *iv, *rec_seq;
1665 int rc = 0;
1666
1667 if (!ctx) {
1668 rc = -EINVAL;
1669 goto out;
1670 }
1671
f66de3ee 1672 if (tx) {
b190a587
BP		 1673 if (!ctx->priv_ctx_tx) {
1674 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
1675 if (!sw_ctx_tx) {
1676 rc = -ENOMEM;
1677 goto out;
1678 }
1679 ctx->priv_ctx_tx = sw_ctx_tx;
1680 } else {
1681 sw_ctx_tx =
1682 (struct tls_sw_context_tx *)ctx->priv_ctx_tx;
c46234eb 1683 }
c46234eb 1684 } else {
b190a587
BP		 1685 if (!ctx->priv_ctx_rx) {
1686 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
1687 if (!sw_ctx_rx) {
1688 rc = -ENOMEM;
1689 goto out;
1690 }
1691 ctx->priv_ctx_rx = sw_ctx_rx;
1692 } else {
1693 sw_ctx_rx =
1694 (struct tls_sw_context_rx *)ctx->priv_ctx_rx;
f66de3ee 1695 }
3c4d7559
DW		 1696 }
1697
c46234eb 1698 if (tx) {
b190a587 1699 crypto_init_wait(&sw_ctx_tx->async_wait);
86029d10 1700 crypto_info = &ctx->crypto_send.info;
c46234eb 1701 cctx = &ctx->tx;
f66de3ee 1702 aead = &sw_ctx_tx->aead_send;
a42055e8
VG		 1703 INIT_LIST_HEAD(&sw_ctx_tx->tx_ready_list);
1704 INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
1705 sw_ctx_tx->tx_work.sk = sk;
c46234eb 1706 } else {
b190a587 1707 crypto_init_wait(&sw_ctx_rx->async_wait);
86029d10 1708 crypto_info = &ctx->crypto_recv.info;
c46234eb 1709 cctx = &ctx->rx;
f66de3ee 1710 aead = &sw_ctx_rx->aead_recv;
c46234eb
DW		 1711 }
1712
3c4d7559
DW		 1713 switch (crypto_info->cipher_type) {
1714 case TLS_CIPHER_AES_GCM_128: {
1715 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1716 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1717 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1718 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
1719 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
1720 rec_seq =
1721 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
1722 gcm_128_info =
1723 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
1724 break;
1725 }
1726 default:
1727 rc = -EINVAL;
cf6d43ef 1728 goto free_priv;
3c4d7559
DW		 1729 }
1730
b16520f7 1731 /* Sanity-check the IV size for stack allocations. */
3463e51d 1732 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) {
b16520f7
KC		 1733 rc = -EINVAL;
1734 goto free_priv;
1735 }
1736
c46234eb
DW		 1737 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
1738 cctx->tag_size = tag_size;
1739 cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
1740 cctx->iv_size = iv_size;
1741 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1742 GFP_KERNEL);
1743 if (!cctx->iv) {
3c4d7559 1744 rc = -ENOMEM;
cf6d43ef 1745 goto free_priv;
3c4d7559 1746 }
c46234eb
DW		 1747 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1748 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
1749 cctx->rec_seq_size = rec_seq_size;
969d5090 1750 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
c46234eb 1751 if (!cctx->rec_seq) {
3c4d7559
DW		 1752 rc = -ENOMEM;
1753 goto free_iv;
1754 }
c46234eb 1755
c46234eb
DW		 1756 if (!*aead) {
1757 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
1758 if (IS_ERR(*aead)) {
1759 rc = PTR_ERR(*aead);
1760 *aead = NULL;
3c4d7559
DW		 1761 goto free_rec_seq;
1762 }
1763 }
1764
1765 ctx->push_pending_record = tls_sw_push_pending_record;
1766
7cba09c6 1767 rc = crypto_aead_setkey(*aead, gcm_128_info->key,
3c4d7559
DW		 1768 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1769 if (rc)
1770 goto free_aead;
1771
c46234eb
DW		 1772 rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
1773 if (rc)
1774 goto free_aead;
1775
f66de3ee 1776 if (sw_ctx_rx) {
c46234eb
DW		 1777 /* Set up strparser */
1778 memset(&cb, 0, sizeof(cb));
1779 cb.rcv_msg = tls_queue;
1780 cb.parse_msg = tls_read_size;
1781
f66de3ee 1782 strp_init(&sw_ctx_rx->strp, sk, &cb);
c46234eb
DW		 1783
1784 write_lock_bh(&sk->sk_callback_lock);
f66de3ee 1785 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
c46234eb
DW		 1786 sk->sk_data_ready = tls_data_ready;
1787 write_unlock_bh(&sk->sk_callback_lock);
1788
a11e1d43 1789 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
c46234eb 1790
f66de3ee 1791 strp_check_rcv(&sw_ctx_rx->strp);
a42055e8
VG		 1792 } else {
1793 ctx->tx_seq_number = be64_to_cpup((const __be64 *)rec_seq);
c46234eb
DW		 1794 }
1795
1796 goto out;
3c4d7559
DW		 1797
1798free_aead:
c46234eb
DW		 1799 crypto_free_aead(*aead);
1800 *aead = NULL;
3c4d7559 1801free_rec_seq:
c46234eb
DW		 1802 kfree(cctx->rec_seq);
1803 cctx->rec_seq = NULL;
3c4d7559 1804free_iv:
f66de3ee
BP		 1805 kfree(cctx->iv);
1806 cctx->iv = NULL;
cf6d43ef 1807free_priv:
f66de3ee
BP		 1808 if (tx) {
1809 kfree(ctx->priv_ctx_tx);
1810 ctx->priv_ctx_tx = NULL;
1811 } else {
1812 kfree(ctx->priv_ctx_rx);
1813 ctx->priv_ctx_rx = NULL;
1814 }
3c4d7559
DW		 1815out:
1816 return rc;
1817}
Linux 6.x block layer and io_uring tree(s)