projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mvebu-dt-4.19-1' of git://git.infradead.org/linux-mvebu into next/dt
[linux-block.git] / net / tls / tls_sw.c
CommitLineData
3c4d7559
DW		 1/*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 */
36
c46234eb 37#include <linux/sched/signal.h>
3c4d7559
DW		 38#include <linux/module.h>
39#include <crypto/aead.h>
40
c46234eb 41#include <net/strparser.h>
3c4d7559
DW		 42#include <net/tls.h>
43
b16520f7
KC		 44#define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
45
c46234eb
DW		 46static int tls_do_decryption(struct sock *sk,
47 struct scatterlist *sgin,
48 struct scatterlist *sgout,
49 char *iv_recv,
50 size_t data_len,
51 struct sk_buff *skb,
52 gfp_t flags)
53{
54 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 55 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 56 struct strp_msg *rxm = strp_msg(skb);
57 struct aead_request *aead_req;
58
59 int ret;
60 unsigned int req_size = sizeof(struct aead_request) +
61 crypto_aead_reqsize(ctx->aead_recv);
62
63 aead_req = kzalloc(req_size, flags);
64 if (!aead_req)
65 return -ENOMEM;
66
67 aead_request_set_tfm(aead_req, ctx->aead_recv);
68 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
69 aead_request_set_crypt(aead_req, sgin, sgout,
70 data_len + tls_ctx->rx.tag_size,
71 (u8 *)iv_recv);
72 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
73 crypto_req_done, &ctx->async_wait);
74
75 ret = crypto_wait_req(crypto_aead_decrypt(aead_req), &ctx->async_wait);
76
77 if (ret < 0)
78 goto out;
79
80 rxm->offset += tls_ctx->rx.prepend_size;
81 rxm->full_len -= tls_ctx->rx.overhead_size;
82 tls_advance_record_sn(sk, &tls_ctx->rx);
83
84 ctx->decrypted = true;
85
86 ctx->saved_data_ready(sk);
87
88out:
89 kfree(aead_req);
90 return ret;
91}
92
3c4d7559
DW		 93static void trim_sg(struct sock *sk, struct scatterlist *sg,
94 int *sg_num_elem, unsigned int *sg_size, int target_size)
95{
96 int i = *sg_num_elem - 1;
97 int trim = *sg_size - target_size;
98
99 if (trim <= 0) {
100 WARN_ON(trim < 0);
101 return;
102 }
103
104 *sg_size = target_size;
105 while (trim >= sg[i].length) {
106 trim -= sg[i].length;
107 sk_mem_uncharge(sk, sg[i].length);
108 put_page(sg_page(&sg[i]));
109 i--;
110
111 if (i < 0)
112 goto out;
113 }
114
115 sg[i].length -= trim;
116 sk_mem_uncharge(sk, trim);
117
118out:
119 *sg_num_elem = i + 1;
120}
121
122static void trim_both_sgl(struct sock *sk, int target_size)
123{
124 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 125 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 126
127 trim_sg(sk, ctx->sg_plaintext_data,
128 &ctx->sg_plaintext_num_elem,
129 &ctx->sg_plaintext_size,
130 target_size);
131
132 if (target_size > 0)
dbe42559 133 target_size += tls_ctx->tx.overhead_size;
3c4d7559
DW		 134
135 trim_sg(sk, ctx->sg_encrypted_data,
136 &ctx->sg_encrypted_num_elem,
137 &ctx->sg_encrypted_size,
138 target_size);
139}
140
3c4d7559
DW		 141static int alloc_encrypted_sg(struct sock *sk, int len)
142{
143 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 144 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 145 int rc = 0;
146
2c3682f0 147 rc = sk_alloc_sg(sk, len,
8c05dbf0 148 ctx->sg_encrypted_data, 0,
2c3682f0
JF		 149 &ctx->sg_encrypted_num_elem,
150 &ctx->sg_encrypted_size, 0);
3c4d7559
DW		 151
152 return rc;
153}
154
155static int alloc_plaintext_sg(struct sock *sk, int len)
156{
157 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 158 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 159 int rc = 0;
160
8c05dbf0 161 rc = sk_alloc_sg(sk, len, ctx->sg_plaintext_data, 0,
2c3682f0
JF		 162 &ctx->sg_plaintext_num_elem, &ctx->sg_plaintext_size,
163 tls_ctx->pending_open_record_frags);
3c4d7559
DW		 164
165 return rc;
166}
167
168static void free_sg(struct sock *sk, struct scatterlist *sg,
169 int *sg_num_elem, unsigned int *sg_size)
170{
171 int i, n = *sg_num_elem;
172
173 for (i = 0; i < n; ++i) {
174 sk_mem_uncharge(sk, sg[i].length);
175 put_page(sg_page(&sg[i]));
176 }
177 *sg_num_elem = 0;
178 *sg_size = 0;
179}
180
181static void tls_free_both_sg(struct sock *sk)
182{
183 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 184 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 185
186 free_sg(sk, ctx->sg_encrypted_data, &ctx->sg_encrypted_num_elem,
187 &ctx->sg_encrypted_size);
188
189 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
190 &ctx->sg_plaintext_size);
191}
192
193static int tls_do_encryption(struct tls_context *tls_ctx,
a447da7d
DB		 194 struct tls_sw_context_tx *ctx,
195 struct aead_request *aead_req,
196 size_t data_len)
3c4d7559 197{
3c4d7559
DW		 198 int rc;
199
dbe42559
DW		 200 ctx->sg_encrypted_data[0].offset += tls_ctx->tx.prepend_size;
201 ctx->sg_encrypted_data[0].length -= tls_ctx->tx.prepend_size;
3c4d7559
DW		 202
203 aead_request_set_tfm(aead_req, ctx->aead_send);
204 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
205 aead_request_set_crypt(aead_req, ctx->sg_aead_in, ctx->sg_aead_out,
dbe42559 206 data_len, tls_ctx->tx.iv);
a54667f6
VG		 207
208 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
209 crypto_req_done, &ctx->async_wait);
210
211 rc = crypto_wait_req(crypto_aead_encrypt(aead_req), &ctx->async_wait);
3c4d7559 212
dbe42559
DW		 213 ctx->sg_encrypted_data[0].offset -= tls_ctx->tx.prepend_size;
214 ctx->sg_encrypted_data[0].length += tls_ctx->tx.prepend_size;
3c4d7559 215
3c4d7559
DW		 216 return rc;
217}
218
219static int tls_push_record(struct sock *sk, int flags,
220 unsigned char record_type)
221{
222 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 223 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
a447da7d 224 struct aead_request *req;
3c4d7559
DW		 225 int rc;
226
a447da7d
DB		 227 req = kzalloc(sizeof(struct aead_request) +
228 crypto_aead_reqsize(ctx->aead_send), sk->sk_allocation);
229 if (!req)
230 return -ENOMEM;
231
3c4d7559
DW		 232 sg_mark_end(ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem - 1);
233 sg_mark_end(ctx->sg_encrypted_data + ctx->sg_encrypted_num_elem - 1);
234
213ef6e7 235 tls_make_aad(ctx->aad_space, ctx->sg_plaintext_size,
dbe42559 236 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
3c4d7559
DW		 237 record_type);
238
239 tls_fill_prepend(tls_ctx,
240 page_address(sg_page(&ctx->sg_encrypted_data[0])) +
241 ctx->sg_encrypted_data[0].offset,
242 ctx->sg_plaintext_size, record_type);
243
244 tls_ctx->pending_open_record_frags = 0;
245 set_bit(TLS_PENDING_CLOSED_RECORD, &tls_ctx->flags);
246
a447da7d 247 rc = tls_do_encryption(tls_ctx, ctx, req, ctx->sg_plaintext_size);
3c4d7559
DW		 248 if (rc < 0) {
249 /* If we are called from write_space and
250 * we fail, we need to set this SOCK_NOSPACE
251 * to trigger another write_space in the future.
252 */
253 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
a447da7d 254 goto out_req;
3c4d7559
DW		 255 }
256
257 free_sg(sk, ctx->sg_plaintext_data, &ctx->sg_plaintext_num_elem,
258 &ctx->sg_plaintext_size);
259
260 ctx->sg_encrypted_num_elem = 0;
261 ctx->sg_encrypted_size = 0;
262
263 /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
264 rc = tls_push_sg(sk, tls_ctx, ctx->sg_encrypted_data, 0, flags);
265 if (rc < 0 && rc != -EAGAIN)
f4a8e43f 266 tls_err_abort(sk, EBADMSG);
3c4d7559 267
dbe42559 268 tls_advance_record_sn(sk, &tls_ctx->tx);
a447da7d
DB		 269out_req:
270 kfree(req);
3c4d7559
DW		 271 return rc;
272}
273
274static int tls_sw_push_pending_record(struct sock *sk, int flags)
275{
276 return tls_push_record(sk, flags, TLS_RECORD_TYPE_DATA);
277}
278
279static int zerocopy_from_iter(struct sock *sk, struct iov_iter *from,
69ca9293
DW		 280 int length, int *pages_used,
281 unsigned int *size_used,
282 struct scatterlist *to, int to_max_pages,
283 bool charge)
3c4d7559 284{
3c4d7559
DW		 285 struct page *pages[MAX_SKB_FRAGS];
286
287 size_t offset;
288 ssize_t copied, use;
289 int i = 0;
69ca9293
DW		 290 unsigned int size = *size_used;
291 int num_elem = *pages_used;
3c4d7559
DW		 292 int rc = 0;
293 int maxpages;
294
295 while (length > 0) {
296 i = 0;
69ca9293 297 maxpages = to_max_pages - num_elem;
3c4d7559
DW		 298 if (maxpages == 0) {
299 rc = -EFAULT;
300 goto out;
301 }
302 copied = iov_iter_get_pages(from, pages,
303 length,
304 maxpages, &offset);
305 if (copied <= 0) {
306 rc = -EFAULT;
307 goto out;
308 }
309
310 iov_iter_advance(from, copied);
311
312 length -= copied;
313 size += copied;
314 while (copied) {
315 use = min_t(int, copied, PAGE_SIZE - offset);
316
69ca9293 317 sg_set_page(&to[num_elem],
3c4d7559 318 pages[i], use, offset);
69ca9293
DW		 319 sg_unmark_end(&to[num_elem]);
320 if (charge)
321 sk_mem_charge(sk, use);
3c4d7559
DW		 322
323 offset = 0;
324 copied -= use;
325
326 ++i;
327 ++num_elem;
328 }
329 }
330
331out:
69ca9293
DW		 332 *size_used = size;
333 *pages_used = num_elem;
334
3c4d7559
DW		 335 return rc;
336}
337
338static int memcopy_from_iter(struct sock *sk, struct iov_iter *from,
339 int bytes)
340{
341 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 342 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 343 struct scatterlist *sg = ctx->sg_plaintext_data;
344 int copy, i, rc = 0;
345
346 for (i = tls_ctx->pending_open_record_frags;
347 i < ctx->sg_plaintext_num_elem; ++i) {
348 copy = sg[i].length;
349 if (copy_from_iter(
350 page_address(sg_page(&sg[i])) + sg[i].offset,
351 copy, from) != copy) {
352 rc = -EFAULT;
353 goto out;
354 }
355 bytes -= copy;
356
357 ++tls_ctx->pending_open_record_frags;
358
359 if (!bytes)
360 break;
361 }
362
363out:
364 return rc;
365}
366
367int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
368{
369 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 370 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 371 int ret = 0;
372 int required_size;
373 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
374 bool eor = !(msg->msg_flags & MSG_MORE);
375 size_t try_to_copy, copied = 0;
376 unsigned char record_type = TLS_RECORD_TYPE_DATA;
377 int record_room;
378 bool full_record;
379 int orig_size;
380
381 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
382 return -ENOTSUPP;
383
384 lock_sock(sk);
385
386 if (tls_complete_pending_work(sk, tls_ctx, msg->msg_flags, &timeo))
387 goto send_end;
388
389 if (unlikely(msg->msg_controllen)) {
390 ret = tls_proccess_cmsg(sk, msg, &record_type);
391 if (ret)
392 goto send_end;
393 }
394
395 while (msg_data_left(msg)) {
396 if (sk->sk_err) {
30be8f8d 397 ret = -sk->sk_err;
3c4d7559
DW		 398 goto send_end;
399 }
400
401 orig_size = ctx->sg_plaintext_size;
402 full_record = false;
403 try_to_copy = msg_data_left(msg);
404 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
405 if (try_to_copy >= record_room) {
406 try_to_copy = record_room;
407 full_record = true;
408 }
409
410 required_size = ctx->sg_plaintext_size + try_to_copy +
dbe42559 411 tls_ctx->tx.overhead_size;
3c4d7559
DW		 412
413 if (!sk_stream_memory_free(sk))
414 goto wait_for_sndbuf;
415alloc_encrypted:
416 ret = alloc_encrypted_sg(sk, required_size);
417 if (ret) {
418 if (ret != -ENOSPC)
419 goto wait_for_memory;
420
421 /* Adjust try_to_copy according to the amount that was
422 * actually allocated. The difference is due
423 * to max sg elements limit
424 */
425 try_to_copy -= required_size - ctx->sg_encrypted_size;
426 full_record = true;
427 }
428
429 if (full_record || eor) {
430 ret = zerocopy_from_iter(sk, &msg->msg_iter,
69ca9293
DW		 431 try_to_copy, &ctx->sg_plaintext_num_elem,
432 &ctx->sg_plaintext_size,
433 ctx->sg_plaintext_data,
434 ARRAY_SIZE(ctx->sg_plaintext_data),
435 true);
3c4d7559
DW		 436 if (ret)
437 goto fallback_to_reg_send;
438
439 copied += try_to_copy;
440 ret = tls_push_record(sk, msg->msg_flags, record_type);
441 if (!ret)
442 continue;
443 if (ret == -EAGAIN)
444 goto send_end;
445
446 copied -= try_to_copy;
447fallback_to_reg_send:
448 iov_iter_revert(&msg->msg_iter,
449 ctx->sg_plaintext_size - orig_size);
450 trim_sg(sk, ctx->sg_plaintext_data,
451 &ctx->sg_plaintext_num_elem,
452 &ctx->sg_plaintext_size,
453 orig_size);
454 }
455
456 required_size = ctx->sg_plaintext_size + try_to_copy;
457alloc_plaintext:
458 ret = alloc_plaintext_sg(sk, required_size);
459 if (ret) {
460 if (ret != -ENOSPC)
461 goto wait_for_memory;
462
463 /* Adjust try_to_copy according to the amount that was
464 * actually allocated. The difference is due
465 * to max sg elements limit
466 */
467 try_to_copy -= required_size - ctx->sg_plaintext_size;
468 full_record = true;
469
470 trim_sg(sk, ctx->sg_encrypted_data,
471 &ctx->sg_encrypted_num_elem,
472 &ctx->sg_encrypted_size,
473 ctx->sg_plaintext_size +
dbe42559 474 tls_ctx->tx.overhead_size);
3c4d7559
DW		 475 }
476
477 ret = memcopy_from_iter(sk, &msg->msg_iter, try_to_copy);
478 if (ret)
479 goto trim_sgl;
480
481 copied += try_to_copy;
482 if (full_record || eor) {
483push_record:
484 ret = tls_push_record(sk, msg->msg_flags, record_type);
485 if (ret) {
486 if (ret == -ENOMEM)
487 goto wait_for_memory;
488
489 goto send_end;
490 }
491 }
492
493 continue;
494
495wait_for_sndbuf:
496 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
497wait_for_memory:
498 ret = sk_stream_wait_memory(sk, &timeo);
499 if (ret) {
500trim_sgl:
501 trim_both_sgl(sk, orig_size);
502 goto send_end;
503 }
504
505 if (tls_is_pending_closed_record(tls_ctx))
506 goto push_record;
507
508 if (ctx->sg_encrypted_size < required_size)
509 goto alloc_encrypted;
510
511 goto alloc_plaintext;
512 }
513
514send_end:
515 ret = sk_stream_error(sk, msg->msg_flags, ret);
516
517 release_sock(sk);
518 return copied ? copied : ret;
519}
520
521int tls_sw_sendpage(struct sock *sk, struct page *page,
522 int offset, size_t size, int flags)
523{
524 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 525 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 526 int ret = 0;
527 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
528 bool eor;
529 size_t orig_size = size;
530 unsigned char record_type = TLS_RECORD_TYPE_DATA;
531 struct scatterlist *sg;
532 bool full_record;
533 int record_room;
534
535 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
536 MSG_SENDPAGE_NOTLAST))
537 return -ENOTSUPP;
538
539 /* No MSG_EOR from splice, only look at MSG_MORE */
540 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
541
542 lock_sock(sk);
543
544 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
545
546 if (tls_complete_pending_work(sk, tls_ctx, flags, &timeo))
547 goto sendpage_end;
548
549 /* Call the sk_stream functions to manage the sndbuf mem. */
550 while (size > 0) {
551 size_t copy, required_size;
552
553 if (sk->sk_err) {
30be8f8d 554 ret = -sk->sk_err;
3c4d7559
DW		 555 goto sendpage_end;
556 }
557
558 full_record = false;
559 record_room = TLS_MAX_PAYLOAD_SIZE - ctx->sg_plaintext_size;
560 copy = size;
561 if (copy >= record_room) {
562 copy = record_room;
563 full_record = true;
564 }
565 required_size = ctx->sg_plaintext_size + copy +
dbe42559 566 tls_ctx->tx.overhead_size;
3c4d7559
DW		 567
568 if (!sk_stream_memory_free(sk))
569 goto wait_for_sndbuf;
570alloc_payload:
571 ret = alloc_encrypted_sg(sk, required_size);
572 if (ret) {
573 if (ret != -ENOSPC)
574 goto wait_for_memory;
575
576 /* Adjust copy according to the amount that was
577 * actually allocated. The difference is due
578 * to max sg elements limit
579 */
580 copy -= required_size - ctx->sg_plaintext_size;
581 full_record = true;
582 }
583
584 get_page(page);
585 sg = ctx->sg_plaintext_data + ctx->sg_plaintext_num_elem;
586 sg_set_page(sg, page, copy, offset);
7a8c4dd9
DW		 587 sg_unmark_end(sg);
588
3c4d7559
DW		 589 ctx->sg_plaintext_num_elem++;
590
591 sk_mem_charge(sk, copy);
592 offset += copy;
593 size -= copy;
594 ctx->sg_plaintext_size += copy;
595 tls_ctx->pending_open_record_frags = ctx->sg_plaintext_num_elem;
596
597 if (full_record || eor ||
598 ctx->sg_plaintext_num_elem ==
599 ARRAY_SIZE(ctx->sg_plaintext_data)) {
600push_record:
601 ret = tls_push_record(sk, flags, record_type);
602 if (ret) {
603 if (ret == -ENOMEM)
604 goto wait_for_memory;
605
606 goto sendpage_end;
607 }
608 }
609 continue;
610wait_for_sndbuf:
611 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
612wait_for_memory:
613 ret = sk_stream_wait_memory(sk, &timeo);
614 if (ret) {
615 trim_both_sgl(sk, ctx->sg_plaintext_size);
616 goto sendpage_end;
617 }
618
619 if (tls_is_pending_closed_record(tls_ctx))
620 goto push_record;
621
622 goto alloc_payload;
623 }
624
625sendpage_end:
626 if (orig_size > size)
627 ret = orig_size - size;
628 else
629 ret = sk_stream_error(sk, flags, ret);
630
631 release_sock(sk);
632 return ret;
633}
634
c46234eb
DW		 635static struct sk_buff *tls_wait_data(struct sock *sk, int flags,
636 long timeo, int *err)
637{
638 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 639 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 640 struct sk_buff *skb;
641 DEFINE_WAIT_FUNC(wait, woken_wake_function);
642
643 while (!(skb = ctx->recv_pkt)) {
644 if (sk->sk_err) {
645 *err = sock_error(sk);
646 return NULL;
647 }
648
649 if (sock_flag(sk, SOCK_DONE))
650 return NULL;
651
652 if ((flags & MSG_DONTWAIT) || !timeo) {
653 *err = -EAGAIN;
654 return NULL;
655 }
656
657 add_wait_queue(sk_sleep(sk), &wait);
658 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
659 sk_wait_event(sk, &timeo, ctx->recv_pkt != skb, &wait);
660 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
661 remove_wait_queue(sk_sleep(sk), &wait);
662
663 /* Handle signals */
664 if (signal_pending(current)) {
665 *err = sock_intr_errno(timeo);
666 return NULL;
667 }
668 }
669
670 return skb;
671}
672
673static int decrypt_skb(struct sock *sk, struct sk_buff *skb,
674 struct scatterlist *sgout)
675{
676 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 677 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
b16520f7 678 char iv[TLS_CIPHER_AES_GCM_128_SALT_SIZE + MAX_IV_SIZE];
c46234eb
DW		 679 struct scatterlist sgin_arr[MAX_SKB_FRAGS + 2];
680 struct scatterlist *sgin = &sgin_arr[0];
681 struct strp_msg *rxm = strp_msg(skb);
682 int ret, nsg = ARRAY_SIZE(sgin_arr);
c46234eb
DW		 683 struct sk_buff *unused;
684
685 ret = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
686 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
687 tls_ctx->rx.iv_size);
688 if (ret < 0)
689 return ret;
690
691 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
692 if (!sgout) {
693 nsg = skb_cow_data(skb, 0, &unused) + 1;
694 sgin = kmalloc_array(nsg, sizeof(*sgin), sk->sk_allocation);
95ad7544 695 sgout = sgin;
c46234eb
DW		 696 }
697
698 sg_init_table(sgin, nsg);
8ab6ffba 699 sg_set_buf(&sgin[0], ctx->rx_aad_ciphertext, TLS_AAD_SPACE_SIZE);
c46234eb
DW		 700
701 nsg = skb_to_sgvec(skb, &sgin[1],
702 rxm->offset + tls_ctx->rx.prepend_size,
703 rxm->full_len - tls_ctx->rx.prepend_size);
704
8ab6ffba 705 tls_make_aad(ctx->rx_aad_ciphertext,
c46234eb
DW		 706 rxm->full_len - tls_ctx->rx.overhead_size,
707 tls_ctx->rx.rec_seq,
708 tls_ctx->rx.rec_seq_size,
709 ctx->control);
710
711 ret = tls_do_decryption(sk, sgin, sgout, iv,
712 rxm->full_len - tls_ctx->rx.overhead_size,
713 skb, sk->sk_allocation);
714
715 if (sgin != &sgin_arr[0])
716 kfree(sgin);
717
718 return ret;
719}
720
721static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
722 unsigned int len)
723{
724 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 725 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 726 struct strp_msg *rxm = strp_msg(skb);
727
728 if (len < rxm->full_len) {
729 rxm->offset += len;
730 rxm->full_len -= len;
731
732 return false;
733 }
734
735 /* Finished with message */
736 ctx->recv_pkt = NULL;
737 kfree_skb(skb);
7170e604 738 __strp_unpause(&ctx->strp);
c46234eb
DW		 739
740 return true;
741}
742
743int tls_sw_recvmsg(struct sock *sk,
744 struct msghdr *msg,
745 size_t len,
746 int nonblock,
747 int flags,
748 int *addr_len)
749{
750 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 751 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 752 unsigned char control;
753 struct strp_msg *rxm;
754 struct sk_buff *skb;
755 ssize_t copied = 0;
756 bool cmsg = false;
06030dba 757 int target, err = 0;
c46234eb
DW		 758 long timeo;
759
760 flags |= nonblock;
761
762 if (unlikely(flags & MSG_ERRQUEUE))
763 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
764
765 lock_sock(sk);
766
06030dba 767 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
c46234eb
DW		 768 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
769 do {
770 bool zc = false;
771 int chunk = 0;
772
773 skb = tls_wait_data(sk, flags, timeo, &err);
774 if (!skb)
775 goto recv_end;
776
777 rxm = strp_msg(skb);
778 if (!cmsg) {
779 int cerr;
780
781 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
782 sizeof(ctx->control), &ctx->control);
783 cmsg = true;
784 control = ctx->control;
785 if (ctx->control != TLS_RECORD_TYPE_DATA) {
786 if (cerr || msg->msg_flags & MSG_CTRUNC) {
787 err = -EIO;
788 goto recv_end;
789 }
790 }
791 } else if (control != ctx->control) {
792 goto recv_end;
793 }
794
795 if (!ctx->decrypted) {
796 int page_count;
797 int to_copy;
798
799 page_count = iov_iter_npages(&msg->msg_iter,
800 MAX_SKB_FRAGS);
801 to_copy = rxm->full_len - tls_ctx->rx.overhead_size;
802 if (to_copy <= len && page_count < MAX_SKB_FRAGS &&
803 likely(!(flags & MSG_PEEK))) {
804 struct scatterlist sgin[MAX_SKB_FRAGS + 1];
c46234eb
DW		 805 int pages = 0;
806
807 zc = true;
808 sg_init_table(sgin, MAX_SKB_FRAGS + 1);
8ab6ffba
MM		 809 sg_set_buf(&sgin[0], ctx->rx_aad_plaintext,
810 TLS_AAD_SPACE_SIZE);
c46234eb
DW		 811
812 err = zerocopy_from_iter(sk, &msg->msg_iter,
813 to_copy, &pages,
814 &chunk, &sgin[1],
815 MAX_SKB_FRAGS, false);
816 if (err < 0)
817 goto fallback_to_reg_recv;
818
819 err = decrypt_skb(sk, skb, sgin);
820 for (; pages > 0; pages--)
821 put_page(sg_page(&sgin[pages]));
822 if (err < 0) {
823 tls_err_abort(sk, EBADMSG);
824 goto recv_end;
825 }
826 } else {
827fallback_to_reg_recv:
828 err = decrypt_skb(sk, skb, NULL);
829 if (err < 0) {
830 tls_err_abort(sk, EBADMSG);
831 goto recv_end;
832 }
833 }
834 ctx->decrypted = true;
835 }
836
837 if (!zc) {
838 chunk = min_t(unsigned int, rxm->full_len, len);
839 err = skb_copy_datagram_msg(skb, rxm->offset, msg,
840 chunk);
841 if (err < 0)
842 goto recv_end;
843 }
844
845 copied += chunk;
846 len -= chunk;
847 if (likely(!(flags & MSG_PEEK))) {
848 u8 control = ctx->control;
849
850 if (tls_sw_advance_skb(sk, skb, chunk)) {
851 /* Return full control message to
852 * userspace before trying to parse
853 * another message type
854 */
855 msg->msg_flags |= MSG_EOR;
856 if (control != TLS_RECORD_TYPE_DATA)
857 goto recv_end;
858 }
859 }
06030dba
DB		 860 /* If we have a new message from strparser, continue now. */
861 if (copied >= target && !ctx->recv_pkt)
862 break;
c46234eb
DW		 863 } while (len);
864
865recv_end:
866 release_sock(sk);
867 return copied ? : err;
868}
869
870ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
871 struct pipe_inode_info *pipe,
872 size_t len, unsigned int flags)
873{
874 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
f66de3ee 875 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 876 struct strp_msg *rxm = NULL;
877 struct sock *sk = sock->sk;
878 struct sk_buff *skb;
879 ssize_t copied = 0;
880 int err = 0;
881 long timeo;
882 int chunk;
883
884 lock_sock(sk);
885
886 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
887
888 skb = tls_wait_data(sk, flags, timeo, &err);
889 if (!skb)
890 goto splice_read_end;
891
892 /* splice does not support reading control messages */
893 if (ctx->control != TLS_RECORD_TYPE_DATA) {
894 err = -ENOTSUPP;
895 goto splice_read_end;
896 }
897
898 if (!ctx->decrypted) {
899 err = decrypt_skb(sk, skb, NULL);
900
901 if (err < 0) {
902 tls_err_abort(sk, EBADMSG);
903 goto splice_read_end;
904 }
905 ctx->decrypted = true;
906 }
907 rxm = strp_msg(skb);
908
909 chunk = min_t(unsigned int, rxm->full_len, len);
910 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
911 if (copied < 0)
912 goto splice_read_end;
913
914 if (likely(!(flags & MSG_PEEK)))
915 tls_sw_advance_skb(sk, skb, copied);
916
917splice_read_end:
918 release_sock(sk);
919 return copied ? : err;
920}
921
a11e1d43
LT		 922unsigned int tls_sw_poll(struct file *file, struct socket *sock,
923 struct poll_table_struct *wait)
c46234eb 924{
a11e1d43 925 unsigned int ret;
c46234eb
DW		 926 struct sock *sk = sock->sk;
927 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 928 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb 929
a11e1d43
LT		 930 /* Grab POLLOUT and POLLHUP from the underlying socket */
931 ret = ctx->sk_poll(file, sock, wait);
c46234eb 932
a11e1d43
LT		 933 /* Clear POLLIN bits, and set based on recv_pkt */
934 ret &= ~(POLLIN | POLLRDNORM);
c46234eb 935 if (ctx->recv_pkt)
a11e1d43 936 ret |= POLLIN | POLLRDNORM;
c46234eb 937
a11e1d43 938 return ret;
c46234eb
DW		 939}
940
941static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
942{
943 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 944 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 945 char header[tls_ctx->rx.prepend_size];
946 struct strp_msg *rxm = strp_msg(skb);
947 size_t cipher_overhead;
948 size_t data_len = 0;
949 int ret;
950
951 /* Verify that we have a full TLS header, or wait for more data */
952 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
953 return 0;
954
955 /* Linearize header to local buffer */
956 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
957
958 if (ret < 0)
959 goto read_failure;
960
961 ctx->control = header[0];
962
963 data_len = ((header[4] & 0xFF) | (header[3] << 8));
964
965 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
966
967 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
968 ret = -EMSGSIZE;
969 goto read_failure;
970 }
971 if (data_len < cipher_overhead) {
972 ret = -EBADMSG;
973 goto read_failure;
974 }
975
976 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.version) ||
977 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.version)) {
978 ret = -EINVAL;
979 goto read_failure;
980 }
981
982 return data_len + TLS_HEADER_SIZE;
983
984read_failure:
985 tls_err_abort(strp->sk, ret);
986
987 return ret;
988}
989
990static void tls_queue(struct strparser *strp, struct sk_buff *skb)
991{
992 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
f66de3ee 993 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 994 struct strp_msg *rxm;
995
996 rxm = strp_msg(skb);
997
998 ctx->decrypted = false;
999
1000 ctx->recv_pkt = skb;
1001 strp_pause(strp);
1002
1003 strp->sk->sk_state_change(strp->sk);
1004}
1005
1006static void tls_data_ready(struct sock *sk)
1007{
1008 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1009 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
c46234eb
DW		 1010
1011 strp_data_ready(&ctx->strp);
1012}
1013
f66de3ee 1014void tls_sw_free_resources_tx(struct sock *sk)
3c4d7559
DW		 1015{
1016 struct tls_context *tls_ctx = tls_get_ctx(sk);
f66de3ee 1017 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
3c4d7559
DW		 1018
1019 if (ctx->aead_send)
1020 crypto_free_aead(ctx->aead_send);
f66de3ee
BP		 1021 tls_free_both_sg(sk);
1022
1023 kfree(ctx);
1024}
1025
1026void tls_sw_free_resources_rx(struct sock *sk)
1027{
1028 struct tls_context *tls_ctx = tls_get_ctx(sk);
1029 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1030
c46234eb
DW		 1031 if (ctx->aead_recv) {
1032 if (ctx->recv_pkt) {
1033 kfree_skb(ctx->recv_pkt);
1034 ctx->recv_pkt = NULL;
1035 }
1036 crypto_free_aead(ctx->aead_recv);
1037 strp_stop(&ctx->strp);
1038 write_lock_bh(&sk->sk_callback_lock);
1039 sk->sk_data_ready = ctx->saved_data_ready;
1040 write_unlock_bh(&sk->sk_callback_lock);
1041 release_sock(sk);
1042 strp_done(&ctx->strp);
1043 lock_sock(sk);
1044 }
3c4d7559 1045
3c4d7559
DW		 1046 kfree(ctx);
1047}
1048
c46234eb 1049int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
3c4d7559
DW		 1050{
1051 char keyval[TLS_CIPHER_AES_GCM_128_KEY_SIZE];
1052 struct tls_crypto_info *crypto_info;
1053 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
f66de3ee
BP		 1054 struct tls_sw_context_tx *sw_ctx_tx = NULL;
1055 struct tls_sw_context_rx *sw_ctx_rx = NULL;
c46234eb
DW		 1056 struct cipher_context *cctx;
1057 struct crypto_aead **aead;
1058 struct strp_callbacks cb;
3c4d7559
DW		 1059 u16 nonce_size, tag_size, iv_size, rec_seq_size;
1060 char *iv, *rec_seq;
1061 int rc = 0;
1062
1063 if (!ctx) {
1064 rc = -EINVAL;
1065 goto out;
1066 }
1067
f66de3ee
BP		 1068 if (tx) {
1069 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
1070 if (!sw_ctx_tx) {
c46234eb
DW		 1071 rc = -ENOMEM;
1072 goto out;
1073 }
f66de3ee
BP		 1074 crypto_init_wait(&sw_ctx_tx->async_wait);
1075 ctx->priv_ctx_tx = sw_ctx_tx;
c46234eb 1076 } else {
f66de3ee
BP		 1077 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
1078 if (!sw_ctx_rx) {
1079 rc = -ENOMEM;
1080 goto out;
1081 }
1082 crypto_init_wait(&sw_ctx_rx->async_wait);
1083 ctx->priv_ctx_rx = sw_ctx_rx;
3c4d7559
DW		 1084 }
1085
c46234eb
DW		 1086 if (tx) {
1087 crypto_info = &ctx->crypto_send;
1088 cctx = &ctx->tx;
f66de3ee 1089 aead = &sw_ctx_tx->aead_send;
c46234eb
DW		 1090 } else {
1091 crypto_info = &ctx->crypto_recv;
1092 cctx = &ctx->rx;
f66de3ee 1093 aead = &sw_ctx_rx->aead_recv;
c46234eb
DW		 1094 }
1095
3c4d7559
DW		 1096 switch (crypto_info->cipher_type) {
1097 case TLS_CIPHER_AES_GCM_128: {
1098 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1099 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1100 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
1101 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
1102 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
1103 rec_seq =
1104 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
1105 gcm_128_info =
1106 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
1107 break;
1108 }
1109 default:
1110 rc = -EINVAL;
cf6d43ef 1111 goto free_priv;
3c4d7559
DW		 1112 }
1113
b16520f7
KC		 1114 /* Sanity-check the IV size for stack allocations. */
1115 if (iv_size > MAX_IV_SIZE) {
1116 rc = -EINVAL;
1117 goto free_priv;
1118 }
1119
c46234eb
DW		 1120 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
1121 cctx->tag_size = tag_size;
1122 cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
1123 cctx->iv_size = iv_size;
1124 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1125 GFP_KERNEL);
1126 if (!cctx->iv) {
3c4d7559 1127 rc = -ENOMEM;
cf6d43ef 1128 goto free_priv;
3c4d7559 1129 }
c46234eb
DW		 1130 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1131 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
1132 cctx->rec_seq_size = rec_seq_size;
1133 cctx->rec_seq = kmalloc(rec_seq_size, GFP_KERNEL);
1134 if (!cctx->rec_seq) {
3c4d7559
DW		 1135 rc = -ENOMEM;
1136 goto free_iv;
1137 }
c46234eb
DW		 1138 memcpy(cctx->rec_seq, rec_seq, rec_seq_size);
1139
f66de3ee
BP		 1140 if (sw_ctx_tx) {
1141 sg_init_table(sw_ctx_tx->sg_encrypted_data,
1142 ARRAY_SIZE(sw_ctx_tx->sg_encrypted_data));
1143 sg_init_table(sw_ctx_tx->sg_plaintext_data,
1144 ARRAY_SIZE(sw_ctx_tx->sg_plaintext_data));
1145
1146 sg_init_table(sw_ctx_tx->sg_aead_in, 2);
1147 sg_set_buf(&sw_ctx_tx->sg_aead_in[0], sw_ctx_tx->aad_space,
1148 sizeof(sw_ctx_tx->aad_space));
1149 sg_unmark_end(&sw_ctx_tx->sg_aead_in[1]);
1150 sg_chain(sw_ctx_tx->sg_aead_in, 2,
1151 sw_ctx_tx->sg_plaintext_data);
1152 sg_init_table(sw_ctx_tx->sg_aead_out, 2);
1153 sg_set_buf(&sw_ctx_tx->sg_aead_out[0], sw_ctx_tx->aad_space,
1154 sizeof(sw_ctx_tx->aad_space));
1155 sg_unmark_end(&sw_ctx_tx->sg_aead_out[1]);
1156 sg_chain(sw_ctx_tx->sg_aead_out, 2,
1157 sw_ctx_tx->sg_encrypted_data);
c46234eb
DW		 1158 }
1159
1160 if (!*aead) {
1161 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
1162 if (IS_ERR(*aead)) {
1163 rc = PTR_ERR(*aead);
1164 *aead = NULL;
3c4d7559
DW		 1165 goto free_rec_seq;
1166 }
1167 }
1168
1169 ctx->push_pending_record = tls_sw_push_pending_record;
1170
1171 memcpy(keyval, gcm_128_info->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1172
c46234eb 1173 rc = crypto_aead_setkey(*aead, keyval,
3c4d7559
DW		 1174 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
1175 if (rc)
1176 goto free_aead;
1177
c46234eb
DW		 1178 rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
1179 if (rc)
1180 goto free_aead;
1181
f66de3ee 1182 if (sw_ctx_rx) {
c46234eb
DW		 1183 /* Set up strparser */
1184 memset(&cb, 0, sizeof(cb));
1185 cb.rcv_msg = tls_queue;
1186 cb.parse_msg = tls_read_size;
1187
f66de3ee 1188 strp_init(&sw_ctx_rx->strp, sk, &cb);
c46234eb
DW		 1189
1190 write_lock_bh(&sk->sk_callback_lock);
f66de3ee 1191 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
c46234eb
DW		 1192 sk->sk_data_ready = tls_data_ready;
1193 write_unlock_bh(&sk->sk_callback_lock);
1194
a11e1d43 1195 sw_ctx_rx->sk_poll = sk->sk_socket->ops->poll;
c46234eb 1196
f66de3ee 1197 strp_check_rcv(&sw_ctx_rx->strp);
c46234eb
DW		 1198 }
1199
1200 goto out;
3c4d7559
DW		 1201
1202free_aead:
c46234eb
DW		 1203 crypto_free_aead(*aead);
1204 *aead = NULL;
3c4d7559 1205free_rec_seq:
c46234eb
DW		 1206 kfree(cctx->rec_seq);
1207 cctx->rec_seq = NULL;
3c4d7559 1208free_iv:
f66de3ee
BP		 1209 kfree(cctx->iv);
1210 cctx->iv = NULL;
cf6d43ef 1211free_priv:
f66de3ee
BP		 1212 if (tx) {
1213 kfree(ctx->priv_ctx_tx);
1214 ctx->priv_ctx_tx = NULL;
1215 } else {
1216 kfree(ctx->priv_ctx_rx);
1217 ctx->priv_ctx_rx = NULL;
1218 }
3c4d7559
DW		 1219out:
1220 return rc;
1221}
Linux 6.x block layer and io_uring tree(s)