projects / linux-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'mm-hotfixes-stable-2025-07-11-16-16' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / net / mptcp / subflow.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0
2/* Multipath TCP
3 *
4 * Copyright (c) 2017 - 2019, Intel Corporation.
5 */
6
7#define pr_fmt(fmt) "MPTCP: " fmt
8
9#include <linux/kernel.h>
10#include <linux/module.h>
11#include <linux/netdevice.h>
12#include <crypto/sha2.h>
13#include <crypto/utils.h>
14#include <net/sock.h>
15#include <net/inet_common.h>
16#include <net/inet_hashtables.h>
17#include <net/protocol.h>
18#if IS_ENABLED(CONFIG_MPTCP_IPV6)
19#include <net/ip6_route.h>
20#include <net/transp_v6.h>
21#endif
22#include <net/mptcp.h>
23
24#include "protocol.h"
25#include "mib.h"
26
27#include <trace/events/mptcp.h>
28#include <trace/events/sock.h>
29
30static void mptcp_subflow_ops_undo_override(struct sock *ssk);
31
32static void SUBFLOW_REQ_INC_STATS(struct request_sock *req,
33 enum linux_mptcp_mib_field field)
34{
35 MPTCP_INC_STATS(sock_net(req_to_sk(req)), field);
36}
37
38static void subflow_req_destructor(struct request_sock *req)
39{
40 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
41
42 pr_debug("subflow_req=%p\n", subflow_req);
43
44 if (subflow_req->msk)
45 sock_put((struct sock *)subflow_req->msk);
46
47 mptcp_token_destroy_request(req);
48}
49
50static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2,
51 void *hmac)
52{
53 u8 msg[8];
54
55 put_unaligned_be32(nonce1, &msg[0]);
56 put_unaligned_be32(nonce2, &msg[4]);
57
58 mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac);
59}
60
61static bool mptcp_can_accept_new_subflow(const struct mptcp_sock *msk)
62{
63 return mptcp_is_fully_established((void *)msk) &&
64 ((mptcp_pm_is_userspace(msk) &&
65 mptcp_userspace_pm_active(msk)) ||
66 READ_ONCE(msk->pm.accept_subflow));
67}
68
69/* validate received token and create truncated hmac and nonce for SYN-ACK */
70static void subflow_req_create_thmac(struct mptcp_subflow_request_sock *subflow_req)
71{
72 struct mptcp_sock *msk = subflow_req->msk;
73 u8 hmac[SHA256_DIGEST_SIZE];
74
75 get_random_bytes(&subflow_req->local_nonce, sizeof(u32));
76
77 subflow_generate_hmac(READ_ONCE(msk->local_key),
78 READ_ONCE(msk->remote_key),
79 subflow_req->local_nonce,
80 subflow_req->remote_nonce, hmac);
81
82 subflow_req->thmac = get_unaligned_be64(hmac);
83}
84
85static struct mptcp_sock *subflow_token_join_request(struct request_sock *req)
86{
87 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
88 struct mptcp_sock *msk;
89 int local_id;
90
91 msk = mptcp_token_get_sock(sock_net(req_to_sk(req)), subflow_req->token);
92 if (!msk) {
93 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN);
94 return NULL;
95 }
96
97 local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req);
98 if (local_id < 0) {
99 sock_put((struct sock *)msk);
100 return NULL;
101 }
102 subflow_req->local_id = local_id;
103 subflow_req->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)req);
104
105 return msk;
106}
107
108static void subflow_init_req(struct request_sock *req, const struct sock *sk_listener)
109{
110 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
111
112 subflow_req->mp_capable = 0;
113 subflow_req->mp_join = 0;
114 subflow_req->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk_listener));
115 subflow_req->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk_listener));
116 subflow_req->msk = NULL;
117 mptcp_token_init_request(req);
118}
119
120static bool subflow_use_different_sport(struct mptcp_sock *msk, const struct sock *sk)
121{
122 return inet_sk(sk)->inet_sport != inet_sk((struct sock *)msk)->inet_sport;
123}
124
125static void subflow_add_reset_reason(struct sk_buff *skb, u8 reason)
126{
127 struct mptcp_ext *mpext = skb_ext_add(skb, SKB_EXT_MPTCP);
128
129 if (mpext) {
130 memset(mpext, 0, sizeof(*mpext));
131 mpext->reset_reason = reason;
132 }
133}
134
135static int subflow_reset_req_endp(struct request_sock *req, struct sk_buff *skb)
136{
137 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEENDPATTEMPT);
138 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
139 return -EPERM;
140}
141
142/* Init mptcp request socket.
143 *
144 * Returns an error code if a JOIN has failed and a TCP reset
145 * should be sent.
146 */
147static int subflow_check_req(struct request_sock *req,
148 const struct sock *sk_listener,
149 struct sk_buff *skb)
150{
151 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
152 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
153 struct mptcp_options_received mp_opt;
154 bool opt_mp_capable, opt_mp_join;
155
156 pr_debug("subflow_req=%p, listener=%p\n", subflow_req, listener);
157
158#ifdef CONFIG_TCP_MD5SIG
159 /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
160 * TCP option space.
161 */
162 if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) {
163 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
164 return -EINVAL;
165 }
166#endif
167
168 mptcp_get_options(skb, &mp_opt);
169
170 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYN);
171 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYN);
172 if (opt_mp_capable) {
173 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE);
174
175 if (unlikely(listener->pm_listener))
176 return subflow_reset_req_endp(req, skb);
177 if (opt_mp_join)
178 return 0;
179 } else if (opt_mp_join) {
180 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX);
181
182 if (mp_opt.backup)
183 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNBACKUPRX);
184 } else if (unlikely(listener->pm_listener)) {
185 return subflow_reset_req_endp(req, skb);
186 }
187
188 if (opt_mp_capable && listener->request_mptcp) {
189 int err, retries = MPTCP_TOKEN_MAX_RETRIES;
190
191 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
192again:
193 do {
194 get_random_bytes(&subflow_req->local_key, sizeof(subflow_req->local_key));
195 } while (subflow_req->local_key == 0);
196
197 if (unlikely(req->syncookie)) {
198 mptcp_crypto_key_sha(subflow_req->local_key,
199 &subflow_req->token,
200 &subflow_req->idsn);
201 if (mptcp_token_exists(subflow_req->token)) {
202 if (retries-- > 0)
203 goto again;
204 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT);
205 } else {
206 subflow_req->mp_capable = 1;
207 }
208 return 0;
209 }
210
211 err = mptcp_token_new_request(req);
212 if (err == 0)
213 subflow_req->mp_capable = 1;
214 else if (retries-- > 0)
215 goto again;
216 else
217 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_TOKENFALLBACKINIT);
218
219 } else if (opt_mp_join && listener->request_mptcp) {
220 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq;
221 subflow_req->mp_join = 1;
222 subflow_req->backup = mp_opt.backup;
223 subflow_req->remote_id = mp_opt.join_id;
224 subflow_req->token = mp_opt.token;
225 subflow_req->remote_nonce = mp_opt.nonce;
226 subflow_req->msk = subflow_token_join_request(req);
227
228 /* Can't fall back to TCP in this case. */
229 if (!subflow_req->msk) {
230 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
231 return -EPERM;
232 }
233
234 if (subflow_use_different_sport(subflow_req->msk, sk_listener)) {
235 pr_debug("syn inet_sport=%d %d\n",
236 ntohs(inet_sk(sk_listener)->inet_sport),
237 ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport));
238 if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) {
239 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX);
240 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
241 return -EPERM;
242 }
243 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTSYNRX);
244 }
245
246 subflow_req_create_thmac(subflow_req);
247
248 if (unlikely(req->syncookie)) {
249 if (!mptcp_can_accept_new_subflow(subflow_req->msk)) {
250 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
251 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
252 return -EPERM;
253 }
254
255 subflow_init_req_cookie_join_save(subflow_req, skb);
256 }
257
258 pr_debug("token=%u, remote_nonce=%u msk=%p\n", subflow_req->token,
259 subflow_req->remote_nonce, subflow_req->msk);
260 }
261
262 return 0;
263}
264
265int mptcp_subflow_init_cookie_req(struct request_sock *req,
266 const struct sock *sk_listener,
267 struct sk_buff *skb)
268{
269 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener);
270 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
271 struct mptcp_options_received mp_opt;
272 bool opt_mp_capable, opt_mp_join;
273 int err;
274
275 subflow_init_req(req, sk_listener);
276 mptcp_get_options(skb, &mp_opt);
277
278 opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK);
279 opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK);
280 if (opt_mp_capable && opt_mp_join)
281 return -EINVAL;
282
283 if (opt_mp_capable && listener->request_mptcp) {
284 if (mp_opt.sndr_key == 0)
285 return -EINVAL;
286
287 subflow_req->local_key = mp_opt.rcvr_key;
288 err = mptcp_token_new_request(req);
289 if (err)
290 return err;
291
292 subflow_req->mp_capable = 1;
293 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
294 } else if (opt_mp_join && listener->request_mptcp) {
295 if (!mptcp_token_join_cookie_init_state(subflow_req, skb))
296 return -EINVAL;
297
298 subflow_req->mp_join = 1;
299 subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq - 1;
300 }
301
302 return 0;
303}
304EXPORT_SYMBOL_GPL(mptcp_subflow_init_cookie_req);
305
306static enum sk_rst_reason mptcp_get_rst_reason(const struct sk_buff *skb)
307{
308 const struct mptcp_ext *mpext = mptcp_get_ext(skb);
309
310 if (!mpext)
311 return SK_RST_REASON_NOT_SPECIFIED;
312
313 return sk_rst_convert_mptcp_reason(mpext->reset_reason);
314}
315
316static struct dst_entry *subflow_v4_route_req(const struct sock *sk,
317 struct sk_buff *skb,
318 struct flowi *fl,
319 struct request_sock *req,
320 u32 tw_isn)
321{
322 struct dst_entry *dst;
323 int err;
324
325 tcp_rsk(req)->is_mptcp = 1;
326 subflow_init_req(req, sk);
327
328 dst = tcp_request_sock_ipv4_ops.route_req(sk, skb, fl, req, tw_isn);
329 if (!dst)
330 return NULL;
331
332 err = subflow_check_req(req, sk, skb);
333 if (err == 0)
334 return dst;
335
336 dst_release(dst);
337 if (!req->syncookie)
338 tcp_request_sock_ops.send_reset(sk, skb,
339 mptcp_get_rst_reason(skb));
340 return NULL;
341}
342
343static void subflow_prep_synack(const struct sock *sk, struct request_sock *req,
344 struct tcp_fastopen_cookie *foc,
345 enum tcp_synack_type synack_type)
346{
347 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
348 struct inet_request_sock *ireq = inet_rsk(req);
349
350 /* clear tstamp_ok, as needed depending on cookie */
351 if (foc && foc->len > -1)
352 ireq->tstamp_ok = 0;
353
354 if (synack_type == TCP_SYNACK_FASTOPEN)
355 mptcp_fastopen_subflow_synack_set_params(subflow, req);
356}
357
358static int subflow_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
359 struct flowi *fl,
360 struct request_sock *req,
361 struct tcp_fastopen_cookie *foc,
362 enum tcp_synack_type synack_type,
363 struct sk_buff *syn_skb)
364{
365 subflow_prep_synack(sk, req, foc, synack_type);
366
367 return tcp_request_sock_ipv4_ops.send_synack(sk, dst, fl, req, foc,
368 synack_type, syn_skb);
369}
370
371#if IS_ENABLED(CONFIG_MPTCP_IPV6)
372static int subflow_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
373 struct flowi *fl,
374 struct request_sock *req,
375 struct tcp_fastopen_cookie *foc,
376 enum tcp_synack_type synack_type,
377 struct sk_buff *syn_skb)
378{
379 subflow_prep_synack(sk, req, foc, synack_type);
380
381 return tcp_request_sock_ipv6_ops.send_synack(sk, dst, fl, req, foc,
382 synack_type, syn_skb);
383}
384
385static struct dst_entry *subflow_v6_route_req(const struct sock *sk,
386 struct sk_buff *skb,
387 struct flowi *fl,
388 struct request_sock *req,
389 u32 tw_isn)
390{
391 struct dst_entry *dst;
392 int err;
393
394 tcp_rsk(req)->is_mptcp = 1;
395 subflow_init_req(req, sk);
396
397 dst = tcp_request_sock_ipv6_ops.route_req(sk, skb, fl, req, tw_isn);
398 if (!dst)
399 return NULL;
400
401 err = subflow_check_req(req, sk, skb);
402 if (err == 0)
403 return dst;
404
405 dst_release(dst);
406 if (!req->syncookie)
407 tcp6_request_sock_ops.send_reset(sk, skb,
408 mptcp_get_rst_reason(skb));
409 return NULL;
410}
411#endif
412
413/* validate received truncated hmac and create hmac for third ACK */
414static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow)
415{
416 u8 hmac[SHA256_DIGEST_SIZE];
417 u64 thmac;
418
419 subflow_generate_hmac(subflow->remote_key, subflow->local_key,
420 subflow->remote_nonce, subflow->local_nonce,
421 hmac);
422
423 thmac = get_unaligned_be64(hmac);
424 pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n",
425 subflow, subflow->token, thmac, subflow->thmac);
426
427 return thmac == subflow->thmac;
428}
429
430void mptcp_subflow_reset(struct sock *ssk)
431{
432 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
433 struct sock *sk = subflow->conn;
434
435 /* mptcp_mp_fail_no_response() can reach here on an already closed
436 * socket
437 */
438 if (ssk->sk_state == TCP_CLOSE)
439 return;
440
441 /* must hold: tcp_done() could drop last reference on parent */
442 sock_hold(sk);
443
444 mptcp_send_active_reset_reason(ssk);
445 tcp_done(ssk);
446 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags))
447 mptcp_schedule_work(sk);
448
449 sock_put(sk);
450}
451
452static bool subflow_use_different_dport(struct mptcp_sock *msk, const struct sock *sk)
453{
454 return inet_sk(sk)->inet_dport != inet_sk((struct sock *)msk)->inet_dport;
455}
456
457void __mptcp_sync_state(struct sock *sk, int state)
458{
459 struct mptcp_subflow_context *subflow;
460 struct mptcp_sock *msk = mptcp_sk(sk);
461 struct sock *ssk = msk->first;
462
463 subflow = mptcp_subflow_ctx(ssk);
464 __mptcp_propagate_sndbuf(sk, ssk);
465 if (!msk->rcvspace_init)
466 mptcp_rcv_space_init(msk, ssk);
467
468 if (sk->sk_state == TCP_SYN_SENT) {
469 /* subflow->idsn is always available is TCP_SYN_SENT state,
470 * even for the FASTOPEN scenarios
471 */
472 WRITE_ONCE(msk->write_seq, subflow->idsn + 1);
473 WRITE_ONCE(msk->snd_nxt, msk->write_seq);
474 mptcp_set_state(sk, state);
475 sk->sk_state_change(sk);
476 }
477}
478
479static void subflow_set_remote_key(struct mptcp_sock *msk,
480 struct mptcp_subflow_context *subflow,
481 const struct mptcp_options_received *mp_opt)
482{
483 /* active MPC subflow will reach here multiple times:
484 * at subflow_finish_connect() time and at 4th ack time
485 */
486 if (subflow->remote_key_valid)
487 return;
488
489 subflow->remote_key_valid = 1;
490 subflow->remote_key = mp_opt->sndr_key;
491 mptcp_crypto_key_sha(subflow->remote_key, NULL, &subflow->iasn);
492 subflow->iasn++;
493
494 WRITE_ONCE(msk->remote_key, subflow->remote_key);
495 WRITE_ONCE(msk->ack_seq, subflow->iasn);
496 WRITE_ONCE(msk->can_ack, true);
497 atomic64_set(&msk->rcv_wnd_sent, subflow->iasn);
498}
499
500static void mptcp_propagate_state(struct sock *sk, struct sock *ssk,
501 struct mptcp_subflow_context *subflow,
502 const struct mptcp_options_received *mp_opt)
503{
504 struct mptcp_sock *msk = mptcp_sk(sk);
505
506 mptcp_data_lock(sk);
507 if (mp_opt) {
508 /* Options are available only in the non fallback cases
509 * avoid updating rx path fields otherwise
510 */
511 WRITE_ONCE(msk->snd_una, subflow->idsn + 1);
512 WRITE_ONCE(msk->wnd_end, subflow->idsn + 1 + tcp_sk(ssk)->snd_wnd);
513 subflow_set_remote_key(msk, subflow, mp_opt);
514 }
515
516 if (!sock_owned_by_user(sk)) {
517 __mptcp_sync_state(sk, ssk->sk_state);
518 } else {
519 msk->pending_state = ssk->sk_state;
520 __set_bit(MPTCP_SYNC_STATE, &msk->cb_flags);
521 }
522 mptcp_data_unlock(sk);
523}
524
525static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb)
526{
527 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
528 struct mptcp_options_received mp_opt;
529 struct sock *parent = subflow->conn;
530 struct mptcp_sock *msk;
531
532 subflow->icsk_af_ops->sk_rx_dst_set(sk, skb);
533
534 /* be sure no special action on any packet other than syn-ack */
535 if (subflow->conn_finished)
536 return;
537
538 msk = mptcp_sk(parent);
539 subflow->rel_write_seq = 1;
540 subflow->conn_finished = 1;
541 subflow->ssn_offset = TCP_SKB_CB(skb)->seq;
542 pr_debug("subflow=%p synack seq=%x\n", subflow, subflow->ssn_offset);
543
544 mptcp_get_options(skb, &mp_opt);
545 if (subflow->request_mptcp) {
546 if (!(mp_opt.suboptions & OPTION_MPTCP_MPC_SYNACK)) {
547 MPTCP_INC_STATS(sock_net(sk),
548 MPTCP_MIB_MPCAPABLEACTIVEFALLBACK);
549 mptcp_do_fallback(sk);
550 pr_fallback(msk);
551 goto fallback;
552 }
553
554 if (mp_opt.suboptions & OPTION_MPTCP_CSUMREQD)
555 WRITE_ONCE(msk->csum_enabled, true);
556 if (mp_opt.deny_join_id0)
557 WRITE_ONCE(msk->pm.remote_deny_join_id0, true);
558 subflow->mp_capable = 1;
559 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVEACK);
560 mptcp_finish_connect(sk);
561 mptcp_active_enable(parent);
562 mptcp_propagate_state(parent, sk, subflow, &mp_opt);
563 } else if (subflow->request_join) {
564 u8 hmac[SHA256_DIGEST_SIZE];
565
566 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_SYNACK)) {
567 subflow->reset_reason = MPTCP_RST_EMPTCP;
568 goto do_reset;
569 }
570
571 subflow->backup = mp_opt.backup;
572 subflow->thmac = mp_opt.thmac;
573 subflow->remote_nonce = mp_opt.nonce;
574 WRITE_ONCE(subflow->remote_id, mp_opt.join_id);
575 pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u backup=%d\n",
576 subflow, subflow->thmac, subflow->remote_nonce,
577 subflow->backup);
578
579 if (!subflow_thmac_valid(subflow)) {
580 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC);
581 subflow->reset_reason = MPTCP_RST_EMPTCP;
582 goto do_reset;
583 }
584
585 if (!mptcp_finish_join(sk))
586 goto do_reset;
587
588 subflow_generate_hmac(subflow->local_key, subflow->remote_key,
589 subflow->local_nonce,
590 subflow->remote_nonce,
591 hmac);
592 memcpy(subflow->hmac, hmac, MPTCPOPT_HMAC_LEN);
593
594 subflow->mp_join = 1;
595 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX);
596
597 if (subflow->backup)
598 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKBACKUPRX);
599
600 if (subflow_use_different_dport(msk, sk)) {
601 pr_debug("synack inet_dport=%d %d\n",
602 ntohs(inet_sk(sk)->inet_dport),
603 ntohs(inet_sk(parent)->inet_dport));
604 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINPORTSYNACKRX);
605 }
606 } else if (mptcp_check_fallback(sk)) {
607 /* It looks like MPTCP is blocked, while TCP is not */
608 if (subflow->mpc_drop)
609 mptcp_active_disable(parent);
610fallback:
611 mptcp_propagate_state(parent, sk, subflow, NULL);
612 }
613 return;
614
615do_reset:
616 subflow->reset_transient = 0;
617 mptcp_subflow_reset(sk);
618}
619
620static void subflow_set_local_id(struct mptcp_subflow_context *subflow, int local_id)
621{
622 WARN_ON_ONCE(local_id < 0 || local_id > 255);
623 WRITE_ONCE(subflow->local_id, local_id);
624}
625
626static int subflow_chk_local_id(struct sock *sk)
627{
628 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
629 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
630 int err;
631
632 if (likely(subflow->local_id >= 0))
633 return 0;
634
635 err = mptcp_pm_get_local_id(msk, (struct sock_common *)sk);
636 if (err < 0)
637 return err;
638
639 subflow_set_local_id(subflow, err);
640 subflow->request_bkup = mptcp_pm_is_backup(msk, (struct sock_common *)sk);
641
642 return 0;
643}
644
645static int subflow_rebuild_header(struct sock *sk)
646{
647 int err = subflow_chk_local_id(sk);
648
649 if (unlikely(err < 0))
650 return err;
651
652 return inet_sk_rebuild_header(sk);
653}
654
655#if IS_ENABLED(CONFIG_MPTCP_IPV6)
656static int subflow_v6_rebuild_header(struct sock *sk)
657{
658 int err = subflow_chk_local_id(sk);
659
660 if (unlikely(err < 0))
661 return err;
662
663 return inet6_sk_rebuild_header(sk);
664}
665#endif
666
667static struct request_sock_ops mptcp_subflow_v4_request_sock_ops __ro_after_init;
668static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops __ro_after_init;
669
670static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb)
671{
672 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
673
674 pr_debug("subflow=%p\n", subflow);
675
676 /* Never answer to SYNs sent to broadcast or multicast */
677 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
678 goto drop;
679
680 return tcp_conn_request(&mptcp_subflow_v4_request_sock_ops,
681 &subflow_request_sock_ipv4_ops,
682 sk, skb);
683drop:
684 tcp_listendrop(sk);
685 return 0;
686}
687
688static void subflow_v4_req_destructor(struct request_sock *req)
689{
690 subflow_req_destructor(req);
691 tcp_request_sock_ops.destructor(req);
692}
693
694#if IS_ENABLED(CONFIG_MPTCP_IPV6)
695static struct request_sock_ops mptcp_subflow_v6_request_sock_ops __ro_after_init;
696static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops __ro_after_init;
697static struct inet_connection_sock_af_ops subflow_v6_specific __ro_after_init;
698static struct inet_connection_sock_af_ops subflow_v6m_specific __ro_after_init;
699static struct proto tcpv6_prot_override __ro_after_init;
700
701static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb)
702{
703 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
704
705 pr_debug("subflow=%p\n", subflow);
706
707 if (skb->protocol == htons(ETH_P_IP))
708 return subflow_v4_conn_request(sk, skb);
709
710 if (!ipv6_unicast_destination(skb))
711 goto drop;
712
713 if (ipv6_addr_v4mapped(&ipv6_hdr(skb)->saddr)) {
714 __IP6_INC_STATS(sock_net(sk), NULL, IPSTATS_MIB_INHDRERRORS);
715 return 0;
716 }
717
718 return tcp_conn_request(&mptcp_subflow_v6_request_sock_ops,
719 &subflow_request_sock_ipv6_ops, sk, skb);
720
721drop:
722 tcp_listendrop(sk);
723 return 0; /* don't send reset */
724}
725
726static void subflow_v6_req_destructor(struct request_sock *req)
727{
728 subflow_req_destructor(req);
729 tcp6_request_sock_ops.destructor(req);
730}
731#endif
732
733struct request_sock *mptcp_subflow_reqsk_alloc(const struct request_sock_ops *ops,
734 struct sock *sk_listener,
735 bool attach_listener)
736{
737 if (ops->family == AF_INET)
738 ops = &mptcp_subflow_v4_request_sock_ops;
739#if IS_ENABLED(CONFIG_MPTCP_IPV6)
740 else if (ops->family == AF_INET6)
741 ops = &mptcp_subflow_v6_request_sock_ops;
742#endif
743
744 return inet_reqsk_alloc(ops, sk_listener, attach_listener);
745}
746EXPORT_SYMBOL(mptcp_subflow_reqsk_alloc);
747
748/* validate hmac received in third ACK */
749static bool subflow_hmac_valid(const struct mptcp_subflow_request_sock *subflow_req,
750 const struct mptcp_options_received *mp_opt)
751{
752 struct mptcp_sock *msk = subflow_req->msk;
753 u8 hmac[SHA256_DIGEST_SIZE];
754
755 subflow_generate_hmac(READ_ONCE(msk->remote_key),
756 READ_ONCE(msk->local_key),
757 subflow_req->remote_nonce,
758 subflow_req->local_nonce, hmac);
759
760 return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
761}
762
763static void subflow_ulp_fallback(struct sock *sk,
764 struct mptcp_subflow_context *old_ctx)
765{
766 struct inet_connection_sock *icsk = inet_csk(sk);
767
768 mptcp_subflow_tcp_fallback(sk, old_ctx);
769 icsk->icsk_ulp_ops = NULL;
770 rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
771 tcp_sk(sk)->is_mptcp = 0;
772
773 mptcp_subflow_ops_undo_override(sk);
774}
775
776void mptcp_subflow_drop_ctx(struct sock *ssk)
777{
778 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
779
780 if (!ctx)
781 return;
782
783 list_del(&mptcp_subflow_ctx(ssk)->node);
784 if (inet_csk(ssk)->icsk_ulp_ops) {
785 subflow_ulp_fallback(ssk, ctx);
786 if (ctx->conn)
787 sock_put(ctx->conn);
788 }
789
790 kfree_rcu(ctx, rcu);
791}
792
793void __mptcp_subflow_fully_established(struct mptcp_sock *msk,
794 struct mptcp_subflow_context *subflow,
795 const struct mptcp_options_received *mp_opt)
796{
797 subflow_set_remote_key(msk, subflow, mp_opt);
798 WRITE_ONCE(subflow->fully_established, true);
799 WRITE_ONCE(msk->fully_established, true);
800}
801
802static struct sock *subflow_syn_recv_sock(const struct sock *sk,
803 struct sk_buff *skb,
804 struct request_sock *req,
805 struct dst_entry *dst,
806 struct request_sock *req_unhash,
807 bool *own_req)
808{
809 struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk);
810 struct mptcp_subflow_request_sock *subflow_req;
811 struct mptcp_options_received mp_opt;
812 bool fallback, fallback_is_fatal;
813 enum sk_rst_reason reason;
814 struct mptcp_sock *owner;
815 struct sock *child;
816
817 pr_debug("listener=%p, req=%p, conn=%p\n", listener, req, listener->conn);
818
819 /* After child creation we must look for MPC even when options
820 * are not parsed
821 */
822 mp_opt.suboptions = 0;
823
824 /* hopefully temporary handling for MP_JOIN+syncookie */
825 subflow_req = mptcp_subflow_rsk(req);
826 fallback_is_fatal = tcp_rsk(req)->is_mptcp && subflow_req->mp_join;
827 fallback = !tcp_rsk(req)->is_mptcp;
828 if (fallback)
829 goto create_child;
830
831 /* if the sk is MP_CAPABLE, we try to fetch the client key */
832 if (subflow_req->mp_capable) {
833 /* we can receive and accept an in-window, out-of-order pkt,
834 * which may not carry the MP_CAPABLE opt even on mptcp enabled
835 * paths: always try to extract the peer key, and fallback
836 * for packets missing it.
837 * Even OoO DSS packets coming legitly after dropped or
838 * reordered MPC will cause fallback, but we don't have other
839 * options.
840 */
841 mptcp_get_options(skb, &mp_opt);
842 if (!(mp_opt.suboptions &
843 (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_ACK)))
844 fallback = true;
845
846 } else if (subflow_req->mp_join) {
847 mptcp_get_options(skb, &mp_opt);
848 if (!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK))
849 fallback = true;
850 }
851
852create_child:
853 child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst,
854 req_unhash, own_req);
855
856 if (child && *own_req) {
857 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child);
858
859 tcp_rsk(req)->drop_req = false;
860
861 /* we need to fallback on ctx allocation failure and on pre-reqs
862 * checking above. In the latter scenario we additionally need
863 * to reset the context to non MPTCP status.
864 */
865 if (!ctx || fallback) {
866 if (fallback_is_fatal) {
867 subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP);
868 goto dispose_child;
869 }
870 goto fallback;
871 }
872
873 /* ssk inherits options of listener sk */
874 ctx->setsockopt_seq = listener->setsockopt_seq;
875
876 if (ctx->mp_capable) {
877 ctx->conn = mptcp_sk_clone_init(listener->conn, &mp_opt, child, req);
878 if (!ctx->conn)
879 goto fallback;
880
881 ctx->subflow_id = 1;
882 owner = mptcp_sk(ctx->conn);
883 mptcp_pm_new_connection(owner, child, 1);
884
885 /* with OoO packets we can reach here without ingress
886 * mpc option
887 */
888 if (mp_opt.suboptions & OPTION_MPTCP_MPC_ACK) {
889 mptcp_pm_fully_established(owner, child);
890 ctx->pm_notified = 1;
891 }
892 } else if (ctx->mp_join) {
893 owner = subflow_req->msk;
894 if (!owner) {
895 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
896 goto dispose_child;
897 }
898
899 if (!subflow_hmac_valid(subflow_req, &mp_opt)) {
900 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC);
901 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
902 goto dispose_child;
903 }
904
905 if (!mptcp_can_accept_new_subflow(owner)) {
906 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINREJECTED);
907 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
908 goto dispose_child;
909 }
910
911 /* move the msk reference ownership to the subflow */
912 subflow_req->msk = NULL;
913 ctx->conn = (struct sock *)owner;
914
915 if (subflow_use_different_sport(owner, sk)) {
916 pr_debug("ack inet_sport=%d %d\n",
917 ntohs(inet_sk(sk)->inet_sport),
918 ntohs(inet_sk((struct sock *)owner)->inet_sport));
919 if (!mptcp_pm_sport_in_anno_list(owner, sk)) {
920 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX);
921 subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT);
922 goto dispose_child;
923 }
924 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINPORTACKRX);
925 }
926
927 if (!mptcp_finish_join(child)) {
928 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(child);
929
930 subflow_add_reset_reason(skb, subflow->reset_reason);
931 goto dispose_child;
932 }
933
934 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX);
935 tcp_rsk(req)->drop_req = true;
936 }
937 }
938
939 /* check for expected invariant - should never trigger, just help
940 * catching earlier subtle bugs
941 */
942 WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp &&
943 (!mptcp_subflow_ctx(child) ||
944 !mptcp_subflow_ctx(child)->conn));
945 return child;
946
947dispose_child:
948 mptcp_subflow_drop_ctx(child);
949 tcp_rsk(req)->drop_req = true;
950 inet_csk_prepare_for_destroy_sock(child);
951 tcp_done(child);
952 reason = mptcp_get_rst_reason(skb);
953 req->rsk_ops->send_reset(sk, skb, reason);
954
955 /* The last child reference will be released by the caller */
956 return child;
957
958fallback:
959 if (fallback)
960 SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK);
961 mptcp_subflow_drop_ctx(child);
962 return child;
963}
964
965static struct inet_connection_sock_af_ops subflow_specific __ro_after_init;
966static struct proto tcp_prot_override __ro_after_init;
967
968enum mapping_status {
969 MAPPING_OK,
970 MAPPING_INVALID,
971 MAPPING_EMPTY,
972 MAPPING_DATA_FIN,
973 MAPPING_DUMMY,
974 MAPPING_BAD_CSUM,
975 MAPPING_NODSS
976};
977
978static void dbg_bad_map(struct mptcp_subflow_context *subflow, u32 ssn)
979{
980 pr_debug("Bad mapping: ssn=%d map_seq=%d map_data_len=%d\n",
981 ssn, subflow->map_subflow_seq, subflow->map_data_len);
982}
983
984static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb)
985{
986 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
987 unsigned int skb_consumed;
988
989 skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq;
990 if (unlikely(skb_consumed >= skb->len)) {
991 DEBUG_NET_WARN_ON_ONCE(1);
992 return true;
993 }
994
995 return skb->len - skb_consumed <= subflow->map_data_len -
996 mptcp_subflow_get_map_offset(subflow);
997}
998
999static bool validate_mapping(struct sock *ssk, struct sk_buff *skb)
1000{
1001 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1002 u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
1003
1004 if (unlikely(before(ssn, subflow->map_subflow_seq))) {
1005 /* Mapping covers data later in the subflow stream,
1006 * currently unsupported.
1007 */
1008 dbg_bad_map(subflow, ssn);
1009 return false;
1010 }
1011 if (unlikely(!before(ssn, subflow->map_subflow_seq +
1012 subflow->map_data_len))) {
1013 /* Mapping does covers past subflow data, invalid */
1014 dbg_bad_map(subflow, ssn);
1015 return false;
1016 }
1017 return true;
1018}
1019
1020static enum mapping_status validate_data_csum(struct sock *ssk, struct sk_buff *skb,
1021 bool csum_reqd)
1022{
1023 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1024 u32 offset, seq, delta;
1025 __sum16 csum;
1026 int len;
1027
1028 if (!csum_reqd)
1029 return MAPPING_OK;
1030
1031 /* mapping already validated on previous traversal */
1032 if (subflow->map_csum_len == subflow->map_data_len)
1033 return MAPPING_OK;
1034
1035 /* traverse the receive queue, ensuring it contains a full
1036 * DSS mapping and accumulating the related csum.
1037 * Preserve the accoumlate csum across multiple calls, to compute
1038 * the csum only once
1039 */
1040 delta = subflow->map_data_len - subflow->map_csum_len;
1041 for (;;) {
1042 seq = tcp_sk(ssk)->copied_seq + subflow->map_csum_len;
1043 offset = seq - TCP_SKB_CB(skb)->seq;
1044
1045 /* if the current skb has not been accounted yet, csum its contents
1046 * up to the amount covered by the current DSS
1047 */
1048 if (offset < skb->len) {
1049 __wsum csum;
1050
1051 len = min(skb->len - offset, delta);
1052 csum = skb_checksum(skb, offset, len, 0);
1053 subflow->map_data_csum = csum_block_add(subflow->map_data_csum, csum,
1054 subflow->map_csum_len);
1055
1056 delta -= len;
1057 subflow->map_csum_len += len;
1058 }
1059 if (delta == 0)
1060 break;
1061
1062 if (skb_queue_is_last(&ssk->sk_receive_queue, skb)) {
1063 /* if this subflow is closed, the partial mapping
1064 * will be never completed; flush the pending skbs, so
1065 * that subflow_sched_work_if_closed() can kick in
1066 */
1067 if (unlikely(ssk->sk_state == TCP_CLOSE))
1068 while ((skb = skb_peek(&ssk->sk_receive_queue)))
1069 sk_eat_skb(ssk, skb);
1070
1071 /* not enough data to validate the csum */
1072 return MAPPING_EMPTY;
1073 }
1074
1075 /* the DSS mapping for next skbs will be validated later,
1076 * when a get_mapping_status call will process such skb
1077 */
1078 skb = skb->next;
1079 }
1080
1081 /* note that 'map_data_len' accounts only for the carried data, does
1082 * not include the eventual seq increment due to the data fin,
1083 * while the pseudo header requires the original DSS data len,
1084 * including that
1085 */
1086 csum = __mptcp_make_csum(subflow->map_seq,
1087 subflow->map_subflow_seq,
1088 subflow->map_data_len + subflow->map_data_fin,
1089 subflow->map_data_csum);
1090 if (unlikely(csum)) {
1091 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DATACSUMERR);
1092 return MAPPING_BAD_CSUM;
1093 }
1094
1095 subflow->valid_csum_seen = 1;
1096 return MAPPING_OK;
1097}
1098
1099static enum mapping_status get_mapping_status(struct sock *ssk,
1100 struct mptcp_sock *msk)
1101{
1102 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1103 bool csum_reqd = READ_ONCE(msk->csum_enabled);
1104 struct mptcp_ext *mpext;
1105 struct sk_buff *skb;
1106 u16 data_len;
1107 u64 map_seq;
1108
1109 skb = skb_peek(&ssk->sk_receive_queue);
1110 if (!skb)
1111 return MAPPING_EMPTY;
1112
1113 if (mptcp_check_fallback(ssk))
1114 return MAPPING_DUMMY;
1115
1116 mpext = mptcp_get_ext(skb);
1117 if (!mpext || !mpext->use_map) {
1118 if (!subflow->map_valid && !skb->len) {
1119 /* the TCP stack deliver 0 len FIN pkt to the receive
1120 * queue, that is the only 0len pkts ever expected here,
1121 * and we can admit no mapping only for 0 len pkts
1122 */
1123 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1124 WARN_ONCE(1, "0len seq %d:%d flags %x",
1125 TCP_SKB_CB(skb)->seq,
1126 TCP_SKB_CB(skb)->end_seq,
1127 TCP_SKB_CB(skb)->tcp_flags);
1128 sk_eat_skb(ssk, skb);
1129 return MAPPING_EMPTY;
1130 }
1131
1132 /* If the required DSS has likely been dropped by a middlebox */
1133 if (!subflow->map_valid)
1134 return MAPPING_NODSS;
1135
1136 goto validate_seq;
1137 }
1138
1139 trace_get_mapping_status(mpext);
1140
1141 data_len = mpext->data_len;
1142 if (data_len == 0) {
1143 pr_debug("infinite mapping received\n");
1144 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX);
1145 return MAPPING_INVALID;
1146 }
1147
1148 if (mpext->data_fin == 1) {
1149 u64 data_fin_seq;
1150
1151 if (data_len == 1) {
1152 bool updated = mptcp_update_rcv_data_fin(msk, mpext->data_seq,
1153 mpext->dsn64);
1154 pr_debug("DATA_FIN with no payload seq=%llu\n", mpext->data_seq);
1155 if (subflow->map_valid) {
1156 /* A DATA_FIN might arrive in a DSS
1157 * option before the previous mapping
1158 * has been fully consumed. Continue
1159 * handling the existing mapping.
1160 */
1161 skb_ext_del(skb, SKB_EXT_MPTCP);
1162 return MAPPING_OK;
1163 }
1164
1165 if (updated)
1166 mptcp_schedule_work((struct sock *)msk);
1167
1168 return MAPPING_DATA_FIN;
1169 }
1170
1171 data_fin_seq = mpext->data_seq + data_len - 1;
1172
1173 /* If mpext->data_seq is a 32-bit value, data_fin_seq must also
1174 * be limited to 32 bits.
1175 */
1176 if (!mpext->dsn64)
1177 data_fin_seq &= GENMASK_ULL(31, 0);
1178
1179 mptcp_update_rcv_data_fin(msk, data_fin_seq, mpext->dsn64);
1180 pr_debug("DATA_FIN with mapping seq=%llu dsn64=%d\n",
1181 data_fin_seq, mpext->dsn64);
1182
1183 /* Adjust for DATA_FIN using 1 byte of sequence space */
1184 data_len--;
1185 }
1186
1187 map_seq = mptcp_expand_seq(READ_ONCE(msk->ack_seq), mpext->data_seq, mpext->dsn64);
1188 WRITE_ONCE(mptcp_sk(subflow->conn)->use_64bit_ack, !!mpext->dsn64);
1189
1190 if (subflow->map_valid) {
1191 /* Allow replacing only with an identical map */
1192 if (subflow->map_seq == map_seq &&
1193 subflow->map_subflow_seq == mpext->subflow_seq &&
1194 subflow->map_data_len == data_len &&
1195 subflow->map_csum_reqd == mpext->csum_reqd) {
1196 skb_ext_del(skb, SKB_EXT_MPTCP);
1197 goto validate_csum;
1198 }
1199
1200 /* If this skb data are fully covered by the current mapping,
1201 * the new map would need caching, which is not supported
1202 */
1203 if (skb_is_fully_mapped(ssk, skb)) {
1204 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH);
1205 return MAPPING_INVALID;
1206 }
1207
1208 /* will validate the next map after consuming the current one */
1209 goto validate_csum;
1210 }
1211
1212 subflow->map_seq = map_seq;
1213 subflow->map_subflow_seq = mpext->subflow_seq;
1214 subflow->map_data_len = data_len;
1215 subflow->map_valid = 1;
1216 subflow->map_data_fin = mpext->data_fin;
1217 subflow->mpc_map = mpext->mpc_map;
1218 subflow->map_csum_reqd = mpext->csum_reqd;
1219 subflow->map_csum_len = 0;
1220 subflow->map_data_csum = csum_unfold(mpext->csum);
1221
1222 /* Cfr RFC 8684 Section 3.3.0 */
1223 if (unlikely(subflow->map_csum_reqd != csum_reqd))
1224 return MAPPING_INVALID;
1225
1226 pr_debug("new map seq=%llu subflow_seq=%u data_len=%u csum=%d:%u\n",
1227 subflow->map_seq, subflow->map_subflow_seq,
1228 subflow->map_data_len, subflow->map_csum_reqd,
1229 subflow->map_data_csum);
1230
1231validate_seq:
1232 /* we revalidate valid mapping on new skb, because we must ensure
1233 * the current skb is completely covered by the available mapping
1234 */
1235 if (!validate_mapping(ssk, skb)) {
1236 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSTCPMISMATCH);
1237 return MAPPING_INVALID;
1238 }
1239
1240 skb_ext_del(skb, SKB_EXT_MPTCP);
1241
1242validate_csum:
1243 return validate_data_csum(ssk, skb, csum_reqd);
1244}
1245
1246static void mptcp_subflow_discard_data(struct sock *ssk, struct sk_buff *skb,
1247 u64 limit)
1248{
1249 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1250 bool fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
1251 struct tcp_sock *tp = tcp_sk(ssk);
1252 u32 offset, incr, avail_len;
1253
1254 offset = tp->copied_seq - TCP_SKB_CB(skb)->seq;
1255 if (WARN_ON_ONCE(offset > skb->len))
1256 goto out;
1257
1258 avail_len = skb->len - offset;
1259 incr = limit >= avail_len ? avail_len + fin : limit;
1260
1261 pr_debug("discarding=%d len=%d offset=%d seq=%d\n", incr, skb->len,
1262 offset, subflow->map_subflow_seq);
1263 MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DUPDATA);
1264 tcp_sk(ssk)->copied_seq += incr;
1265
1266out:
1267 if (!before(tcp_sk(ssk)->copied_seq, TCP_SKB_CB(skb)->end_seq))
1268 sk_eat_skb(ssk, skb);
1269 if (mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len)
1270 subflow->map_valid = 0;
1271}
1272
1273static bool subflow_is_done(const struct sock *sk)
1274{
1275 return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE;
1276}
1277
1278/* sched mptcp worker for subflow cleanup if no more data is pending */
1279static void subflow_sched_work_if_closed(struct mptcp_sock *msk, struct sock *ssk)
1280{
1281 struct sock *sk = (struct sock *)msk;
1282
1283 if (likely(ssk->sk_state != TCP_CLOSE &&
1284 (ssk->sk_state != TCP_CLOSE_WAIT ||
1285 inet_sk_state_load(sk) != TCP_ESTABLISHED)))
1286 return;
1287
1288 if (!skb_queue_empty(&ssk->sk_receive_queue))
1289 return;
1290
1291 if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
1292 mptcp_schedule_work(sk);
1293
1294 /* when the fallback subflow closes the rx side, trigger a 'dummy'
1295 * ingress data fin, so that the msk state will follow along
1296 */
1297 if (__mptcp_check_fallback(msk) && subflow_is_done(ssk) &&
1298 msk->first == ssk &&
1299 mptcp_update_rcv_data_fin(msk, READ_ONCE(msk->ack_seq), true))
1300 mptcp_schedule_work(sk);
1301}
1302
1303static void mptcp_subflow_fail(struct mptcp_sock *msk, struct sock *ssk)
1304{
1305 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1306 unsigned long fail_tout;
1307
1308 /* graceful failure can happen only on the MPC subflow */
1309 if (WARN_ON_ONCE(ssk != READ_ONCE(msk->first)))
1310 return;
1311
1312 /* since the close timeout take precedence on the fail one,
1313 * no need to start the latter when the first is already set
1314 */
1315 if (sock_flag((struct sock *)msk, SOCK_DEAD))
1316 return;
1317
1318 /* we don't need extreme accuracy here, use a zero fail_tout as special
1319 * value meaning no fail timeout at all;
1320 */
1321 fail_tout = jiffies + TCP_RTO_MAX;
1322 if (!fail_tout)
1323 fail_tout = 1;
1324 WRITE_ONCE(subflow->fail_tout, fail_tout);
1325 tcp_send_ack(ssk);
1326
1327 mptcp_reset_tout_timer(msk, subflow->fail_tout);
1328}
1329
1330static bool subflow_check_data_avail(struct sock *ssk)
1331{
1332 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1333 enum mapping_status status;
1334 struct mptcp_sock *msk;
1335 struct sk_buff *skb;
1336
1337 if (!skb_peek(&ssk->sk_receive_queue))
1338 WRITE_ONCE(subflow->data_avail, false);
1339 if (subflow->data_avail)
1340 return true;
1341
1342 msk = mptcp_sk(subflow->conn);
1343 for (;;) {
1344 u64 ack_seq;
1345 u64 old_ack;
1346
1347 status = get_mapping_status(ssk, msk);
1348 trace_subflow_check_data_avail(status, skb_peek(&ssk->sk_receive_queue));
1349 if (unlikely(status == MAPPING_INVALID || status == MAPPING_DUMMY ||
1350 status == MAPPING_BAD_CSUM || status == MAPPING_NODSS))
1351 goto fallback;
1352
1353 if (status != MAPPING_OK)
1354 goto no_data;
1355
1356 skb = skb_peek(&ssk->sk_receive_queue);
1357 if (WARN_ON_ONCE(!skb))
1358 goto no_data;
1359
1360 if (unlikely(!READ_ONCE(msk->can_ack)))
1361 goto fallback;
1362
1363 old_ack = READ_ONCE(msk->ack_seq);
1364 ack_seq = mptcp_subflow_get_mapped_dsn(subflow);
1365 pr_debug("msk ack_seq=%llx subflow ack_seq=%llx\n", old_ack,
1366 ack_seq);
1367 if (unlikely(before64(ack_seq, old_ack))) {
1368 mptcp_subflow_discard_data(ssk, skb, old_ack - ack_seq);
1369 continue;
1370 }
1371
1372 WRITE_ONCE(subflow->data_avail, true);
1373 break;
1374 }
1375 return true;
1376
1377no_data:
1378 subflow_sched_work_if_closed(msk, ssk);
1379 return false;
1380
1381fallback:
1382 if (!__mptcp_check_fallback(msk)) {
1383 /* RFC 8684 section 3.7. */
1384 if (status == MAPPING_BAD_CSUM &&
1385 (subflow->mp_join || subflow->valid_csum_seen)) {
1386 subflow->send_mp_fail = 1;
1387
1388 if (!READ_ONCE(msk->allow_infinite_fallback)) {
1389 subflow->reset_transient = 0;
1390 subflow->reset_reason = MPTCP_RST_EMIDDLEBOX;
1391 goto reset;
1392 }
1393 mptcp_subflow_fail(msk, ssk);
1394 WRITE_ONCE(subflow->data_avail, true);
1395 return true;
1396 }
1397
1398 if (!READ_ONCE(msk->allow_infinite_fallback)) {
1399 /* fatal protocol error, close the socket.
1400 * subflow_error_report() will introduce the appropriate barriers
1401 */
1402 subflow->reset_transient = 0;
1403 subflow->reset_reason = status == MAPPING_NODSS ?
1404 MPTCP_RST_EMIDDLEBOX :
1405 MPTCP_RST_EMPTCP;
1406
1407reset:
1408 WRITE_ONCE(ssk->sk_err, EBADMSG);
1409 tcp_set_state(ssk, TCP_CLOSE);
1410 while ((skb = skb_peek(&ssk->sk_receive_queue)))
1411 sk_eat_skb(ssk, skb);
1412 mptcp_send_active_reset_reason(ssk);
1413 WRITE_ONCE(subflow->data_avail, false);
1414 return false;
1415 }
1416
1417 mptcp_do_fallback(ssk);
1418 }
1419
1420 skb = skb_peek(&ssk->sk_receive_queue);
1421 subflow->map_valid = 1;
1422 subflow->map_seq = READ_ONCE(msk->ack_seq);
1423 subflow->map_data_len = skb->len;
1424 subflow->map_subflow_seq = tcp_sk(ssk)->copied_seq - subflow->ssn_offset;
1425 WRITE_ONCE(subflow->data_avail, true);
1426 return true;
1427}
1428
1429bool mptcp_subflow_data_available(struct sock *sk)
1430{
1431 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1432
1433 /* check if current mapping is still valid */
1434 if (subflow->map_valid &&
1435 mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) {
1436 subflow->map_valid = 0;
1437 WRITE_ONCE(subflow->data_avail, false);
1438
1439 pr_debug("Done with mapping: seq=%u data_len=%u\n",
1440 subflow->map_subflow_seq,
1441 subflow->map_data_len);
1442 }
1443
1444 return subflow_check_data_avail(sk);
1445}
1446
1447/* If ssk has an mptcp parent socket, use the mptcp rcvbuf occupancy,
1448 * not the ssk one.
1449 *
1450 * In mptcp, rwin is about the mptcp-level connection data.
1451 *
1452 * Data that is still on the ssk rx queue can thus be ignored,
1453 * as far as mptcp peer is concerned that data is still inflight.
1454 * DSS ACK is updated when skb is moved to the mptcp rx queue.
1455 */
1456void mptcp_space(const struct sock *ssk, int *space, int *full_space)
1457{
1458 const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
1459 const struct sock *sk = subflow->conn;
1460
1461 *space = __mptcp_space(sk);
1462 *full_space = mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));
1463}
1464
1465static void subflow_error_report(struct sock *ssk)
1466{
1467 struct sock *sk = mptcp_subflow_ctx(ssk)->conn;
1468
1469 /* bail early if this is a no-op, so that we avoid introducing a
1470 * problematic lockdep dependency between TCP accept queue lock
1471 * and msk socket spinlock
1472 */
1473 if (!sk->sk_socket)
1474 return;
1475
1476 mptcp_data_lock(sk);
1477 if (!sock_owned_by_user(sk))
1478 __mptcp_error_report(sk);
1479 else
1480 __set_bit(MPTCP_ERROR_REPORT, &mptcp_sk(sk)->cb_flags);
1481 mptcp_data_unlock(sk);
1482}
1483
1484static void subflow_data_ready(struct sock *sk)
1485{
1486 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1487 u16 state = 1 << inet_sk_state_load(sk);
1488 struct sock *parent = subflow->conn;
1489 struct mptcp_sock *msk;
1490
1491 trace_sk_data_ready(sk);
1492
1493 msk = mptcp_sk(parent);
1494 if (state & TCPF_LISTEN) {
1495 /* MPJ subflow are removed from accept queue before reaching here,
1496 * avoid stray wakeups
1497 */
1498 if (reqsk_queue_empty(&inet_csk(sk)->icsk_accept_queue))
1499 return;
1500
1501 parent->sk_data_ready(parent);
1502 return;
1503 }
1504
1505 WARN_ON_ONCE(!__mptcp_check_fallback(msk) && !subflow->mp_capable &&
1506 !subflow->mp_join && !(state & TCPF_CLOSE));
1507
1508 if (mptcp_subflow_data_available(sk)) {
1509 mptcp_data_ready(parent, sk);
1510
1511 /* subflow-level lowat test are not relevant.
1512 * respect the msk-level threshold eventually mandating an immediate ack
1513 */
1514 if (mptcp_data_avail(msk) < parent->sk_rcvlowat &&
1515 (tcp_sk(sk)->rcv_nxt - tcp_sk(sk)->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss)
1516 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_NOW;
1517 } else if (unlikely(sk->sk_err)) {
1518 subflow_error_report(sk);
1519 }
1520}
1521
1522static void subflow_write_space(struct sock *ssk)
1523{
1524 struct sock *sk = mptcp_subflow_ctx(ssk)->conn;
1525
1526 mptcp_propagate_sndbuf(sk, ssk);
1527 mptcp_write_space(sk);
1528}
1529
1530static const struct inet_connection_sock_af_ops *
1531subflow_default_af_ops(struct sock *sk)
1532{
1533#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1534 if (sk->sk_family == AF_INET6)
1535 return &subflow_v6_specific;
1536#endif
1537 return &subflow_specific;
1538}
1539
1540#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1541void mptcpv6_handle_mapped(struct sock *sk, bool mapped)
1542{
1543 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1544 struct inet_connection_sock *icsk = inet_csk(sk);
1545 const struct inet_connection_sock_af_ops *target;
1546
1547 target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk);
1548
1549 pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d\n",
1550 subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped);
1551
1552 if (likely(icsk->icsk_af_ops == target))
1553 return;
1554
1555 subflow->icsk_af_ops = icsk->icsk_af_ops;
1556 icsk->icsk_af_ops = target;
1557}
1558#endif
1559
1560void mptcp_info2sockaddr(const struct mptcp_addr_info *info,
1561 struct sockaddr_storage *addr,
1562 unsigned short family)
1563{
1564 memset(addr, 0, sizeof(*addr));
1565 addr->ss_family = family;
1566 if (addr->ss_family == AF_INET) {
1567 struct sockaddr_in *in_addr = (struct sockaddr_in *)addr;
1568
1569 if (info->family == AF_INET)
1570 in_addr->sin_addr = info->addr;
1571#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1572 else if (ipv6_addr_v4mapped(&info->addr6))
1573 in_addr->sin_addr.s_addr = info->addr6.s6_addr32[3];
1574#endif
1575 in_addr->sin_port = info->port;
1576 }
1577#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1578 else if (addr->ss_family == AF_INET6) {
1579 struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr;
1580
1581 if (info->family == AF_INET)
1582 ipv6_addr_set_v4mapped(info->addr.s_addr,
1583 &in6_addr->sin6_addr);
1584 else
1585 in6_addr->sin6_addr = info->addr6;
1586 in6_addr->sin6_port = info->port;
1587 }
1588#endif
1589}
1590
1591int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_pm_local *local,
1592 const struct mptcp_addr_info *remote)
1593{
1594 struct mptcp_sock *msk = mptcp_sk(sk);
1595 struct mptcp_subflow_context *subflow;
1596 int local_id = local->addr.id;
1597 struct sockaddr_storage addr;
1598 int remote_id = remote->id;
1599 int err = -ENOTCONN;
1600 struct socket *sf;
1601 struct sock *ssk;
1602 u32 remote_token;
1603 int addrlen;
1604
1605 /* The userspace PM sent the request too early? */
1606 if (!mptcp_is_fully_established(sk))
1607 goto err_out;
1608
1609 err = mptcp_subflow_create_socket(sk, local->addr.family, &sf);
1610 if (err) {
1611 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCREATSKERR);
1612 pr_debug("msk=%p local=%d remote=%d create sock error: %d\n",
1613 msk, local_id, remote_id, err);
1614 goto err_out;
1615 }
1616
1617 ssk = sf->sk;
1618 subflow = mptcp_subflow_ctx(ssk);
1619 do {
1620 get_random_bytes(&subflow->local_nonce, sizeof(u32));
1621 } while (!subflow->local_nonce);
1622
1623 /* if 'IPADDRANY', the ID will be set later, after the routing */
1624 if (local->addr.family == AF_INET) {
1625 if (!local->addr.addr.s_addr)
1626 local_id = -1;
1627#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1628 } else if (sk->sk_family == AF_INET6) {
1629 if (ipv6_addr_any(&local->addr.addr6))
1630 local_id = -1;
1631#endif
1632 }
1633
1634 if (local_id >= 0)
1635 subflow_set_local_id(subflow, local_id);
1636
1637 subflow->remote_key_valid = 1;
1638 subflow->remote_key = READ_ONCE(msk->remote_key);
1639 subflow->local_key = READ_ONCE(msk->local_key);
1640 subflow->token = msk->token;
1641 mptcp_info2sockaddr(&local->addr, &addr, ssk->sk_family);
1642
1643 addrlen = sizeof(struct sockaddr_in);
1644#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1645 if (addr.ss_family == AF_INET6)
1646 addrlen = sizeof(struct sockaddr_in6);
1647#endif
1648 ssk->sk_bound_dev_if = local->ifindex;
1649 err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen);
1650 if (err) {
1651 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXBINDERR);
1652 pr_debug("msk=%p local=%d remote=%d bind error: %d\n",
1653 msk, local_id, remote_id, err);
1654 goto failed;
1655 }
1656
1657 mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL);
1658 pr_debug("msk=%p remote_token=%u local_id=%d remote_id=%d\n", msk,
1659 remote_token, local_id, remote_id);
1660 subflow->remote_token = remote_token;
1661 WRITE_ONCE(subflow->remote_id, remote_id);
1662 subflow->request_join = 1;
1663 subflow->request_bkup = !!(local->flags & MPTCP_PM_ADDR_FLAG_BACKUP);
1664 subflow->subflow_id = msk->subflow_id++;
1665 mptcp_info2sockaddr(remote, &addr, ssk->sk_family);
1666
1667 sock_hold(ssk);
1668 list_add_tail(&subflow->node, &msk->conn_list);
1669 err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK);
1670 if (err && err != -EINPROGRESS) {
1671 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTXCONNECTERR);
1672 pr_debug("msk=%p local=%d remote=%d connect error: %d\n",
1673 msk, local_id, remote_id, err);
1674 goto failed_unlink;
1675 }
1676
1677 MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNTX);
1678
1679 /* discard the subflow socket */
1680 mptcp_sock_graft(ssk, sk->sk_socket);
1681 iput(SOCK_INODE(sf));
1682 WRITE_ONCE(msk->allow_infinite_fallback, false);
1683 mptcp_stop_tout_timer(sk);
1684 return 0;
1685
1686failed_unlink:
1687 list_del(&subflow->node);
1688 sock_put(mptcp_subflow_tcp_sock(subflow));
1689
1690failed:
1691 subflow->disposable = 1;
1692 sock_release(sf);
1693
1694err_out:
1695 /* we account subflows before the creation, and this failures will not
1696 * be caught by sk_state_change()
1697 */
1698 mptcp_pm_close_subflow(msk);
1699 return err;
1700}
1701
1702static void mptcp_attach_cgroup(struct sock *parent, struct sock *child)
1703{
1704#ifdef CONFIG_SOCK_CGROUP_DATA
1705 struct sock_cgroup_data *parent_skcd = &parent->sk_cgrp_data,
1706 *child_skcd = &child->sk_cgrp_data;
1707
1708 /* only the additional subflows created by kworkers have to be modified */
1709 if (cgroup_id(sock_cgroup_ptr(parent_skcd)) !=
1710 cgroup_id(sock_cgroup_ptr(child_skcd))) {
1711#ifdef CONFIG_MEMCG
1712 struct mem_cgroup *memcg = parent->sk_memcg;
1713
1714 mem_cgroup_sk_free(child);
1715 if (memcg && css_tryget(&memcg->css))
1716 child->sk_memcg = memcg;
1717#endif /* CONFIG_MEMCG */
1718
1719 cgroup_sk_free(child_skcd);
1720 *child_skcd = *parent_skcd;
1721 cgroup_sk_clone(child_skcd);
1722 }
1723#endif /* CONFIG_SOCK_CGROUP_DATA */
1724}
1725
1726static void mptcp_subflow_ops_override(struct sock *ssk)
1727{
1728#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1729 if (ssk->sk_prot == &tcpv6_prot)
1730 ssk->sk_prot = &tcpv6_prot_override;
1731 else
1732#endif
1733 ssk->sk_prot = &tcp_prot_override;
1734}
1735
1736static void mptcp_subflow_ops_undo_override(struct sock *ssk)
1737{
1738#if IS_ENABLED(CONFIG_MPTCP_IPV6)
1739 if (ssk->sk_prot == &tcpv6_prot_override)
1740 ssk->sk_prot = &tcpv6_prot;
1741 else
1742#endif
1743 ssk->sk_prot = &tcp_prot;
1744}
1745
1746int mptcp_subflow_create_socket(struct sock *sk, unsigned short family,
1747 struct socket **new_sock)
1748{
1749 struct mptcp_subflow_context *subflow;
1750 struct net *net = sock_net(sk);
1751 struct socket *sf;
1752 int err;
1753
1754 /* un-accepted server sockets can reach here - on bad configuration
1755 * bail early to avoid greater trouble later
1756 */
1757 if (unlikely(!sk->sk_socket))
1758 return -EINVAL;
1759
1760 err = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP, &sf);
1761 if (err)
1762 return err;
1763
1764 lock_sock_nested(sf->sk, SINGLE_DEPTH_NESTING);
1765
1766 err = security_mptcp_add_subflow(sk, sf->sk);
1767 if (err)
1768 goto err_free;
1769
1770 /* the newly created socket has to be in the same cgroup as its parent */
1771 mptcp_attach_cgroup(sk, sf->sk);
1772
1773 /* kernel sockets do not by default acquire net ref, but TCP timer
1774 * needs it.
1775 * Update ns_tracker to current stack trace and refcounted tracker.
1776 */
1777 sk_net_refcnt_upgrade(sf->sk);
1778 err = tcp_set_ulp(sf->sk, "mptcp");
1779 if (err)
1780 goto err_free;
1781
1782 mptcp_sockopt_sync_locked(mptcp_sk(sk), sf->sk);
1783 release_sock(sf->sk);
1784
1785 /* the newly created socket really belongs to the owning MPTCP
1786 * socket, even if for additional subflows the allocation is performed
1787 * by a kernel workqueue. Adjust inode references, so that the
1788 * procfs/diag interfaces really show this one belonging to the correct
1789 * user.
1790 */
1791 SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino;
1792 SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid;
1793 SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid;
1794
1795 subflow = mptcp_subflow_ctx(sf->sk);
1796 pr_debug("subflow=%p\n", subflow);
1797
1798 *new_sock = sf;
1799 sock_hold(sk);
1800 subflow->conn = sk;
1801 mptcp_subflow_ops_override(sf->sk);
1802
1803 return 0;
1804
1805err_free:
1806 release_sock(sf->sk);
1807 sock_release(sf);
1808 return err;
1809}
1810
1811static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk,
1812 gfp_t priority)
1813{
1814 struct inet_connection_sock *icsk = inet_csk(sk);
1815 struct mptcp_subflow_context *ctx;
1816
1817 ctx = kzalloc(sizeof(*ctx), priority);
1818 if (!ctx)
1819 return NULL;
1820
1821 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
1822 INIT_LIST_HEAD(&ctx->node);
1823 INIT_LIST_HEAD(&ctx->delegated_node);
1824
1825 pr_debug("subflow=%p\n", ctx);
1826
1827 ctx->tcp_sock = sk;
1828 WRITE_ONCE(ctx->local_id, -1);
1829
1830 return ctx;
1831}
1832
1833static void __subflow_state_change(struct sock *sk)
1834{
1835 struct socket_wq *wq;
1836
1837 rcu_read_lock();
1838 wq = rcu_dereference(sk->sk_wq);
1839 if (skwq_has_sleeper(wq))
1840 wake_up_interruptible_all(&wq->wait);
1841 rcu_read_unlock();
1842}
1843
1844static void subflow_state_change(struct sock *sk)
1845{
1846 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1847 struct sock *parent = subflow->conn;
1848 struct mptcp_sock *msk;
1849
1850 __subflow_state_change(sk);
1851
1852 msk = mptcp_sk(parent);
1853 if (subflow_simultaneous_connect(sk)) {
1854 mptcp_do_fallback(sk);
1855 pr_fallback(msk);
1856 subflow->conn_finished = 1;
1857 mptcp_propagate_state(parent, sk, subflow, NULL);
1858 }
1859
1860 /* as recvmsg() does not acquire the subflow socket for ssk selection
1861 * a fin packet carrying a DSS can be unnoticed if we don't trigger
1862 * the data available machinery here.
1863 */
1864 if (mptcp_subflow_data_available(sk))
1865 mptcp_data_ready(parent, sk);
1866 else if (unlikely(sk->sk_err))
1867 subflow_error_report(sk);
1868
1869 subflow_sched_work_if_closed(mptcp_sk(parent), sk);
1870}
1871
1872void mptcp_subflow_queue_clean(struct sock *listener_sk, struct sock *listener_ssk)
1873{
1874 struct request_sock_queue *queue = &inet_csk(listener_ssk)->icsk_accept_queue;
1875 struct request_sock *req, *head, *tail;
1876 struct mptcp_subflow_context *subflow;
1877 struct sock *sk, *ssk;
1878
1879 /* Due to lock dependencies no relevant lock can be acquired under rskq_lock.
1880 * Splice the req list, so that accept() can not reach the pending ssk after
1881 * the listener socket is released below.
1882 */
1883 spin_lock_bh(&queue->rskq_lock);
1884 head = queue->rskq_accept_head;
1885 tail = queue->rskq_accept_tail;
1886 queue->rskq_accept_head = NULL;
1887 queue->rskq_accept_tail = NULL;
1888 spin_unlock_bh(&queue->rskq_lock);
1889
1890 if (!head)
1891 return;
1892
1893 /* can't acquire the msk socket lock under the subflow one,
1894 * or will cause ABBA deadlock
1895 */
1896 release_sock(listener_ssk);
1897
1898 for (req = head; req; req = req->dl_next) {
1899 ssk = req->sk;
1900 if (!sk_is_mptcp(ssk))
1901 continue;
1902
1903 subflow = mptcp_subflow_ctx(ssk);
1904 if (!subflow || !subflow->conn)
1905 continue;
1906
1907 sk = subflow->conn;
1908 sock_hold(sk);
1909
1910 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
1911 __mptcp_unaccepted_force_close(sk);
1912 release_sock(sk);
1913
1914 /* lockdep will report a false positive ABBA deadlock
1915 * between cancel_work_sync and the listener socket.
1916 * The involved locks belong to different sockets WRT
1917 * the existing AB chain.
1918 * Using a per socket key is problematic as key
1919 * deregistration requires process context and must be
1920 * performed at socket disposal time, in atomic
1921 * context.
1922 * Just tell lockdep to consider the listener socket
1923 * released here.
1924 */
1925 mutex_release(&listener_sk->sk_lock.dep_map, _RET_IP_);
1926 mptcp_cancel_work(sk);
1927 mutex_acquire(&listener_sk->sk_lock.dep_map, 0, 0, _RET_IP_);
1928
1929 sock_put(sk);
1930 }
1931
1932 /* we are still under the listener msk socket lock */
1933 lock_sock_nested(listener_ssk, SINGLE_DEPTH_NESTING);
1934
1935 /* restore the listener queue, to let the TCP code clean it up */
1936 spin_lock_bh(&queue->rskq_lock);
1937 WARN_ON_ONCE(queue->rskq_accept_head);
1938 queue->rskq_accept_head = head;
1939 queue->rskq_accept_tail = tail;
1940 spin_unlock_bh(&queue->rskq_lock);
1941}
1942
1943static int subflow_ulp_init(struct sock *sk)
1944{
1945 struct inet_connection_sock *icsk = inet_csk(sk);
1946 struct mptcp_subflow_context *ctx;
1947 struct tcp_sock *tp = tcp_sk(sk);
1948 int err = 0;
1949
1950 /* disallow attaching ULP to a socket unless it has been
1951 * created with sock_create_kern()
1952 */
1953 if (!sk->sk_kern_sock) {
1954 err = -EOPNOTSUPP;
1955 goto out;
1956 }
1957
1958 ctx = subflow_create_ctx(sk, GFP_KERNEL);
1959 if (!ctx) {
1960 err = -ENOMEM;
1961 goto out;
1962 }
1963
1964 pr_debug("subflow=%p, family=%d\n", ctx, sk->sk_family);
1965
1966 tp->is_mptcp = 1;
1967 ctx->icsk_af_ops = icsk->icsk_af_ops;
1968 icsk->icsk_af_ops = subflow_default_af_ops(sk);
1969 ctx->tcp_state_change = sk->sk_state_change;
1970 ctx->tcp_error_report = sk->sk_error_report;
1971
1972 WARN_ON_ONCE(sk->sk_data_ready != sock_def_readable);
1973 WARN_ON_ONCE(sk->sk_write_space != sk_stream_write_space);
1974
1975 sk->sk_data_ready = subflow_data_ready;
1976 sk->sk_write_space = subflow_write_space;
1977 sk->sk_state_change = subflow_state_change;
1978 sk->sk_error_report = subflow_error_report;
1979out:
1980 return err;
1981}
1982
1983static void subflow_ulp_release(struct sock *ssk)
1984{
1985 struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(ssk);
1986 bool release = true;
1987 struct sock *sk;
1988
1989 if (!ctx)
1990 return;
1991
1992 sk = ctx->conn;
1993 if (sk) {
1994 /* if the msk has been orphaned, keep the ctx
1995 * alive, will be freed by __mptcp_close_ssk(),
1996 * when the subflow is still unaccepted
1997 */
1998 release = ctx->disposable || list_empty(&ctx->node);
1999
2000 /* inet_child_forget() does not call sk_state_change(),
2001 * explicitly trigger the socket close machinery
2002 */
2003 if (!release && !test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW,
2004 &mptcp_sk(sk)->flags))
2005 mptcp_schedule_work(sk);
2006 sock_put(sk);
2007 }
2008
2009 mptcp_subflow_ops_undo_override(ssk);
2010 if (release)
2011 kfree_rcu(ctx, rcu);
2012}
2013
2014static void subflow_ulp_clone(const struct request_sock *req,
2015 struct sock *newsk,
2016 const gfp_t priority)
2017{
2018 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
2019 struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk);
2020 struct mptcp_subflow_context *new_ctx;
2021
2022 if (!tcp_rsk(req)->is_mptcp ||
2023 (!subflow_req->mp_capable && !subflow_req->mp_join)) {
2024 subflow_ulp_fallback(newsk, old_ctx);
2025 return;
2026 }
2027
2028 new_ctx = subflow_create_ctx(newsk, priority);
2029 if (!new_ctx) {
2030 subflow_ulp_fallback(newsk, old_ctx);
2031 return;
2032 }
2033
2034 new_ctx->conn_finished = 1;
2035 new_ctx->icsk_af_ops = old_ctx->icsk_af_ops;
2036 new_ctx->tcp_state_change = old_ctx->tcp_state_change;
2037 new_ctx->tcp_error_report = old_ctx->tcp_error_report;
2038 new_ctx->rel_write_seq = 1;
2039
2040 if (subflow_req->mp_capable) {
2041 /* see comments in subflow_syn_recv_sock(), MPTCP connection
2042 * is fully established only after we receive the remote key
2043 */
2044 new_ctx->mp_capable = 1;
2045 new_ctx->local_key = subflow_req->local_key;
2046 new_ctx->token = subflow_req->token;
2047 new_ctx->ssn_offset = subflow_req->ssn_offset;
2048 new_ctx->idsn = subflow_req->idsn;
2049
2050 /* this is the first subflow, id is always 0 */
2051 subflow_set_local_id(new_ctx, 0);
2052 } else if (subflow_req->mp_join) {
2053 new_ctx->ssn_offset = subflow_req->ssn_offset;
2054 new_ctx->mp_join = 1;
2055 WRITE_ONCE(new_ctx->fully_established, true);
2056 new_ctx->remote_key_valid = 1;
2057 new_ctx->backup = subflow_req->backup;
2058 new_ctx->request_bkup = subflow_req->request_bkup;
2059 WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id);
2060 new_ctx->token = subflow_req->token;
2061 new_ctx->thmac = subflow_req->thmac;
2062
2063 /* the subflow req id is valid, fetched via subflow_check_req()
2064 * and subflow_token_join_request()
2065 */
2066 subflow_set_local_id(new_ctx, subflow_req->local_id);
2067 }
2068}
2069
2070static void tcp_release_cb_override(struct sock *ssk)
2071{
2072 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
2073 long status;
2074
2075 /* process and clear all the pending actions, but leave the subflow into
2076 * the napi queue. To respect locking, only the same CPU that originated
2077 * the action can touch the list. mptcp_napi_poll will take care of it.
2078 */
2079 status = set_mask_bits(&subflow->delegated_status, MPTCP_DELEGATE_ACTIONS_MASK, 0);
2080 if (status)
2081 mptcp_subflow_process_delegated(ssk, status);
2082
2083 tcp_release_cb(ssk);
2084}
2085
2086static int tcp_abort_override(struct sock *ssk, int err)
2087{
2088 /* closing a listener subflow requires a great deal of care.
2089 * keep it simple and just prevent such operation
2090 */
2091 if (inet_sk_state_load(ssk) == TCP_LISTEN)
2092 return -EINVAL;
2093
2094 return tcp_abort(ssk, err);
2095}
2096
2097static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = {
2098 .name = "mptcp",
2099 .owner = THIS_MODULE,
2100 .init = subflow_ulp_init,
2101 .release = subflow_ulp_release,
2102 .clone = subflow_ulp_clone,
2103};
2104
2105static int subflow_ops_init(struct request_sock_ops *subflow_ops)
2106{
2107 subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock);
2108
2109 subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,
2110 subflow_ops->obj_size, 0,
2111 SLAB_ACCOUNT |
2112 SLAB_TYPESAFE_BY_RCU,
2113 NULL);
2114 if (!subflow_ops->slab)
2115 return -ENOMEM;
2116
2117 return 0;
2118}
2119
2120void __init mptcp_subflow_init(void)
2121{
2122 mptcp_subflow_v4_request_sock_ops = tcp_request_sock_ops;
2123 mptcp_subflow_v4_request_sock_ops.slab_name = "request_sock_subflow_v4";
2124 mptcp_subflow_v4_request_sock_ops.destructor = subflow_v4_req_destructor;
2125
2126 if (subflow_ops_init(&mptcp_subflow_v4_request_sock_ops) != 0)
2127 panic("MPTCP: failed to init subflow v4 request sock ops\n");
2128
2129 subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops;
2130 subflow_request_sock_ipv4_ops.route_req = subflow_v4_route_req;
2131 subflow_request_sock_ipv4_ops.send_synack = subflow_v4_send_synack;
2132
2133 subflow_specific = ipv4_specific;
2134 subflow_specific.conn_request = subflow_v4_conn_request;
2135 subflow_specific.syn_recv_sock = subflow_syn_recv_sock;
2136 subflow_specific.sk_rx_dst_set = subflow_finish_connect;
2137 subflow_specific.rebuild_header = subflow_rebuild_header;
2138
2139 tcp_prot_override = tcp_prot;
2140 tcp_prot_override.release_cb = tcp_release_cb_override;
2141 tcp_prot_override.diag_destroy = tcp_abort_override;
2142
2143#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2144 /* In struct mptcp_subflow_request_sock, we assume the TCP request sock
2145 * structures for v4 and v6 have the same size. It should not changed in
2146 * the future but better to make sure to be warned if it is no longer
2147 * the case.
2148 */
2149 BUILD_BUG_ON(sizeof(struct tcp_request_sock) != sizeof(struct tcp6_request_sock));
2150
2151 mptcp_subflow_v6_request_sock_ops = tcp6_request_sock_ops;
2152 mptcp_subflow_v6_request_sock_ops.slab_name = "request_sock_subflow_v6";
2153 mptcp_subflow_v6_request_sock_ops.destructor = subflow_v6_req_destructor;
2154
2155 if (subflow_ops_init(&mptcp_subflow_v6_request_sock_ops) != 0)
2156 panic("MPTCP: failed to init subflow v6 request sock ops\n");
2157
2158 subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops;
2159 subflow_request_sock_ipv6_ops.route_req = subflow_v6_route_req;
2160 subflow_request_sock_ipv6_ops.send_synack = subflow_v6_send_synack;
2161
2162 subflow_v6_specific = ipv6_specific;
2163 subflow_v6_specific.conn_request = subflow_v6_conn_request;
2164 subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock;
2165 subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect;
2166 subflow_v6_specific.rebuild_header = subflow_v6_rebuild_header;
2167
2168 subflow_v6m_specific = subflow_v6_specific;
2169 subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit;
2170 subflow_v6m_specific.send_check = ipv4_specific.send_check;
2171 subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len;
2172 subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced;
2173 subflow_v6m_specific.rebuild_header = subflow_rebuild_header;
2174
2175 tcpv6_prot_override = tcpv6_prot;
2176 tcpv6_prot_override.release_cb = tcp_release_cb_override;
2177 tcpv6_prot_override.diag_destroy = tcp_abort_override;
2178#endif
2179
2180 mptcp_diag_subflow_init(&subflow_ulp_ops);
2181
2182 if (tcp_register_ulp(&subflow_ulp_ops) != 0)
2183 panic("MPTCP: failed to register subflows to ULP\n");
2184}
Linux 6.x block layer and io_uring tree(s)