Commit | Line | Data |
---|---|---|
f870fa0b MM |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Multipath TCP | |
3 | * | |
4 | * Copyright (c) 2017 - 2019, Intel Corporation. | |
5 | */ | |
6 | ||
7 | #ifndef __MPTCP_PROTOCOL_H | |
8 | #define __MPTCP_PROTOCOL_H | |
9 | ||
79c0949e PK |
10 | #include <linux/random.h> |
11 | #include <net/tcp.h> | |
12 | #include <net/inet_connection_sock.h> | |
b911c97c | 13 | #include <uapi/linux/mptcp.h> |
9ab4807c | 14 | #include <net/genetlink.h> |
79c0949e | 15 | |
cc7972ea | 16 | #define MPTCP_SUPPORTED_VERSION 1 |
eda7acdd PK |
17 | |
18 | /* MPTCP option bits */ | |
19 | #define OPTION_MPTCP_MPC_SYN BIT(0) | |
20 | #define OPTION_MPTCP_MPC_SYNACK BIT(1) | |
21 | #define OPTION_MPTCP_MPC_ACK BIT(2) | |
f296234c PK |
22 | #define OPTION_MPTCP_MPJ_SYN BIT(3) |
23 | #define OPTION_MPTCP_MPJ_SYNACK BIT(4) | |
24 | #define OPTION_MPTCP_MPJ_ACK BIT(5) | |
3df523ab | 25 | #define OPTION_MPTCP_ADD_ADDR BIT(6) |
fef6b7ec GT |
26 | #define OPTION_MPTCP_RM_ADDR BIT(7) |
27 | #define OPTION_MPTCP_FASTCLOSE BIT(8) | |
28 | #define OPTION_MPTCP_PRIO BIT(9) | |
29 | #define OPTION_MPTCP_RST BIT(10) | |
1bff1e43 | 30 | #define OPTION_MPTCP_DSS BIT(11) |
c25aeb4e | 31 | #define OPTION_MPTCP_FAIL BIT(12) |
eda7acdd | 32 | |
74c7dfbe PA |
33 | #define OPTION_MPTCP_CSUMREQD BIT(13) |
34 | ||
35 | #define OPTIONS_MPTCP_MPC (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \ | |
36 | OPTION_MPTCP_MPC_ACK) | |
37 | #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ | |
780aa120 | 38 | OPTION_MPTCP_MPJ_ACK) |
74c7dfbe | 39 | |
eda7acdd PK |
40 | /* MPTCP option subtypes */ |
41 | #define MPTCPOPT_MP_CAPABLE 0 | |
42 | #define MPTCPOPT_MP_JOIN 1 | |
43 | #define MPTCPOPT_DSS 2 | |
44 | #define MPTCPOPT_ADD_ADDR 3 | |
45 | #define MPTCPOPT_RM_ADDR 4 | |
46 | #define MPTCPOPT_MP_PRIO 5 | |
47 | #define MPTCPOPT_MP_FAIL 6 | |
48 | #define MPTCPOPT_MP_FASTCLOSE 7 | |
dc87efdb | 49 | #define MPTCPOPT_RST 8 |
eda7acdd PK |
50 | |
51 | /* MPTCP suboption lengths */ | |
cc7972ea | 52 | #define TCPOLEN_MPTCP_MPC_SYN 4 |
eda7acdd PK |
53 | #define TCPOLEN_MPTCP_MPC_SYNACK 12 |
54 | #define TCPOLEN_MPTCP_MPC_ACK 20 | |
cc7972ea | 55 | #define TCPOLEN_MPTCP_MPC_ACK_DATA 22 |
f296234c PK |
56 | #define TCPOLEN_MPTCP_MPJ_SYN 12 |
57 | #define TCPOLEN_MPTCP_MPJ_SYNACK 16 | |
58 | #define TCPOLEN_MPTCP_MPJ_ACK 24 | |
6d0060f6 | 59 | #define TCPOLEN_MPTCP_DSS_BASE 4 |
648ef4b8 | 60 | #define TCPOLEN_MPTCP_DSS_ACK32 4 |
6d0060f6 | 61 | #define TCPOLEN_MPTCP_DSS_ACK64 8 |
648ef4b8 | 62 | #define TCPOLEN_MPTCP_DSS_MAP32 10 |
6d0060f6 MM |
63 | #define TCPOLEN_MPTCP_DSS_MAP64 14 |
64 | #define TCPOLEN_MPTCP_DSS_CHECKSUM 2 | |
3df523ab | 65 | #define TCPOLEN_MPTCP_ADD_ADDR 16 |
27ab92d9 | 66 | #define TCPOLEN_MPTCP_ADD_ADDR_PORT 18 |
3df523ab | 67 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE 8 |
27ab92d9 | 68 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT 10 |
3df523ab | 69 | #define TCPOLEN_MPTCP_ADD_ADDR6 28 |
27ab92d9 | 70 | #define TCPOLEN_MPTCP_ADD_ADDR6_PORT 30 |
3df523ab | 71 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE 20 |
27ab92d9 DC |
72 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT 22 |
73 | #define TCPOLEN_MPTCP_PORT_LEN 2 | |
74 | #define TCPOLEN_MPTCP_PORT_ALIGN 2 | |
6445e17a | 75 | #define TCPOLEN_MPTCP_RM_ADDR_BASE 3 |
ec99a470 DC |
76 | #define TCPOLEN_MPTCP_PRIO 3 |
77 | #define TCPOLEN_MPTCP_PRIO_ALIGN 4 | |
50c504a2 | 78 | #define TCPOLEN_MPTCP_FASTCLOSE 12 |
dc87efdb | 79 | #define TCPOLEN_MPTCP_RST 4 |
c25aeb4e | 80 | #define TCPOLEN_MPTCP_FAIL 12 |
eda7acdd | 81 | |
208e8f66 GT |
82 | #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA) |
83 | ||
ec3edaa7 | 84 | /* MPTCP MP_JOIN flags */ |
f296234c | 85 | #define MPTCPOPT_BACKUP BIT(0) |
ec3edaa7 | 86 | #define MPTCPOPT_THMAC_LEN 8 |
f296234c | 87 | |
eda7acdd PK |
88 | /* MPTCP MP_CAPABLE flags */ |
89 | #define MPTCP_VERSION_MASK (0x0F) | |
90 | #define MPTCP_CAP_CHECKSUM_REQD BIT(7) | |
91 | #define MPTCP_CAP_EXTENSIBILITY BIT(6) | |
bab6b88e | 92 | #define MPTCP_CAP_DENY_JOIN_ID0 BIT(5) |
65492c5a | 93 | #define MPTCP_CAP_HMAC_SHA256 BIT(0) |
bab6b88e | 94 | #define MPTCP_CAP_FLAG_MASK (0x1F) |
eda7acdd | 95 | |
6d0060f6 MM |
96 | /* MPTCP DSS flags */ |
97 | #define MPTCP_DSS_DATA_FIN BIT(4) | |
98 | #define MPTCP_DSS_DSN64 BIT(3) | |
99 | #define MPTCP_DSS_HAS_MAP BIT(2) | |
100 | #define MPTCP_DSS_ACK64 BIT(1) | |
101 | #define MPTCP_DSS_HAS_ACK BIT(0) | |
648ef4b8 MM |
102 | #define MPTCP_DSS_FLAG_MASK (0x1F) |
103 | ||
3df523ab PK |
104 | /* MPTCP ADD_ADDR flags */ |
105 | #define MPTCP_ADDR_ECHO BIT(0) | |
3df523ab | 106 | |
40453a5c GT |
107 | /* MPTCP MP_PRIO flags */ |
108 | #define MPTCP_PRIO_BKUP BIT(0) | |
109 | ||
dc87efdb FW |
110 | /* MPTCP TCPRST flags */ |
111 | #define MPTCP_RST_TRANSIENT BIT(0) | |
112 | ||
e9d09bac | 113 | /* MPTCP socket atomic flags */ |
8edf0864 | 114 | #define MPTCP_NOSPACE 1 |
3b1d6210 | 115 | #define MPTCP_WORK_RTX 2 |
59832e24 | 116 | #define MPTCP_WORK_EOF 3 |
e1ff9e82 | 117 | #define MPTCP_FALLBACK_DONE 4 |
0e4f35d7 | 118 | #define MPTCP_WORK_CLOSE_SUBFLOW 5 |
e9d09bac PA |
119 | |
120 | /* MPTCP socket release cb flags */ | |
121 | #define MPTCP_PUSH_PENDING 1 | |
122 | #define MPTCP_CLEAN_UNA 2 | |
123 | #define MPTCP_ERROR_REPORT 3 | |
124 | #define MPTCP_RETRANSMIT 4 | |
125 | #define MPTCP_FLUSH_JOIN_LIST 5 | |
126 | #define MPTCP_CONNECTED 6 | |
43f5b111 | 127 | #define MPTCP_RESET_SCHEDULER 7 |
6d0060f6 | 128 | |
dfc8d060 DS |
129 | struct mptcp_skb_cb { |
130 | u64 map_seq; | |
131 | u64 end_seq; | |
132 | u32 offset; | |
133 | u8 has_rxtstamp:1; | |
134 | }; | |
135 | ||
136 | #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) | |
137 | ||
eaa2ffab PA |
138 | static inline bool before64(__u64 seq1, __u64 seq2) |
139 | { | |
140 | return (__s64)(seq1 - seq2) < 0; | |
141 | } | |
142 | ||
143 | #define after64(seq2, seq1) before64(seq1, seq2) | |
144 | ||
cfde141e PA |
145 | struct mptcp_options_received { |
146 | u64 sndr_key; | |
147 | u64 rcvr_key; | |
148 | u64 data_ack; | |
149 | u64 data_seq; | |
150 | u32 subflow_seq; | |
151 | u16 data_len; | |
208e8f66 | 152 | __sum16 csum; |
74c7dfbe | 153 | u16 suboptions; |
cfde141e PA |
154 | u32 token; |
155 | u32 nonce; | |
a086aeba | 156 | u16 use_map:1, |
cfde141e PA |
157 | dsn64:1, |
158 | data_fin:1, | |
159 | use_ack:1, | |
160 | ack64:1, | |
161 | mpc_map:1, | |
a086aeba PA |
162 | reset_reason:4, |
163 | reset_transient:1, | |
164 | echo:1, | |
165 | backup:1, | |
166 | deny_join_id0:1, | |
cfde141e | 167 | __unused:2; |
a086aeba PA |
168 | u8 join_id; |
169 | u64 thmac; | |
170 | u8 hmac[MPTCPOPT_HMAC_LEN]; | |
f7dafee1 | 171 | struct mptcp_addr_info addr; |
5c4a824d | 172 | struct mptcp_rm_list rm_list; |
cfde141e | 173 | u64 ahmac; |
5580d41b | 174 | u64 fail_seq; |
cfde141e PA |
175 | }; |
176 | ||
3df523ab PK |
177 | static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field) |
178 | { | |
179 | return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) | | |
180 | ((nib & 0xF) << 8) | field); | |
181 | } | |
182 | ||
1b1c7a0e PK |
183 | enum mptcp_pm_status { |
184 | MPTCP_PM_ADD_ADDR_RECEIVED, | |
84dfe367 | 185 | MPTCP_PM_ADD_ADDR_SEND_ACK, |
d0876b22 | 186 | MPTCP_PM_RM_ADDR_RECEIVED, |
1b1c7a0e PK |
187 | MPTCP_PM_ESTABLISHED, |
188 | MPTCP_PM_SUBFLOW_ESTABLISHED, | |
86e39e04 PA |
189 | MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ |
190 | MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is | |
191 | * accounted int id_avail_bitmap | |
192 | */ | |
1b1c7a0e PK |
193 | }; |
194 | ||
d85a8fde MM |
195 | enum mptcp_pm_type { |
196 | MPTCP_PM_TYPE_KERNEL = 0, | |
197 | MPTCP_PM_TYPE_USERSPACE, | |
198 | ||
199 | __MPTCP_PM_TYPE_NR, | |
200 | __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1, | |
201 | }; | |
202 | ||
86e39e04 PA |
203 | /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */ |
204 | #define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1) | |
205 | ||
13ad9f01 | 206 | enum mptcp_addr_signal_status { |
d91d322a GT |
207 | MPTCP_ADD_ADDR_SIGNAL, |
208 | MPTCP_ADD_ADDR_ECHO, | |
42842a42 | 209 | MPTCP_RM_ADDR_SIGNAL, |
d91d322a GT |
210 | }; |
211 | ||
86e39e04 PA |
212 | /* max value of mptcp_addr_info.id */ |
213 | #define MPTCP_PM_MAX_ADDR_ID U8_MAX | |
214 | ||
1b1c7a0e PK |
215 | struct mptcp_pm_data { |
216 | struct mptcp_addr_info local; | |
217 | struct mptcp_addr_info remote; | |
b6c08380 | 218 | struct list_head anno_list; |
4638de5a | 219 | struct list_head userspace_pm_local_addr_list; |
1b1c7a0e PK |
220 | |
221 | spinlock_t lock; /*protects the whole PM data */ | |
222 | ||
13ad9f01 | 223 | u8 addr_signal; |
1b1c7a0e PK |
224 | bool server_side; |
225 | bool work_pending; | |
226 | bool accept_addr; | |
227 | bool accept_subflow; | |
df377be3 | 228 | bool remote_deny_join_id0; |
1b1c7a0e PK |
229 | u8 add_addr_signaled; |
230 | u8 add_addr_accepted; | |
231 | u8 local_addr_used; | |
d85a8fde | 232 | u8 pm_type; |
1b1c7a0e | 233 | u8 subflows; |
1b1c7a0e | 234 | u8 status; |
86e39e04 | 235 | DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); |
cbde2787 | 236 | struct mptcp_rm_list rm_list_tx; |
b5c55f33 | 237 | struct mptcp_rm_list rm_list_rx; |
1b1c7a0e PK |
238 | }; |
239 | ||
4638de5a KM |
240 | struct mptcp_pm_addr_entry { |
241 | struct list_head list; | |
242 | struct mptcp_addr_info addr; | |
243 | u8 flags; | |
244 | int ifindex; | |
245 | struct socket *lsk; | |
246 | }; | |
247 | ||
18b683bf PA |
248 | struct mptcp_data_frag { |
249 | struct list_head list; | |
250 | u64 data_seq; | |
f0e6a4cf PA |
251 | u16 data_len; |
252 | u16 offset; | |
253 | u16 overhead; | |
254 | u16 already_sent; | |
18b683bf PA |
255 | struct page *page; |
256 | }; | |
257 | ||
f870fa0b MM |
258 | /* MPTCP connection sock */ |
259 | struct mptcp_sock { | |
260 | /* inet_connection_sock must be the first member */ | |
261 | struct inet_connection_sock sk; | |
cec37a6e PK |
262 | u64 local_key; |
263 | u64 remote_key; | |
6d0060f6 | 264 | u64 write_seq; |
eaa2ffab | 265 | u64 snd_nxt; |
6d0060f6 | 266 | u64 ack_seq; |
f3589be0 | 267 | atomic64_t rcv_wnd_sent; |
3721b9b6 | 268 | u64 rcv_data_fin_seq; |
6511882c | 269 | int rmem_fwd_alloc; |
d5f49190 PA |
270 | struct sock *last_snd; |
271 | int snd_burst; | |
ea4ca586 | 272 | int old_wspace; |
1e1d9d6f PA |
273 | u64 recovery_snd_nxt; /* in recovery mode accept up to this seq; |
274 | * recovery related fields are under data_lock | |
275 | * protection | |
276 | */ | |
7439d687 PA |
277 | u64 snd_una; |
278 | u64 wnd_end; | |
b51f9b80 | 279 | unsigned long timer_ival; |
79c0949e | 280 | u32 token; |
87952603 | 281 | int rmem_released; |
648ef4b8 | 282 | unsigned long flags; |
e9d09bac PA |
283 | unsigned long cb_flags; |
284 | unsigned long push_pending; | |
1e1d9d6f | 285 | bool recovery; /* closing subflow write queue reinjected */ |
d22f4988 | 286 | bool can_ack; |
b93df08c | 287 | bool fully_established; |
3721b9b6 | 288 | bool rcv_data_fin; |
7279da61 | 289 | bool snd_data_fin_enable; |
50c504a2 | 290 | bool rcv_fastclose; |
37198e93 | 291 | bool use_64bit_ack; /* Set when we received a 64-bit DSN */ |
752e9067 | 292 | bool csum_enabled; |
0530020a | 293 | bool allow_infinite_fallback; |
3ad14f54 | 294 | u8 mpc_endpoint_id; |
4f6e14bd MG |
295 | u8 recvmsg_inq:1, |
296 | cork:1, | |
54f1944e | 297 | nodelay:1, |
b6985b9b PA |
298 | fastopening:1, |
299 | in_accept_queue:1; | |
80992017 | 300 | struct work_struct work; |
ab174ad8 PA |
301 | struct sk_buff *ooo_last_skb; |
302 | struct rb_root out_of_order_queue; | |
87952603 | 303 | struct sk_buff_head receive_queue; |
cec37a6e | 304 | struct list_head conn_list; |
18b683bf | 305 | struct list_head rtx_queue; |
f0e6a4cf | 306 | struct mptcp_data_frag *first_pending; |
ec3edaa7 | 307 | struct list_head join_list; |
5b825727 PA |
308 | struct socket *subflow; /* outgoing connect/listener/!mp_capable |
309 | * The mptcp ops can safely dereference, using suitable | |
310 | * ONCE annotation, the subflow outside the socket | |
311 | * lock as such sock is freed after close(). | |
312 | */ | |
8ab183de | 313 | struct sock *first; |
1b1c7a0e | 314 | struct mptcp_pm_data pm; |
a6b118fe FW |
315 | struct { |
316 | u32 space; /* bytes copied in last measurement window */ | |
317 | u32 copied; /* bytes copied in this measurement window */ | |
318 | u64 time; /* start time of measurement window */ | |
319 | u64 rtt_us; /* last maximum rtt of subflows */ | |
320 | } rcvq_space; | |
df00b087 FW |
321 | |
322 | u32 setsockopt_seq; | |
20b5759f | 323 | char ca_name[TCP_CA_NAME_MAX]; |
6aeed904 | 324 | struct mptcp_sock *dl_next; |
f870fa0b MM |
325 | }; |
326 | ||
87952603 PA |
327 | #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock) |
328 | #define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock) | |
329 | ||
cec37a6e PK |
330 | #define mptcp_for_each_subflow(__msk, __subflow) \ |
331 | list_for_each_entry(__subflow, &((__msk)->conn_list), node) | |
5efbf6f7 MB |
332 | #define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ |
333 | list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node) | |
cec37a6e | 334 | |
3abc05d9 FW |
335 | static inline void msk_owned_by_me(const struct mptcp_sock *msk) |
336 | { | |
337 | sock_owned_by_me((const struct sock *)msk); | |
338 | } | |
339 | ||
403a40f2 | 340 | #define mptcp_sk(ptr) container_of_const(ptr, struct mptcp_sock, sk.icsk_inet.sk) |
f870fa0b | 341 | |
ce599c51 PA |
342 | /* the msk socket don't use the backlog, also account for the bulk |
343 | * free memory | |
344 | */ | |
345 | static inline int __mptcp_rmem(const struct sock *sk) | |
346 | { | |
347 | return atomic_read(&sk->sk_rmem_alloc) - READ_ONCE(mptcp_sk(sk)->rmem_released); | |
348 | } | |
349 | ||
ea4ca586 PA |
350 | static inline int __mptcp_space(const struct sock *sk) |
351 | { | |
ce599c51 | 352 | return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk)); |
ea4ca586 PA |
353 | } |
354 | ||
f0e6a4cf PA |
355 | static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) |
356 | { | |
357 | const struct mptcp_sock *msk = mptcp_sk(sk); | |
358 | ||
359 | return READ_ONCE(msk->first_pending); | |
360 | } | |
361 | ||
362 | static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) | |
363 | { | |
364 | struct mptcp_sock *msk = mptcp_sk(sk); | |
365 | struct mptcp_data_frag *cur; | |
366 | ||
367 | cur = msk->first_pending; | |
368 | return list_is_last(&cur->list, &msk->rtx_queue) ? NULL : | |
369 | list_next_entry(cur, list); | |
370 | } | |
371 | ||
372 | static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk) | |
373 | { | |
403a40f2 | 374 | const struct mptcp_sock *msk = mptcp_sk(sk); |
f0e6a4cf PA |
375 | |
376 | if (!msk->first_pending) | |
377 | return NULL; | |
378 | ||
379 | if (WARN_ON_ONCE(list_empty(&msk->rtx_queue))) | |
380 | return NULL; | |
381 | ||
382 | return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list); | |
383 | } | |
384 | ||
403a40f2 | 385 | static inline struct mptcp_data_frag *mptcp_rtx_head(struct sock *sk) |
18b683bf PA |
386 | { |
387 | struct mptcp_sock *msk = mptcp_sk(sk); | |
388 | ||
64b9cea7 | 389 | if (msk->snd_una == READ_ONCE(msk->snd_nxt)) |
18b683bf PA |
390 | return NULL; |
391 | ||
a386bc5b | 392 | return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list); |
7948f6cc FW |
393 | } |
394 | ||
d0cc2987 GT |
395 | struct csum_pseudo_header { |
396 | __be64 data_seq; | |
397 | __be32 subflow_seq; | |
398 | __be16 data_len; | |
399 | __sum16 csum; | |
400 | }; | |
401 | ||
cec37a6e PK |
402 | struct mptcp_subflow_request_sock { |
403 | struct tcp_request_sock sk; | |
d22f4988 | 404 | u16 mp_capable : 1, |
cec37a6e | 405 | mp_join : 1, |
06fe1719 | 406 | backup : 1, |
bab6b88e GT |
407 | csum_reqd : 1, |
408 | allow_join_id0 : 1; | |
1b1c7a0e | 409 | u8 local_id; |
f296234c | 410 | u8 remote_id; |
cec37a6e | 411 | u64 local_key; |
79c0949e PK |
412 | u64 idsn; |
413 | u32 token; | |
648ef4b8 | 414 | u32 ssn_offset; |
f296234c PK |
415 | u64 thmac; |
416 | u32 local_nonce; | |
417 | u32 remote_nonce; | |
8fd4de12 | 418 | struct mptcp_sock *msk; |
2c5ebd00 | 419 | struct hlist_nulls_node token_node; |
cec37a6e PK |
420 | }; |
421 | ||
422 | static inline struct mptcp_subflow_request_sock * | |
423 | mptcp_subflow_rsk(const struct request_sock *rsk) | |
424 | { | |
425 | return (struct mptcp_subflow_request_sock *)rsk; | |
426 | } | |
427 | ||
6719331c PA |
428 | enum mptcp_data_avail { |
429 | MPTCP_SUBFLOW_NODATA, | |
430 | MPTCP_SUBFLOW_DATA_AVAIL, | |
6719331c PA |
431 | }; |
432 | ||
b19bc294 PA |
433 | struct mptcp_delegated_action { |
434 | struct napi_struct napi; | |
435 | struct list_head head; | |
436 | }; | |
437 | ||
438 | DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); | |
439 | ||
440 | #define MPTCP_DELEGATE_SEND 0 | |
bcd97734 | 441 | #define MPTCP_DELEGATE_ACK 1 |
b19bc294 | 442 | |
2303f994 PK |
443 | /* MPTCP subflow context */ |
444 | struct mptcp_subflow_context { | |
cec37a6e | 445 | struct list_head node;/* conn_list of subflows */ |
b29fcfb5 | 446 | |
63ec72bd | 447 | struct_group(reset, |
b29fcfb5 | 448 | |
3ce0852c | 449 | unsigned long avg_pacing_rate; /* protected by msk socket lock */ |
cec37a6e PK |
450 | u64 local_key; |
451 | u64 remote_key; | |
79c0949e | 452 | u64 idsn; |
648ef4b8 | 453 | u64 map_seq; |
cc7972ea | 454 | u32 snd_isn; |
79c0949e | 455 | u32 token; |
6d0060f6 | 456 | u32 rel_write_seq; |
648ef4b8 MM |
457 | u32 map_subflow_seq; |
458 | u32 ssn_offset; | |
459 | u32 map_data_len; | |
dd8bcd17 PA |
460 | __wsum map_data_csum; |
461 | u32 map_csum_len; | |
cec37a6e | 462 | u32 request_mptcp : 1, /* send MP_CAPABLE */ |
ec3edaa7 PK |
463 | request_join : 1, /* send MP_JOIN */ |
464 | request_bkup : 1, | |
cec37a6e | 465 | mp_capable : 1, /* remote is MPTCP capable */ |
f296234c | 466 | mp_join : 1, /* remote is JOINing */ |
0be534f5 | 467 | fully_established : 1, /* path validated */ |
f296234c | 468 | pm_notified : 1, /* PM hook called for established status */ |
648ef4b8 MM |
469 | conn_finished : 1, |
470 | map_valid : 1, | |
dd8bcd17 PA |
471 | map_csum_reqd : 1, |
472 | map_data_fin : 1, | |
d22f4988 | 473 | mpc_map : 1, |
f296234c | 474 | backup : 1, |
06706542 | 475 | send_mp_prio : 1, |
c25aeb4e | 476 | send_mp_fail : 1, |
f284c0c7 | 477 | send_fastclose : 1, |
1e39e5a3 | 478 | send_infinite_map : 1, |
d22f4988 | 479 | rx_eof : 1, |
b3ea6b27 | 480 | remote_key_valid : 1, /* received the peer key from */ |
ff5a0b42 | 481 | disposable : 1, /* ctx can be free at ulp release time */ |
4cf86ae8 | 482 | stale : 1, /* unable to snd/rcv data, do not use for xmit */ |
ae66fb2b | 483 | local_id_valid : 1, /* local_id is correctly initialized */ |
dfc8d060 DS |
484 | valid_csum_seen : 1, /* at least one csum validated */ |
485 | is_mptfo : 1, /* subflow is doing TFO */ | |
486 | __unused : 8; | |
6719331c | 487 | enum mptcp_data_avail data_avail; |
f296234c PK |
488 | u32 remote_nonce; |
489 | u64 thmac; | |
490 | u32 local_nonce; | |
ec3edaa7 | 491 | u32 remote_token; |
b3ea6b27 PA |
492 | union { |
493 | u8 hmac[MPTCPOPT_HMAC_LEN]; /* MPJ subflow only */ | |
494 | u64 iasn; /* initial ack sequence number, MPC subflows only */ | |
495 | }; | |
f296234c PK |
496 | u8 local_id; |
497 | u8 remote_id; | |
dc87efdb FW |
498 | u8 reset_seen:1; |
499 | u8 reset_transient:1; | |
500 | u8 reset_reason:4; | |
71b7dec2 | 501 | u8 stale_count; |
648ef4b8 | 502 | |
b19bc294 | 503 | long delegated_status; |
76a13b31 | 504 | unsigned long fail_tout; |
b29fcfb5 | 505 | |
63ec72bd | 506 | ); |
b29fcfb5 | 507 | |
b19bc294 PA |
508 | struct list_head delegated_node; /* link into delegated_action, protected by local BH */ |
509 | ||
71b7dec2 PA |
510 | u32 setsockopt_seq; |
511 | u32 stale_rcv_tstamp; | |
df00b087 | 512 | |
2303f994 PK |
513 | struct sock *tcp_sock; /* tcp sk backpointer */ |
514 | struct sock *conn; /* parent mptcp_sock */ | |
cec37a6e | 515 | const struct inet_connection_sock_af_ops *icsk_af_ops; |
648ef4b8 | 516 | void (*tcp_state_change)(struct sock *sk); |
15cc1045 | 517 | void (*tcp_error_report)(struct sock *sk); |
648ef4b8 | 518 | |
2303f994 PK |
519 | struct rcu_head rcu; |
520 | }; | |
521 | ||
522 | static inline struct mptcp_subflow_context * | |
523 | mptcp_subflow_ctx(const struct sock *sk) | |
524 | { | |
525 | struct inet_connection_sock *icsk = inet_csk(sk); | |
526 | ||
527 | /* Use RCU on icsk_ulp_data only for sock diag code */ | |
528 | return (__force struct mptcp_subflow_context *)icsk->icsk_ulp_data; | |
529 | } | |
530 | ||
531 | static inline struct sock * | |
532 | mptcp_subflow_tcp_sock(const struct mptcp_subflow_context *subflow) | |
533 | { | |
534 | return subflow->tcp_sock; | |
535 | } | |
536 | ||
b29fcfb5 PA |
537 | static inline void |
538 | mptcp_subflow_ctx_reset(struct mptcp_subflow_context *subflow) | |
539 | { | |
63ec72bd | 540 | memset(&subflow->reset, 0, sizeof(subflow->reset)); |
b29fcfb5 PA |
541 | subflow->request_mptcp = 1; |
542 | } | |
543 | ||
648ef4b8 MM |
544 | static inline u64 |
545 | mptcp_subflow_get_map_offset(const struct mptcp_subflow_context *subflow) | |
546 | { | |
547 | return tcp_sk(mptcp_subflow_tcp_sock(subflow))->copied_seq - | |
548 | subflow->ssn_offset - | |
549 | subflow->map_subflow_seq; | |
550 | } | |
551 | ||
552 | static inline u64 | |
553 | mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow) | |
554 | { | |
555 | return subflow->map_seq + mptcp_subflow_get_map_offset(subflow); | |
556 | } | |
557 | ||
b19bc294 PA |
558 | void mptcp_subflow_process_delegated(struct sock *ssk); |
559 | ||
bcd97734 | 560 | static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow, int action) |
b19bc294 PA |
561 | { |
562 | struct mptcp_delegated_action *delegated; | |
563 | bool schedule; | |
564 | ||
bcd97734 PA |
565 | /* the caller held the subflow bh socket lock */ |
566 | lockdep_assert_in_softirq(); | |
567 | ||
b19bc294 PA |
568 | /* The implied barrier pairs with mptcp_subflow_delegated_done(), and |
569 | * ensures the below list check sees list updates done prior to status | |
570 | * bit changes | |
571 | */ | |
bcd97734 | 572 | if (!test_and_set_bit(action, &subflow->delegated_status)) { |
b19bc294 PA |
573 | /* still on delegated list from previous scheduling */ |
574 | if (!list_empty(&subflow->delegated_node)) | |
575 | return; | |
576 | ||
b19bc294 PA |
577 | delegated = this_cpu_ptr(&mptcp_delegated_actions); |
578 | schedule = list_empty(&delegated->head); | |
579 | list_add_tail(&subflow->delegated_node, &delegated->head); | |
580 | sock_hold(mptcp_subflow_tcp_sock(subflow)); | |
581 | if (schedule) | |
582 | napi_schedule(&delegated->napi); | |
583 | } | |
584 | } | |
585 | ||
586 | static inline struct mptcp_subflow_context * | |
587 | mptcp_subflow_delegated_next(struct mptcp_delegated_action *delegated) | |
588 | { | |
589 | struct mptcp_subflow_context *ret; | |
590 | ||
591 | if (list_empty(&delegated->head)) | |
592 | return NULL; | |
593 | ||
594 | ret = list_first_entry(&delegated->head, struct mptcp_subflow_context, delegated_node); | |
595 | list_del_init(&ret->delegated_node); | |
596 | return ret; | |
597 | } | |
598 | ||
599 | static inline bool mptcp_subflow_has_delegated_action(const struct mptcp_subflow_context *subflow) | |
600 | { | |
bcd97734 | 601 | return !!READ_ONCE(subflow->delegated_status); |
b19bc294 PA |
602 | } |
603 | ||
bcd97734 | 604 | static inline void mptcp_subflow_delegated_done(struct mptcp_subflow_context *subflow, int action) |
b19bc294 PA |
605 | { |
606 | /* pairs with mptcp_subflow_delegate, ensures delegate_node is updated before | |
607 | * touching the status bit | |
608 | */ | |
609 | smp_wmb(); | |
bcd97734 | 610 | clear_bit(action, &subflow->delegated_status); |
b19bc294 PA |
611 | } |
612 | ||
6da14d74 PA |
613 | int mptcp_is_enabled(const struct net *net); |
614 | unsigned int mptcp_get_add_addr_timeout(const struct net *net); | |
615 | int mptcp_is_checksum_enabled(const struct net *net); | |
616 | int mptcp_allow_join_id0(const struct net *net); | |
ff5a0b42 | 617 | unsigned int mptcp_stale_loss_cnt(const struct net *net); |
6bb63ccc | 618 | int mptcp_get_pm_type(const struct net *net); |
b93df08c | 619 | void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, |
b3ea6b27 | 620 | const struct mptcp_options_received *mp_opt); |
1e1d9d6f | 621 | bool __mptcp_retransmit_pending_data(struct sock *sk); |
8b38217a | 622 | void mptcp_check_and_set_pending(struct sock *sk); |
ff5a0b42 | 623 | void __mptcp_push_pending(struct sock *sk, unsigned int flags); |
648ef4b8 | 624 | bool mptcp_subflow_data_available(struct sock *sk); |
d39dceca | 625 | void __init mptcp_subflow_init(void); |
d0876b22 | 626 | void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how); |
a141e02e FW |
627 | void mptcp_close_ssk(struct sock *sk, struct sock *ssk, |
628 | struct mptcp_subflow_context *subflow); | |
a6574302 | 629 | void __mptcp_subflow_send_ack(struct sock *ssk); |
d5824847 | 630 | void mptcp_subflow_reset(struct sock *ssk); |
2a6a870e | 631 | void mptcp_subflow_queue_clean(struct sock *sk, struct sock *ssk); |
866f26f2 | 632 | void mptcp_sock_graft(struct sock *sk, struct socket *parent); |
ddb1a072 | 633 | struct socket *__mptcp_nmpc_socket(struct mptcp_sock *msk); |
26d3e21c | 634 | bool __mptcp_close(struct sock *sk, long timeout); |
30e51b92 | 635 | void mptcp_cancel_work(struct sock *sk); |
63740448 | 636 | void __mptcp_unaccepted_force_close(struct sock *sk); |
36b122ba | 637 | void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk); |
ec3edaa7 | 638 | |
4638de5a KM |
639 | bool mptcp_addresses_equal(const struct mptcp_addr_info *a, |
640 | const struct mptcp_addr_info *b, bool use_port); | |
dc886bce | 641 | void mptcp_local_address(const struct sock_common *skc, struct mptcp_addr_info *addr); |
4638de5a | 642 | |
ec3edaa7 | 643 | /* called with sk socket lock held */ |
ef0da3b8 | 644 | int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, |
ee285257 | 645 | const struct mptcp_addr_info *remote); |
6bc1fe7d PA |
646 | int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, |
647 | struct socket **new_sock); | |
1729cf18 GT |
648 | void mptcp_info2sockaddr(const struct mptcp_addr_info *info, |
649 | struct sockaddr_storage *addr, | |
650 | unsigned short family); | |
2303f994 | 651 | |
c886d702 | 652 | static inline bool __tcp_can_send(const struct sock *ssk) |
43f1140b | 653 | { |
c886d702 PA |
654 | /* only send if our side has not closed yet */ |
655 | return ((1 << inet_sk_state_load(ssk)) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)); | |
656 | } | |
43f1140b | 657 | |
c886d702 PA |
658 | static inline bool __mptcp_subflow_active(struct mptcp_subflow_context *subflow) |
659 | { | |
43f1140b GT |
660 | /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */ |
661 | if (subflow->request_join && !subflow->fully_established) | |
662 | return false; | |
663 | ||
c886d702 | 664 | return __tcp_can_send(mptcp_subflow_tcp_sock(subflow)); |
43f1140b GT |
665 | } |
666 | ||
ff5a0b42 PA |
667 | void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow); |
668 | ||
669 | bool mptcp_subflow_active(struct mptcp_subflow_context *subflow); | |
670 | ||
b6985b9b PA |
671 | void mptcp_subflow_drop_ctx(struct sock *ssk); |
672 | ||
648ef4b8 MM |
673 | static inline void mptcp_subflow_tcp_fallback(struct sock *sk, |
674 | struct mptcp_subflow_context *ctx) | |
675 | { | |
952382c6 | 676 | sk->sk_data_ready = sock_def_readable; |
648ef4b8 | 677 | sk->sk_state_change = ctx->tcp_state_change; |
952382c6 | 678 | sk->sk_write_space = sk_stream_write_space; |
15cc1045 | 679 | sk->sk_error_report = ctx->tcp_error_report; |
648ef4b8 MM |
680 | |
681 | inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops; | |
682 | } | |
683 | ||
d39dceca | 684 | void __init mptcp_proto_init(void); |
784325e9 | 685 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) |
d39dceca | 686 | int __init mptcp_proto_v6_init(void); |
784325e9 | 687 | #endif |
648ef4b8 | 688 | |
7e8b88ec PA |
689 | struct sock *mptcp_sk_clone_init(const struct sock *sk, |
690 | const struct mptcp_options_received *mp_opt, | |
691 | struct sock *ssk, | |
692 | struct request_sock *req); | |
0799e21b | 693 | void mptcp_get_options(const struct sk_buff *skb, |
cfde141e | 694 | struct mptcp_options_received *mp_opt); |
cec37a6e PK |
695 | |
696 | void mptcp_finish_connect(struct sock *sk); | |
490274b4 | 697 | void __mptcp_set_connected(struct sock *sk); |
76a13b31 | 698 | void mptcp_reset_timeout(struct mptcp_sock *msk, unsigned long fail_tout); |
b93df08c PA |
699 | static inline bool mptcp_is_fully_established(struct sock *sk) |
700 | { | |
701 | return inet_sk_state_load(sk) == TCP_ESTABLISHED && | |
702 | READ_ONCE(mptcp_sk(sk)->fully_established); | |
703 | } | |
a6b118fe | 704 | void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); |
2e52213c | 705 | void mptcp_data_ready(struct sock *sk, struct sock *ssk); |
f296234c | 706 | bool mptcp_finish_join(struct sock *sk); |
ba8f48f7 | 707 | bool mptcp_schedule_work(struct sock *sk); |
0abdde82 PA |
708 | int mptcp_setsockopt(struct sock *sk, int level, int optname, |
709 | sockptr_t optval, unsigned int optlen); | |
710 | int mptcp_getsockopt(struct sock *sk, int level, int optname, | |
711 | char __user *optval, int __user *option); | |
712 | ||
1502328f PA |
713 | u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq); |
714 | static inline u64 mptcp_expand_seq(u64 old_seq, u64 cur_seq, bool use_64bit) | |
715 | { | |
716 | if (use_64bit) | |
717 | return cur_seq; | |
718 | ||
719 | return __mptcp_expand_seq(old_seq, cur_seq); | |
720 | } | |
219d0499 | 721 | void __mptcp_check_push(struct sock *sk, struct sock *ssk); |
7439d687 | 722 | void __mptcp_data_acked(struct sock *sk); |
15cc1045 | 723 | void __mptcp_error_report(struct sock *sk); |
59832e24 | 724 | void mptcp_subflow_eof(struct sock *sk); |
1a49b2c2 | 725 | bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit); |
e16163b6 PA |
726 | static inline bool mptcp_data_fin_enabled(const struct mptcp_sock *msk) |
727 | { | |
728 | return READ_ONCE(msk->snd_data_fin_enable) && | |
729 | READ_ONCE(msk->write_seq) == READ_ONCE(msk->snd_nxt); | |
730 | } | |
731 | ||
5cf92bba PA |
732 | static inline bool mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk) |
733 | { | |
734 | if ((sk->sk_userlocks & SOCK_SNDBUF_LOCK) || ssk->sk_sndbuf <= READ_ONCE(sk->sk_sndbuf)) | |
735 | return false; | |
736 | ||
737 | WRITE_ONCE(sk->sk_sndbuf, ssk->sk_sndbuf); | |
738 | return true; | |
739 | } | |
740 | ||
741 | static inline void mptcp_write_space(struct sock *sk) | |
742 | { | |
743 | if (sk_stream_is_writeable(sk)) { | |
744 | /* pairs with memory barrier in mptcp_poll */ | |
745 | smp_mb(); | |
746 | if (test_and_clear_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags)) | |
747 | sk_stream_write_space(sk); | |
748 | } | |
749 | } | |
750 | ||
c0bf3c6a | 751 | void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags); |
cec37a6e | 752 | |
c68a0cd1 JW |
753 | #define MPTCP_TOKEN_MAX_RETRIES 4 |
754 | ||
2c5ebd00 PA |
755 | void __init mptcp_token_init(void); |
756 | static inline void mptcp_token_init_request(struct request_sock *req) | |
757 | { | |
758 | mptcp_subflow_rsk(req)->token_node.pprev = NULL; | |
759 | } | |
760 | ||
79c0949e | 761 | int mptcp_token_new_request(struct request_sock *req); |
2c5ebd00 | 762 | void mptcp_token_destroy_request(struct request_sock *req); |
294de909 | 763 | int mptcp_token_new_connect(struct sock *ssk); |
2c5ebd00 PA |
764 | void mptcp_token_accept(struct mptcp_subflow_request_sock *r, |
765 | struct mptcp_sock *msk); | |
c83a47e5 | 766 | bool mptcp_token_exists(u32 token); |
ea1300b9 | 767 | struct mptcp_sock *mptcp_token_get_sock(struct net *net, u32 token); |
96d890da PA |
768 | struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot, |
769 | long *s_num); | |
2c5ebd00 | 770 | void mptcp_token_destroy(struct mptcp_sock *msk); |
79c0949e PK |
771 | |
772 | void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn); | |
79c0949e | 773 | |
3df523ab | 774 | void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac); |
ba2c89e0 | 775 | __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum); |
79c0949e | 776 | |
d39dceca | 777 | void __init mptcp_pm_init(void); |
1b1c7a0e | 778 | void mptcp_pm_data_init(struct mptcp_sock *msk); |
b29fcfb5 | 779 | void mptcp_pm_data_reset(struct mptcp_sock *msk); |
702c2f64 FW |
780 | int mptcp_pm_parse_addr(struct nlattr *attr, struct genl_info *info, |
781 | struct mptcp_addr_info *addr); | |
9ab4807c KM |
782 | int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, |
783 | bool require_family, | |
784 | struct mptcp_pm_addr_entry *entry); | |
fb00ee4f MB |
785 | bool mptcp_pm_addr_families_match(const struct sock *sk, |
786 | const struct mptcp_addr_info *loc, | |
787 | const struct mptcp_addr_info *rem); | |
71b7dec2 | 788 | void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
ff5a0b42 | 789 | void mptcp_pm_nl_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
6c714f1b | 790 | void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side); |
7a486c44 | 791 | void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk); |
1b1c7a0e PK |
792 | bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk); |
793 | void mptcp_pm_connection_closed(struct mptcp_sock *msk); | |
62535200 | 794 | void mptcp_pm_subflow_established(struct mptcp_sock *msk); |
a88c9e49 PA |
795 | bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk); |
796 | void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, const struct sock *ssk, | |
797 | const struct mptcp_subflow_context *subflow); | |
d1ace2d9 | 798 | void mptcp_pm_add_addr_received(const struct sock *ssk, |
1b1c7a0e | 799 | const struct mptcp_addr_info *addr); |
557963c3 | 800 | void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, |
90d93088 | 801 | const struct mptcp_addr_info *addr); |
84dfe367 | 802 | void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk); |
b46a0238 | 803 | void mptcp_pm_nl_addr_send_ack(struct mptcp_sock *msk); |
5c4a824d GT |
804 | void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, |
805 | const struct mptcp_rm_list *rm_list); | |
40453a5c | 806 | void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup); |
5580d41b | 807 | void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq); |
892f396c KM |
808 | int mptcp_pm_nl_mp_prio_send_ack(struct mptcp_sock *msk, |
809 | struct mptcp_addr_info *addr, | |
810 | struct mptcp_addr_info *rem, | |
811 | u8 bkup); | |
9ab4807c KM |
812 | bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, |
813 | const struct mptcp_pm_addr_entry *entry); | |
b6c08380 | 814 | void mptcp_pm_free_anno_list(struct mptcp_sock *msk); |
5bc56388 | 815 | bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk); |
00cfd77b GT |
816 | struct mptcp_pm_add_entry * |
817 | mptcp_pm_del_add_timer(struct mptcp_sock *msk, | |
90d93088 | 818 | const struct mptcp_addr_info *addr, bool check_id); |
d88c476f | 819 | struct mptcp_pm_add_entry * |
90d93088 PA |
820 | mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, |
821 | const struct mptcp_addr_info *addr); | |
8b201370 KM |
822 | int mptcp_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
823 | unsigned int id, | |
ee285257 | 824 | u8 *flags, int *ifindex); |
f40be0db GT |
825 | int mptcp_pm_nl_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, unsigned int id, |
826 | u8 *flags, int *ifindex); | |
8b201370 KM |
827 | int mptcp_userspace_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
828 | unsigned int id, | |
829 | u8 *flags, int *ifindex); | |
6ba7ce89 GT |
830 | int mptcp_pm_set_flags(struct net *net, struct nlattr *token, |
831 | struct mptcp_pm_addr_entry *loc, | |
832 | struct mptcp_pm_addr_entry *rem, u8 bkup); | |
833 | int mptcp_pm_nl_set_flags(struct net *net, struct mptcp_pm_addr_entry *addr, u8 bkup); | |
892f396c KM |
834 | int mptcp_userspace_pm_set_flags(struct net *net, struct nlattr *token, |
835 | struct mptcp_pm_addr_entry *loc, | |
836 | struct mptcp_pm_addr_entry *rem, u8 bkup); | |
1b1c7a0e | 837 | int mptcp_pm_announce_addr(struct mptcp_sock *msk, |
6a6c05a8 | 838 | const struct mptcp_addr_info *addr, |
f7efc777 | 839 | bool echo); |
cbde2787 | 840 | int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
ddd14bb8 | 841 | int mptcp_pm_remove_subflow(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
8b1c94da | 842 | void mptcp_pm_remove_addrs(struct mptcp_sock *msk, struct list_head *rm_list); |
d9a4594e KM |
843 | void mptcp_pm_remove_addrs_and_subflows(struct mptcp_sock *msk, |
844 | struct list_head *rm_list); | |
845 | ||
4638de5a | 846 | void mptcp_free_local_addr_list(struct mptcp_sock *msk); |
9ab4807c | 847 | int mptcp_nl_cmd_announce(struct sk_buff *skb, struct genl_info *info); |
d9a4594e | 848 | int mptcp_nl_cmd_remove(struct sk_buff *skb, struct genl_info *info); |
702c2f64 FW |
849 | int mptcp_nl_cmd_sf_create(struct sk_buff *skb, struct genl_info *info); |
850 | int mptcp_nl_cmd_sf_destroy(struct sk_buff *skb, struct genl_info *info); | |
1b1c7a0e | 851 | |
b911c97c FW |
852 | void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, |
853 | const struct sock *ssk, gfp_t gfp); | |
d1ace2d9 | 854 | void mptcp_event_addr_announced(const struct sock *ssk, const struct mptcp_addr_info *info); |
b911c97c | 855 | void mptcp_event_addr_removed(const struct mptcp_sock *msk, u8 id); |
f8c9dfbd GT |
856 | void mptcp_event_pm_listener(const struct sock *ssk, |
857 | enum mptcp_event_type event); | |
4d25247d | 858 | bool mptcp_userspace_pm_active(const struct mptcp_sock *msk); |
b911c97c | 859 | |
dfc8d060 DS |
860 | void mptcp_fastopen_gen_msk_ackseq(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow, |
861 | const struct mptcp_options_received *mp_opt); | |
36b122ba DS |
862 | void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subflow, |
863 | struct request_sock *req); | |
dfc8d060 | 864 | |
f643b803 | 865 | static inline bool mptcp_pm_should_add_signal(struct mptcp_sock *msk) |
18fc1a92 YL |
866 | { |
867 | return READ_ONCE(msk->pm.addr_signal) & | |
868 | (BIT(MPTCP_ADD_ADDR_SIGNAL) | BIT(MPTCP_ADD_ADDR_ECHO)); | |
869 | } | |
870 | ||
871 | static inline bool mptcp_pm_should_add_signal_addr(struct mptcp_sock *msk) | |
1b1c7a0e | 872 | { |
13ad9f01 | 873 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_SIGNAL); |
d91d322a GT |
874 | } |
875 | ||
876 | static inline bool mptcp_pm_should_add_signal_echo(struct mptcp_sock *msk) | |
877 | { | |
13ad9f01 | 878 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_ECHO); |
1b1c7a0e PK |
879 | } |
880 | ||
5cb104ae GT |
881 | static inline bool mptcp_pm_should_rm_signal(struct mptcp_sock *msk) |
882 | { | |
13ad9f01 | 883 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_RM_ADDR_SIGNAL); |
5cb104ae GT |
884 | } |
885 | ||
14b06811 MM |
886 | static inline bool mptcp_pm_is_userspace(const struct mptcp_sock *msk) |
887 | { | |
888 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_USERSPACE; | |
889 | } | |
890 | ||
4d25247d KM |
891 | static inline bool mptcp_pm_is_kernel(const struct mptcp_sock *msk) |
892 | { | |
893 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_KERNEL; | |
894 | } | |
895 | ||
4a2777a8 | 896 | static inline unsigned int mptcp_add_addr_len(int family, bool echo, bool port) |
1b1c7a0e | 897 | { |
2ec72fae GT |
898 | u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE; |
899 | ||
900 | if (family == AF_INET6) | |
901 | len = TCPOLEN_MPTCP_ADD_ADDR6_BASE; | |
902 | if (!echo) | |
903 | len += MPTCPOPT_THMAC_LEN; | |
27ab92d9 | 904 | /* account for 2 trailing 'nop' options */ |
4a2777a8 | 905 | if (port) |
27ab92d9 | 906 | len += TCPOLEN_MPTCP_PORT_LEN + TCPOLEN_MPTCP_PORT_ALIGN; |
2ec72fae GT |
907 | |
908 | return len; | |
1b1c7a0e PK |
909 | } |
910 | ||
6445e17a GT |
911 | static inline int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list) |
912 | { | |
913 | if (rm_list->nr == 0 || rm_list->nr > MPTCP_RM_IDS_MAX) | |
914 | return -EINVAL; | |
915 | ||
916 | return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1; | |
917 | } | |
918 | ||
90d93088 | 919 | bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb, |
1f5e9e2f | 920 | unsigned int opt_size, unsigned int remaining, |
f462a446 | 921 | struct mptcp_addr_info *addr, bool *echo, |
af7939f3 | 922 | bool *drop_other_suboptions); |
5cb104ae | 923 | bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, |
6445e17a | 924 | struct mptcp_rm_list *rm_list); |
1b1c7a0e | 925 | int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc); |
9bbec87e | 926 | int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
8b201370 | 927 | int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
1b1c7a0e | 928 | |
d39dceca | 929 | void __init mptcp_pm_nl_init(void); |
e9801430 | 930 | void mptcp_pm_nl_work(struct mptcp_sock *msk); |
ddd14bb8 GT |
931 | void mptcp_pm_nl_rm_subflow_received(struct mptcp_sock *msk, |
932 | const struct mptcp_rm_list *rm_list); | |
90d93088 PA |
933 | unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk); |
934 | unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk); | |
935 | unsigned int mptcp_pm_get_subflows_max(const struct mptcp_sock *msk); | |
936 | unsigned int mptcp_pm_get_local_addr_max(const struct mptcp_sock *msk); | |
78962489 | 937 | |
95d68651 PA |
938 | /* called under PM lock */ |
939 | static inline void __mptcp_pm_close_subflow(struct mptcp_sock *msk) | |
940 | { | |
941 | if (--msk->pm.subflows < mptcp_pm_get_subflows_max(msk)) | |
942 | WRITE_ONCE(msk->pm.accept_subflow, true); | |
943 | } | |
944 | ||
945 | static inline void mptcp_pm_close_subflow(struct mptcp_sock *msk) | |
946 | { | |
947 | spin_lock_bh(&msk->pm.lock); | |
948 | __mptcp_pm_close_subflow(msk); | |
949 | spin_unlock_bh(&msk->pm.lock); | |
950 | } | |
951 | ||
78962489 | 952 | void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk); |
3e501490 | 953 | void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk); |
01cacb00 | 954 | |
dc87efdb | 955 | static inline struct mptcp_ext *mptcp_get_ext(const struct sk_buff *skb) |
6d0060f6 MM |
956 | { |
957 | return (struct mptcp_ext *)skb_ext_find(skb, SKB_EXT_MPTCP); | |
958 | } | |
959 | ||
5147dfb5 DC |
960 | void mptcp_diag_subflow_init(struct tcp_ulp_ops *ops); |
961 | ||
d5f49190 | 962 | static inline bool __mptcp_check_fallback(const struct mptcp_sock *msk) |
e1ff9e82 DC |
963 | { |
964 | return test_bit(MPTCP_FALLBACK_DONE, &msk->flags); | |
965 | } | |
966 | ||
d5f49190 | 967 | static inline bool mptcp_check_fallback(const struct sock *sk) |
e1ff9e82 DC |
968 | { |
969 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
970 | struct mptcp_sock *msk = mptcp_sk(subflow->conn); | |
971 | ||
972 | return __mptcp_check_fallback(msk); | |
973 | } | |
974 | ||
975 | static inline void __mptcp_do_fallback(struct mptcp_sock *msk) | |
976 | { | |
977 | if (test_bit(MPTCP_FALLBACK_DONE, &msk->flags)) { | |
978 | pr_debug("TCP fallback already done (msk=%p)", msk); | |
979 | return; | |
980 | } | |
981 | set_bit(MPTCP_FALLBACK_DONE, &msk->flags); | |
982 | } | |
983 | ||
d51991e2 | 984 | static inline void mptcp_do_fallback(struct sock *ssk) |
e1ff9e82 | 985 | { |
d51991e2 PA |
986 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); |
987 | struct sock *sk = subflow->conn; | |
988 | struct mptcp_sock *msk; | |
e1ff9e82 | 989 | |
d51991e2 | 990 | msk = mptcp_sk(sk); |
e1ff9e82 | 991 | __mptcp_do_fallback(msk); |
d51991e2 PA |
992 | if (READ_ONCE(msk->snd_data_fin_enable) && !(ssk->sk_shutdown & SEND_SHUTDOWN)) { |
993 | gfp_t saved_allocation = ssk->sk_allocation; | |
994 | ||
995 | /* we are in a atomic (BH) scope, override ssk default for data | |
996 | * fin allocation | |
997 | */ | |
998 | ssk->sk_allocation = GFP_ATOMIC; | |
999 | ssk->sk_shutdown |= SEND_SHUTDOWN; | |
1000 | tcp_shutdown(ssk, SEND_SHUTDOWN); | |
1001 | ssk->sk_allocation = saved_allocation; | |
1002 | } | |
e1ff9e82 DC |
1003 | } |
1004 | ||
1005 | #define pr_fallback(a) pr_debug("%s:fallback to TCP (msk=%p)", __func__, a) | |
1006 | ||
1e39e5a3 GT |
1007 | static inline bool mptcp_check_infinite_map(struct sk_buff *skb) |
1008 | { | |
1009 | struct mptcp_ext *mpext; | |
1010 | ||
1011 | mpext = skb ? mptcp_get_ext(skb) : NULL; | |
1012 | if (mpext && mpext->infinite_map) | |
1013 | return true; | |
1014 | ||
1015 | return false; | |
1016 | } | |
1017 | ||
70c708e8 KM |
1018 | static inline bool is_active_ssk(struct mptcp_subflow_context *subflow) |
1019 | { | |
1020 | return (subflow->request_mptcp || subflow->request_join); | |
1021 | } | |
1022 | ||
8fd73804 DC |
1023 | static inline bool subflow_simultaneous_connect(struct sock *sk) |
1024 | { | |
1025 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
8fd73804 DC |
1026 | |
1027 | return sk->sk_state == TCP_ESTABLISHED && | |
70c708e8 | 1028 | is_active_ssk(subflow) && |
8fd73804 DC |
1029 | !subflow->conn_finished; |
1030 | } | |
1031 | ||
9466a1cc FW |
1032 | #ifdef CONFIG_SYN_COOKIES |
1033 | void subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, | |
1034 | struct sk_buff *skb); | |
1035 | bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, | |
1036 | struct sk_buff *skb); | |
1037 | void __init mptcp_join_cookie_init(void); | |
1038 | #else | |
1039 | static inline void | |
1040 | subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, | |
1041 | struct sk_buff *skb) {} | |
1042 | static inline bool | |
1043 | mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, | |
1044 | struct sk_buff *skb) | |
1045 | { | |
1046 | return false; | |
1047 | } | |
1048 | ||
1049 | static inline void mptcp_join_cookie_init(void) {} | |
1050 | #endif | |
1051 | ||
f870fa0b | 1052 | #endif /* __MPTCP_PROTOCOL_H */ |