projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'cxl-fixes-6.10-rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl
[linux-2.6-block.git] / net / ipv4 / tcp.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *
02c30a84 9 * Authors: Ross Biro
1da177e4
LT		 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Mark Evans, <evansmp@uhura.aston.ac.uk>
12 * Corey Minyard <wf-rch!minyard@relay.EU.net>
13 * Florian La Roche, <flla@stud.uni-sb.de>
14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15 * Linus Torvalds, <torvalds@cs.helsinki.fi>
16 * Alan Cox, <gw4pts@gw4pts.ampr.org>
17 * Matthew Dillon, <dillon@apollo.west.oic.com>
18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19 * Jorge Cwik, <jorge@laser.satlink.net>
20 *
21 * Fixes:
22 * Alan Cox : Numerous verify_area() calls
23 * Alan Cox : Set the ACK bit on a reset
24 * Alan Cox : Stopped it crashing if it closed while
25 * sk->inuse=1 and was trying to connect
26 * (tcp_err()).
27 * Alan Cox : All icmp error handling was broken
28 * pointers passed where wrong and the
29 * socket was looked up backwards. Nobody
30 * tested any icmp error code obviously.
31 * Alan Cox : tcp_err() now handled properly. It
32 * wakes people on errors. poll
33 * behaves and the icmp error race
34 * has gone by moving it into sock.c
35 * Alan Cox : tcp_send_reset() fixed to work for
36 * everything not just packets for
37 * unknown sockets.
38 * Alan Cox : tcp option processing.
39 * Alan Cox : Reset tweaked (still not 100%) [Had
40 * syn rule wrong]
41 * Herp Rosmanith : More reset fixes
42 * Alan Cox : No longer acks invalid rst frames.
43 * Acking any kind of RST is right out.
44 * Alan Cox : Sets an ignore me flag on an rst
45 * receive otherwise odd bits of prattle
46 * escape still
47 * Alan Cox : Fixed another acking RST frame bug.
48 * Should stop LAN workplace lockups.
49 * Alan Cox : Some tidyups using the new skb list
50 * facilities
51 * Alan Cox : sk->keepopen now seems to work
52 * Alan Cox : Pulls options out correctly on accepts
53 * Alan Cox : Fixed assorted sk->rqueue->next errors
54 * Alan Cox : PSH doesn't end a TCP read. Switched a
55 * bit to skb ops.
56 * Alan Cox : Tidied tcp_data to avoid a potential
57 * nasty.
58 * Alan Cox : Added some better commenting, as the
59 * tcp is hard to follow
60 * Alan Cox : Removed incorrect check for 20 * psh
61 * Michael O'Reilly : ack < copied bug fix.
62 * Johannes Stille : Misc tcp fixes (not all in yet).
63 * Alan Cox : FIN with no memory -> CRASH
64 * Alan Cox : Added socket option proto entries.
65 * Also added awareness of them to accept.
66 * Alan Cox : Added TCP options (SOL_TCP)
67 * Alan Cox : Switched wakeup calls to callbacks,
68 * so the kernel can layer network
69 * sockets.
70 * Alan Cox : Use ip_tos/ip_ttl settings.
71 * Alan Cox : Handle FIN (more) properly (we hope).
72 * Alan Cox : RST frames sent on unsynchronised
73 * state ack error.
74 * Alan Cox : Put in missing check for SYN bit.
75 * Alan Cox : Added tcp_select_window() aka NET2E
76 * window non shrink trick.
77 * Alan Cox : Added a couple of small NET2E timer
78 * fixes
79 * Charles Hedrick : TCP fixes
80 * Toomas Tamm : TCP window fixes
81 * Alan Cox : Small URG fix to rlogin ^C ack fight
82 * Charles Hedrick : Rewrote most of it to actually work
83 * Linus : Rewrote tcp_read() and URG handling
84 * completely
85 * Gerhard Koerting: Fixed some missing timer handling
86 * Matthew Dillon : Reworked TCP machine states as per RFC
87 * Gerhard Koerting: PC/TCP workarounds
88 * Adam Caldwell : Assorted timer/timing errors
89 * Matthew Dillon : Fixed another RST bug
90 * Alan Cox : Move to kernel side addressing changes.
91 * Alan Cox : Beginning work on TCP fastpathing
92 * (not yet usable)
93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine.
94 * Alan Cox : TCP fast path debugging
95 * Alan Cox : Window clamping
96 * Michael Riepe : Bug in tcp_check()
97 * Matt Dillon : More TCP improvements and RST bug fixes
98 * Matt Dillon : Yet more small nasties remove from the
99 * TCP code (Be very nice to this man if
100 * tcp finally works 100%) 8)
101 * Alan Cox : BSD accept semantics.
102 * Alan Cox : Reset on closedown bug.
103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
104 * Michael Pall : Handle poll() after URG properly in
105 * all cases.
106 * Michael Pall : Undo the last fix in tcp_read_urg()
107 * (multi URG PUSH broke rlogin).
108 * Michael Pall : Fix the multi URG PUSH problem in
109 * tcp_readable(), poll() after URG
110 * works now.
111 * Michael Pall : recv(...,MSG_OOB) never blocks in the
112 * BSD api.
113 * Alan Cox : Changed the semantics of sk->socket to
114 * fix a race and a signal problem with
115 * accept() and async I/O.
116 * Alan Cox : Relaxed the rules on tcp_sendto().
117 * Yury Shevchuk : Really fixed accept() blocking problem.
118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
119 * clients/servers which listen in on
120 * fixed ports.
121 * Alan Cox : Cleaned the above up and shrank it to
122 * a sensible code size.
123 * Alan Cox : Self connect lockup fix.
124 * Alan Cox : No connect to multicast.
125 * Ross Biro : Close unaccepted children on master
126 * socket close.
127 * Alan Cox : Reset tracing code.
128 * Alan Cox : Spurious resets on shutdown.
129 * Alan Cox : Giant 15 minute/60 second timer error
130 * Alan Cox : Small whoops in polling before an
131 * accept.
132 * Alan Cox : Kept the state trace facility since
133 * it's handy for debugging.
134 * Alan Cox : More reset handler fixes.
135 * Alan Cox : Started rewriting the code based on
136 * the RFC's for other useful protocol
137 * references see: Comer, KA9Q NOS, and
138 * for a reference on the difference
139 * between specifications and how BSD
140 * works see the 4.4lite source.
141 * A.N.Kuznetsov : Don't time wait on completion of tidy
142 * close.
143 * Linus Torvalds : Fin/Shutdown & copied_seq changes.
144 * Linus Torvalds : Fixed BSD port reuse to work first syn
145 * Alan Cox : Reimplemented timers as per the RFC
146 * and using multiple timers for sanity.
147 * Alan Cox : Small bug fixes, and a lot of new
148 * comments.
149 * Alan Cox : Fixed dual reader crash by locking
150 * the buffers (much like datagram.c)
151 * Alan Cox : Fixed stuck sockets in probe. A probe
152 * now gets fed up of retrying without
153 * (even a no space) answer.
154 * Alan Cox : Extracted closing code better
155 * Alan Cox : Fixed the closing state machine to
156 * resemble the RFC.
157 * Alan Cox : More 'per spec' fixes.
158 * Jorge Cwik : Even faster checksumming.
159 * Alan Cox : tcp_data() doesn't ack illegal PSH
160 * only frames. At least one pc tcp stack
161 * generates them.
162 * Alan Cox : Cache last socket.
163 * Alan Cox : Per route irtt.
164 * Matt Day : poll()->select() match BSD precisely on error
165 * Alan Cox : New buffers
166 * Marc Tamsky : Various sk->prot->retransmits and
167 * sk->retransmits misupdating fixed.
168 * Fixed tcp_write_timeout: stuck close,
169 * and TCP syn retries gets used now.
170 * Mark Yarvis : In tcp_read_wakeup(), don't send an
171 * ack if state is TCP_CLOSED.
172 * Alan Cox : Look up device on a retransmit - routes may
173 * change. Doesn't yet cope with MSS shrink right
174 * but it's a start!
175 * Marc Tamsky : Closing in closing fixes.
176 * Mike Shaver : RFC1122 verifications.
177 * Alan Cox : rcv_saddr errors.
178 * Alan Cox : Block double connect().
179 * Alan Cox : Small hooks for enSKIP.
180 * Alexey Kuznetsov: Path MTU discovery.
181 * Alan Cox : Support soft errors.
182 * Alan Cox : Fix MTU discovery pathological case
183 * when the remote claims no mtu!
184 * Marc Tamsky : TCP_CLOSE fix.
185 * Colin (G3TNE) : Send a reset on syn ack replies in
186 * window but wrong (fixes NT lpd problems)
187 * Pedro Roque : Better TCP window handling, delayed ack.
188 * Joerg Reuter : No modification of locked buffers in
189 * tcp_do_retransmit()
190 * Eric Schenk : Changed receiver side silly window
191 * avoidance algorithm to BSD style
192 * algorithm. This doubles throughput
193 * against machines running Solaris,
194 * and seems to result in general
195 * improvement.
196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
197 * Willy Konynenberg : Transparent proxying support.
198 * Mike McLagan : Routing by source
199 * Keith Owens : Do proper merging with partial SKB's in
200 * tcp_do_sendmsg to avoid burstiness.
201 * Eric Schenk : Fix fast close down bug with
202 * shutdown() followed by close().
203 * Andi Kleen : Make poll agree with SIGIO
204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
205 * lingertime == 0 (RFC 793 ABORT Call)
206 * Hirokazu Takahashi : Use copy_from_user() instead of
207 * csum_and_copy_from_user() if possible.
208 *
1da177e4
LT		 209 * Description of States:
210 *
211 * TCP_SYN_SENT sent a connection request, waiting for ack
212 *
213 * TCP_SYN_RECV received a connection request, sent ack,
214 * waiting for final ack in three-way handshake.
215 *
216 * TCP_ESTABLISHED connection established
217 *
218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete
219 * transmission of remaining buffered data
220 *
221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote
222 * to shutdown
223 *
224 * TCP_CLOSING both sides have shutdown but we still have
225 * data we have to finish sending
226 *
227 * TCP_TIME_WAIT timeout to catch resent junk before entering
228 * closed, can only be entered from FIN_WAIT2
229 * or CLOSING. Required because the other end
230 * may not have gotten our last ACK causing it
231 * to retransmit the data packet (which we ignore)
232 *
233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for
234 * us to finish writing our data and to shutdown
235 * (we have to close() to move on to LAST_ACK)
236 *
237 * TCP_LAST_ACK out side has shutdown after remote has
238 * shutdown. There may still be data in our
239 * buffer that we have to finish sending
240 *
241 * TCP_CLOSE socket is finished
242 */
243
afd46503
JP		 244#define pr_fmt(fmt) "TCP: " fmt
245
cf80e0e4 246#include <crypto/hash.h>
172589cc 247#include <linux/kernel.h>
1da177e4
LT		 248#include <linux/module.h>
249#include <linux/types.h>
250#include <linux/fcntl.h>
251#include <linux/poll.h>
6e9250f5 252#include <linux/inet_diag.h>
1da177e4 253#include <linux/init.h>
1da177e4 254#include <linux/fs.h>
9c55e01c 255#include <linux/skbuff.h>
81b23b4a 256#include <linux/scatterlist.h>
9c55e01c
JA		 257#include <linux/splice.h>
258#include <linux/net.h>
259#include <linux/socket.h>
1da177e4 260#include <linux/random.h>
57c8a661 261#include <linux/memblock.h>
57413ebc 262#include <linux/highmem.h>
b8059ead 263#include <linux/cache.h>
f4c50d99 264#include <linux/err.h>
da5c78c8 265#include <linux/time.h>
5a0e3ad6 266#include <linux/slab.h>
98aaa913 267#include <linux/errqueue.h>
60e2a778 268#include <linux/static_key.h>
97a19caf 269#include <linux/btf.h>
1da177e4
LT		 270
271#include <net/icmp.h>
cf60af03 272#include <net/inet_common.h>
1da177e4 273#include <net/tcp.h>
f870fa0b 274#include <net/mptcp.h>
f3d93817 275#include <net/proto_memory.h>
1da177e4
LT		 276#include <net/xfrm.h>
277#include <net/ip.h>
9c55e01c 278#include <net/sock.h>
5691276b 279#include <net/rstreason.h>
1da177e4 280
7c0f6ba6 281#include <linux/uaccess.h>
1da177e4 282#include <asm/ioctls.h>
076bb0c8 283#include <net/busy_poll.h>
a86a0661 284#include <net/hotdata.h>
490a79fa 285#include <net/rps.h>
1da177e4 286
925bba24
AR		 287/* Track pending CMSGs. */
288enum {
289 TCP_CMSG_INQ = 1,
290 TCP_CMSG_TS = 2
291};
292
19757ceb
ED		 293DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
294EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
0a5578cf 295
41eecbd7
ED		 296DEFINE_PER_CPU(u32, tcp_tw_isn);
297EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
298
a4fe34bf 299long sysctl_tcp_mem[3] __read_mostly;
a4fe34bf 300EXPORT_SYMBOL(sysctl_tcp_mem);
1da177e4 301
91b6d325 302atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */
1da177e4 303EXPORT_SYMBOL(tcp_memory_allocated);
0defbb0a
ED		 304DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
305EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
1748376b 306
60e2a778
UB		 307#if IS_ENABLED(CONFIG_SMC)
308DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
309EXPORT_SYMBOL(tcp_have_smc);
310#endif
311
1748376b
ED		 312/*
313 * Current number of TCP sockets.
314 */
91b6d325 315struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp;
1da177e4
LT		 316EXPORT_SYMBOL(tcp_sockets_allocated);
317
9c55e01c
JA		 318/*
319 * TCP splice context
320 */
321struct tcp_splice_state {
322 struct pipe_inode_info *pipe;
323 size_t len;
324 unsigned int flags;
325};
326
1da177e4
LT		 327/*
328 * Pressure flag: try to collapse.
329 * Technical note: it is used by multiple contexts non atomically.
3ab224be 330 * All the __sk_mem_schedule() is of this nature: accounting
1da177e4
LT		 331 * is strict, actions are advisory and have some latency.
332 */
06044751
ED		 333unsigned long tcp_memory_pressure __read_mostly;
334EXPORT_SYMBOL_GPL(tcp_memory_pressure);
1da177e4 335
5c52ba17 336void tcp_enter_memory_pressure(struct sock *sk)
1da177e4 337{
06044751
ED		 338 unsigned long val;
339
1f142c17 340 if (READ_ONCE(tcp_memory_pressure))
06044751
ED		 341 return;
342 val = jiffies;
343
344 if (!val)
345 val--;
346 if (!cmpxchg(&tcp_memory_pressure, 0, val))
4e673444 347 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
1da177e4 348}
06044751
ED		 349EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure);
350
351void tcp_leave_memory_pressure(struct sock *sk)
352{
353 unsigned long val;
354
1f142c17 355 if (!READ_ONCE(tcp_memory_pressure))
06044751
ED		 356 return;
357 val = xchg(&tcp_memory_pressure, 0);
358 if (val)
359 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
360 jiffies_to_msecs(jiffies - val));
361}
362EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure);
1da177e4 363
b103cf34
JA		 364/* Convert seconds to retransmits based on initial and max timeout */
365static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
366{
367 u8 res = 0;
368
369 if (seconds > 0) {
370 int period = timeout;
371
372 res = 1;
373 while (seconds > period && res < 255) {
374 res++;
375 timeout <<= 1;
376 if (timeout > rto_max)
377 timeout = rto_max;
378 period += timeout;
379 }
380 }
381 return res;
382}
383
384/* Convert retransmits to seconds based on initial and max timeout */
385static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
386{
387 int period = 0;
388
389 if (retrans > 0) {
390 period = timeout;
391 while (--retrans) {
392 timeout <<= 1;
393 if (timeout > rto_max)
394 timeout = rto_max;
395 period += timeout;
396 }
397 }
398 return period;
399}
400
0263598c
WW		 401static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
402{
403 u32 rate = READ_ONCE(tp->rate_delivered);
404 u32 intv = READ_ONCE(tp->rate_interval_us);
405 u64 rate64 = 0;
406
407 if (rate && intv) {
408 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
409 do_div(rate64, intv);
410 }
411 return rate64;
412}
413
900f65d3
NC		 414/* Address-family independent initialization for a tcp_sock.
415 *
416 * NOTE: A lot of things set to zero explicitly by call to
417 * sk_alloc() so need not be done here.
418 */
419void tcp_init_sock(struct sock *sk)
420{
421 struct inet_connection_sock *icsk = inet_csk(sk);
422 struct tcp_sock *tp = tcp_sk(sk);
423
9f5afeae 424 tp->out_of_order_queue = RB_ROOT;
75c119af 425 sk->tcp_rtx_queue = RB_ROOT;
900f65d3 426 tcp_init_xmit_timers(sk);
46d3ceab 427 INIT_LIST_HEAD(&tp->tsq_node);
e2080072 428 INIT_LIST_HEAD(&tp->tsorted_sent_queue);
900f65d3
NC		 429
430 icsk->icsk_rto = TCP_TIMEOUT_INIT;
ca584ba0 431 icsk->icsk_rto_min = TCP_RTO_MIN;
2b8ee4f0 432 icsk->icsk_delack_max = TCP_DELACK_MAX;
740b0f18 433 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
ac9517fc 434 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
900f65d3
NC		 435
436 /* So many TCP implementations out there (incorrectly) count the
437 * initial SYN frame in their delayed-ACK and congestion control
438 * algorithms that we must have the following bandaid to talk
439 * efficiently to them. -DaveM
440 */
40570375 441 tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
900f65d3 442
d7722e85
SHY		 443 /* There's a bubble in the pipe until at least the first ACK. */
444 tp->app_limited = ~0U;
300b655d 445 tp->rate_app_limited = 1;
d7722e85 446
900f65d3
NC		 447 /* See draft-stevens-tcpca-spec-01 for discussion of the
448 * initialization of these values.
449 */
450 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
451 tp->snd_cwnd_clamp = ~0;
452 tp->mss_cache = TCP_MSS_DEFAULT;
453
46778cd1 454 tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
55d8694f 455 tcp_assign_congestion_control(sk);
900f65d3 456
ceaa1fef 457 tp->tsoffset = 0;
1f255691 458 tp->rack.reo_wnd_steps = 1;
ceaa1fef 459
900f65d3
NC		 460 sk->sk_write_space = sk_stream_write_space;
461 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
462
463 icsk->icsk_sync_mss = tcp_sync_mss;
464
02739545
KI		 465 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
466 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
dfa2f048 467 tcp_scaling_ratio_init(sk);
900f65d3 468
e993ffe3 469 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
900f65d3 470 sk_sockets_allocated_inc(sk);
900f65d3
NC		 471}
472EXPORT_SYMBOL(tcp_init_sock);
473
4e8cc228 474static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
4ed2d765 475{
4e8cc228
ED		 476 struct sk_buff *skb = tcp_write_queue_tail(sk);
477
ad02c4f5 478 if (tsflags && skb) {
f066e2b0 479 struct skb_shared_info *shinfo = skb_shinfo(skb);
6b084928 480 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
4ed2d765 481
c14ac945 482 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
0a2cf20c
SHY		 483 if (tsflags & SOF_TIMESTAMPING_TX_ACK)
484 tcb->txstamp_ack = 1;
485 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
f066e2b0
WB		 486 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
487 }
4ed2d765
WB		 488}
489
05dc72ab 490static bool tcp_stream_is_readable(struct sock *sk, int target)
8934ce2f 491{
05dc72ab
ED		 492 if (tcp_epollin_ready(sk, target))
493 return true;
7b50ecfc 494 return sk_is_readable(sk);
8934ce2f
JF		 495}
496
1da177e4 497/*
a11e1d43
LT		 498 * Wait for a TCP event.
499 *
500 * Note that we don't need to lock the socket, as the upper poll layers
501 * take care of normal races (between the test and the event) and we don't
502 * go look at any of the socket buffers directly.
1da177e4 503 */
a11e1d43 504__poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
1da177e4 505{
a11e1d43 506 __poll_t mask;
1da177e4 507 struct sock *sk = sock->sk;
cf533ea5 508 const struct tcp_sock *tp = tcp_sk(sk);
e14cadfd 509 u8 shutdown;
00fd38d9 510 int state;
1da177e4 511
89ab066d 512 sock_poll_wait(file, sock, wait);
a11e1d43 513
986ffdfd 514 state = inet_sk_state_load(sk);
00fd38d9 515 if (state == TCP_LISTEN)
dc40c7bc 516 return inet_csk_listen_poll(sk);
1da177e4 517
a11e1d43
LT		 518 /* Socket is not locked. We are protected from async events
519 * by poll logic and correct handling of state changes
520 * made by other threads is impossible in any case.
521 */
522
523 mask = 0;
524
1da177e4 525 /*
a9a08845 526 * EPOLLHUP is certainly not done right. But poll() doesn't
1da177e4
LT		 527 * have a notion of HUP in just one direction, and for a
528 * socket the read side is more interesting.
529 *
a9a08845
LT		 530 * Some poll() documentation says that EPOLLHUP is incompatible
531 * with the EPOLLOUT/POLLWR flags, so somebody should check this
1da177e4
LT		 532 * all. But careful, it tends to be safer to return too many
533 * bits than too few, and you can easily break real applications
534 * if you don't tell them that something has hung up!
535 *
536 * Check-me.
537 *
a9a08845 538 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
1da177e4
LT		 539 * our fs/select.c). It means that after we received EOF,
540 * poll always returns immediately, making impossible poll() on write()
a9a08845 541 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
1da177e4
LT		 542 * if and only if shutdown has been made in both directions.
543 * Actually, it is interesting to look how Solaris and DUX
a9a08845 544 * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
1da177e4
LT		 545 * then we could set it on SND_SHUTDOWN. BTW examples given
546 * in Stevens' books assume exactly this behaviour, it explains
a9a08845 547 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK
1da177e4
LT		 548 *
549 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
550 * blocking on fresh not-connected or disconnected socket. --ANK
551 */
e14cadfd
ED		 552 shutdown = READ_ONCE(sk->sk_shutdown);
553 if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
a9a08845 554 mask |= EPOLLHUP;
e14cadfd 555 if (shutdown & RCV_SHUTDOWN)
a9a08845 556 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
1da177e4 557
8336886f 558 /* Connected or passive Fast Open socket? */
00fd38d9 559 if (state != TCP_SYN_SENT &&
d983ea6f 560 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
c7004482 561 int target = sock_rcvlowat(sk, 0, INT_MAX);
7b6a893a 562 u16 urg_data = READ_ONCE(tp->urg_data);
c7004482 563
b96c51bd 564 if (unlikely(urg_data) &&
7b6a893a
ED		 565 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
566 !sock_flag(sk, SOCK_URGINLINE))
b634f875 567 target++;
c7004482 568
05dc72ab 569 if (tcp_stream_is_readable(sk, target))
a9a08845 570 mask |= EPOLLIN | EPOLLRDNORM;
1da177e4 571
e14cadfd 572 if (!(shutdown & SEND_SHUTDOWN)) {
8ba3c9d1 573 if (__sk_stream_is_writeable(sk, 1)) {
a9a08845 574 mask |= EPOLLOUT | EPOLLWRNORM;
1da177e4 575 } else { /* send SIGIO later */
9cd3e072 576 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 577 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
578
579 /* Race breaker. If space is freed after
580 * wspace test but before the flags are set,
3c715127 581 * IO signal will be lost. Memory barrier
582 * pairs with the input side.
1da177e4 583 */
3c715127 584 smp_mb__after_atomic();
8ba3c9d1 585 if (__sk_stream_is_writeable(sk, 1))
a9a08845 586 mask |= EPOLLOUT | EPOLLWRNORM;
1da177e4 587 }
d84ba638 588 } else
a9a08845 589 mask |= EPOLLOUT | EPOLLWRNORM;
1da177e4 590
7b6a893a 591 if (urg_data & TCP_URG_VALID)
a9a08845 592 mask |= EPOLLPRI;
08e39c0d
ED		 593 } else if (state == TCP_SYN_SENT &&
594 inet_test_bit(DEFER_CONNECT, sk)) {
19f6d3f3 595 /* Active TCP fastopen socket with defer_connect
a9a08845 596 * Return EPOLLOUT so application can call write()
19f6d3f3
WW		 597 * in order for kernel to generate SYN+data
598 */
a9a08845 599 mask |= EPOLLOUT | EPOLLWRNORM;
1da177e4 600 }
a4d25803
TM		 601 /* This barrier is coupled with smp_wmb() in tcp_reset() */
602 smp_rmb();
e13ec3da
ED		 603 if (READ_ONCE(sk->sk_err) ||
604 !skb_queue_empty_lockless(&sk->sk_error_queue))
a9a08845 605 mask |= EPOLLERR;
a4d25803 606
1da177e4
LT		 607 return mask;
608}
a11e1d43 609EXPORT_SYMBOL(tcp_poll);
1da177e4 610
e1d001fa 611int tcp_ioctl(struct sock *sk, int cmd, int *karg)
1da177e4
LT		 612{
613 struct tcp_sock *tp = tcp_sk(sk);
614 int answ;
0e71c55c 615 bool slow;
1da177e4
LT		 616
617 switch (cmd) {
618 case SIOCINQ:
619 if (sk->sk_state == TCP_LISTEN)
620 return -EINVAL;
621
0e71c55c 622 slow = lock_sock_fast(sk);
473bd239 623 answ = tcp_inq(sk);
0e71c55c 624 unlock_sock_fast(sk, slow);
1da177e4
LT		 625 break;
626 case SIOCATMARK:
7b6a893a 627 answ = READ_ONCE(tp->urg_data) &&
d9b55bf7 628 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
1da177e4
LT		 629 break;
630 case SIOCOUTQ:
631 if (sk->sk_state == TCP_LISTEN)
632 return -EINVAL;
633
634 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
635 answ = 0;
636 else
0f317464 637 answ = READ_ONCE(tp->write_seq) - tp->snd_una;
1da177e4 638 break;
2f4e1b39
MS		 639 case SIOCOUTQNSD:
640 if (sk->sk_state == TCP_LISTEN)
641 return -EINVAL;
642
643 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
644 answ = 0;
645 else
e0d694d6
ED		 646 answ = READ_ONCE(tp->write_seq) -
647 READ_ONCE(tp->snd_nxt);
2f4e1b39 648 break;
1da177e4
LT		 649 default:
650 return -ENOIOCTLCMD;
3ff50b79 651 }
1da177e4 652
e1d001fa
BL		 653 *karg = answ;
654 return 0;
1da177e4 655}
4bc2f18b 656EXPORT_SYMBOL(tcp_ioctl);
1da177e4 657
04d8825c 658void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
1da177e4 659{
4de075e0 660 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
1da177e4
LT		 661 tp->pushed_seq = tp->write_seq;
662}
663
a2a385d6 664static inline bool forced_push(const struct tcp_sock *tp)
1da177e4
LT		 665{
666 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
667}
668
04d8825c 669void tcp_skb_entail(struct sock *sk, struct sk_buff *skb)
1da177e4 670{
9e412ba7 671 struct tcp_sock *tp = tcp_sk(sk);
352d4800
ACM		 672 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
673
352d4800 674 tcb->seq = tcb->end_seq = tp->write_seq;
4de075e0 675 tcb->tcp_flags = TCPHDR_ACK;
f4a775d1 676 __skb_header_release(skb);
fe067e8a 677 tcp_add_write_queue_tail(sk, skb);
ab4e846a 678 sk_wmem_queued_add(sk, skb->truesize);
3ab224be 679 sk_mem_charge(sk, skb->truesize);
89ebd197 680 if (tp->nonagle & TCP_NAGLE_PUSH)
e905a9ed 681 tp->nonagle &= ~TCP_NAGLE_PUSH;
6f021c62
ED		 682
683 tcp_slow_start_after_idle_check(sk);
1da177e4
LT		 684}
685
afeca340 686static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
1da177e4 687{
33f5f57e 688 if (flags & MSG_OOB)
1da177e4 689 tp->snd_up = tp->write_seq;
1da177e4
LT		 690}
691
f54b3111 692/* If a not yet filled skb is pushed, do not send it if
a181ceb5 693 * we have data packets in Qdisc or NIC queues :
f54b3111
ED		 694 * Because TX completion will happen shortly, it gives a chance
695 * to coalesce future sendmsg() payload into this skb, without
696 * need for a timer, and with no latency trade off.
697 * As packets containing data payload have a bigger truesize
a181ceb5
ED		 698 * than pure acks (dataless) packets, the last checks prevent
699 * autocorking if we only have an ACK in Qdisc/NIC queues,
700 * or if TX completion was delayed after we processed ACK packet.
f54b3111
ED		 701 */
702static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
703 int size_goal)
1da177e4 704{
f54b3111 705 return skb->len < size_goal &&
85225e6f 706 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
114f39fe 707 !tcp_rtx_queue_empty(sk) &&
b0de0cf4
ED		 708 refcount_read(&sk->sk_wmem_alloc) > skb->truesize &&
709 tcp_skb_can_collapse_to(skb);
f54b3111
ED		 710}
711
35b2c321
MM		 712void tcp_push(struct sock *sk, int flags, int mss_now,
713 int nonagle, int size_goal)
f54b3111
ED		 714{
715 struct tcp_sock *tp = tcp_sk(sk);
716 struct sk_buff *skb;
afeca340 717
f54b3111 718 skb = tcp_write_queue_tail(sk);
75c119af
ED		 719 if (!skb)
720 return;
f54b3111
ED		 721 if (!(flags & MSG_MORE) || forced_push(tp))
722 tcp_mark_push(tp, skb);
723
724 tcp_mark_urg(tp, flags);
725
726 if (tcp_should_autocork(sk, skb, size_goal)) {
727
728 /* avoid atomic op if TSQ_THROTTLED bit is already set */
7aa5470c 729 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
f54b3111 730 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
7aa5470c 731 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
7267e8dc 732 smp_mb__after_atomic();
f54b3111 733 }
a181ceb5
ED		 734 /* It is possible TX completion already happened
735 * before we set TSQ_THROTTLED.
736 */
14afee4b 737 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
a181ceb5 738 return;
1da177e4 739 }
f54b3111
ED		 740
741 if (flags & MSG_MORE)
742 nonagle = TCP_NAGLE_CORK;
743
744 __tcp_push_pending_frames(sk, mss_now, nonagle);
1da177e4
LT		 745}
746
6ff7751d
AB		 747static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
748 unsigned int offset, size_t len)
9c55e01c
JA		 749{
750 struct tcp_splice_state *tss = rd_desc->arg.data;
33966dd0 751 int ret;
9c55e01c 752
a60e3cc7 753 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
25869262 754 min(rd_desc->count, len), tss->flags);
33966dd0
WT		 755 if (ret > 0)
756 rd_desc->count -= ret;
757 return ret;
9c55e01c
JA		 758}
759
760static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
761{
762 /* Store TCP splice context information in read_descriptor_t. */
763 read_descriptor_t rd_desc = {
764 .arg.data = tss,
33966dd0 765 .count = tss->len,
9c55e01c
JA		 766 };
767
768 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
769}
770
771/**
772 * tcp_splice_read - splice data from TCP socket to a pipe
773 * @sock: socket to splice from
774 * @ppos: position (not valid)
775 * @pipe: pipe to splice to
776 * @len: number of bytes to splice
777 * @flags: splice modifier flags
778 *
779 * Description:
780 * Will read pages from given socket and fill them into a pipe.
781 *
782 **/
783ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
784 struct pipe_inode_info *pipe, size_t len,
785 unsigned int flags)
786{
787 struct sock *sk = sock->sk;
788 struct tcp_splice_state tss = {
789 .pipe = pipe,
790 .len = len,
791 .flags = flags,
792 };
793 long timeo;
794 ssize_t spliced;
795 int ret;
796
3a047bf8 797 sock_rps_record_flow(sk);
9c55e01c
JA		 798 /*
799 * We can't seek on a socket input
800 */
801 if (unlikely(*ppos))
802 return -ESPIPE;
803
804 ret = spliced = 0;
805
806 lock_sock(sk);
807
42324c62 808 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
9c55e01c
JA		 809 while (tss.len) {
810 ret = __tcp_splice_read(sk, &tss);
811 if (ret < 0)
812 break;
813 else if (!ret) {
814 if (spliced)
815 break;
9c55e01c
JA		 816 if (sock_flag(sk, SOCK_DONE))
817 break;
818 if (sk->sk_err) {
819 ret = sock_error(sk);
820 break;
821 }
822 if (sk->sk_shutdown & RCV_SHUTDOWN)
823 break;
824 if (sk->sk_state == TCP_CLOSE) {
825 /*
826 * This occurs when user tries to read
827 * from never connected socket.
828 */
c47078d6 829 ret = -ENOTCONN;
9c55e01c
JA		 830 break;
831 }
832 if (!timeo) {
833 ret = -EAGAIN;
834 break;
835 }
ccf7abb9
ED		 836 /* if __tcp_splice_read() got nothing while we have
837 * an skb in receive queue, we do not want to loop.
838 * This might happen with URG data.
839 */
840 if (!skb_queue_empty(&sk->sk_receive_queue))
841 break;
419ce133
PA		 842 ret = sk_wait_data(sk, &timeo, NULL);
843 if (ret < 0)
844 break;
9c55e01c
JA		 845 if (signal_pending(current)) {
846 ret = sock_intr_errno(timeo);
847 break;
848 }
849 continue;
850 }
851 tss.len -= ret;
852 spliced += ret;
853
2fe11c9d 854 if (!tss.len || !timeo)
33966dd0 855 break;
9c55e01c
JA		 856 release_sock(sk);
857 lock_sock(sk);
858
859 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
33966dd0 860 (sk->sk_shutdown & RCV_SHUTDOWN) ||
9c55e01c
JA		 861 signal_pending(current))
862 break;
863 }
864
865 release_sock(sk);
866
867 if (spliced)
868 return spliced;
869
870 return ret;
871}
4bc2f18b 872EXPORT_SYMBOL(tcp_splice_read);
9c55e01c 873
5882efff 874struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
f8dd3b8d 875 bool force_schedule)
f561d0f2
PE		 876{
877 struct sk_buff *skb;
878
5882efff 879 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
8e4d980a 880 if (likely(skb)) {
eb934478 881 bool mem_scheduled;
8e4d980a 882
9b65b17d 883 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
eb934478
ED		 884 if (force_schedule) {
885 mem_scheduled = true;
8e4d980a
ED		 886 sk_forced_mem_schedule(sk, skb->truesize);
887 } else {
eb934478 888 mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
8e4d980a 889 }
eb934478 890 if (likely(mem_scheduled)) {
8a794df6 891 skb_reserve(skb, MAX_TCP_HEADER);
a52fe46e 892 skb->ip_summed = CHECKSUM_PARTIAL;
e2080072 893 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
f561d0f2
PE		 894 return skb;
895 }
896 __kfree_skb(skb);
897 } else {
5c52ba17 898 sk->sk_prot->enter_memory_pressure(sk);
f561d0f2
PE		 899 sk_stream_moderate_sndbuf(sk);
900 }
901 return NULL;
902}
903
0c54b85f
IJ		 904static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
905 int large_allowed)
906{
907 struct tcp_sock *tp = tcp_sk(sk);
6c09fa09 908 u32 new_size_goal, size_goal;
605ad7f1 909
74d4a8f8 910 if (!large_allowed)
605ad7f1
ED		 911 return mss_now;
912
6c09fa09 913 /* Note : tcp_tso_autosize() will eventually split this later */
ab14f180 914 new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size);
605ad7f1
ED		 915
916 /* We try hard to avoid divides here */
917 size_goal = tp->gso_segs * mss_now;
918 if (unlikely(new_size_goal < size_goal ||
919 new_size_goal >= size_goal + mss_now)) {
920 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
921 sk->sk_gso_max_segs);
922 size_goal = tp->gso_segs * mss_now;
0c54b85f
IJ		 923 }
924
605ad7f1 925 return max(size_goal, mss_now);
0c54b85f
IJ		 926}
927
35b2c321 928int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
0c54b85f
IJ		 929{
930 int mss_now;
931
932 mss_now = tcp_current_mss(sk);
933 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
934
935 return mss_now;
936}
937
72bf4f17
ED		 938/* In some cases, sendmsg() could have added an skb to the write queue,
939 * but failed adding payload on it. We need to remove it to consume less
dc97391e 940 * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger
72bf4f17
ED		 941 * epoll() users. Another reason is that tcp_write_xmit() does not like
942 * finding an empty skb in the write queue.
fdfc5c85 943 */
27728ba8 944void tcp_remove_empty_skb(struct sock *sk)
fdfc5c85 945{
27728ba8
ED		 946 struct sk_buff *skb = tcp_write_queue_tail(sk);
947
cf12e6f9 948 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
fdfc5c85
ED		 949 tcp_unlink_write_queue(skb, sk);
950 if (tcp_write_queue_empty(sk))
951 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
03271f3a 952 tcp_wmem_free_skb(sk, skb);
fdfc5c85
ED		 953 }
954}
955
f8d9d938
ED		 956/* skb changing from pure zc to mixed, must charge zc */
957static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
958{
959 if (unlikely(skb_zcopy_pure(skb))) {
960 u32 extra = skb->truesize -
961 SKB_TRUESIZE(skb_end_offset(skb));
962
963 if (!sk_wmem_schedule(sk, extra))
964 return -ENOMEM;
965
966 sk_mem_charge(sk, extra);
967 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
968 }
969 return 0;
970}
971
f54755f6 972
fbf93406 973int tcp_wmem_schedule(struct sock *sk, int copy)
849b425c
ED		 974{
975 int left;
976
977 if (likely(sk_wmem_schedule(sk, copy)))
978 return copy;
979
980 /* We could be in trouble if we have nothing queued.
981 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
982 * to guarantee some progress.
983 */
683a67da 984 left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued;
849b425c
ED		 985 if (left > 0)
986 sk_forced_mem_schedule(sk, min(left, copy));
987 return min(copy, sk->sk_forward_alloc);
988}
989
cf60af03
YC		 990void tcp_free_fastopen_req(struct tcp_sock *tp)
991{
00db4124 992 if (tp->fastopen_req) {
cf60af03
YC		 993 kfree(tp->fastopen_req);
994 tp->fastopen_req = NULL;
995 }
996}
997
3242abeb
BH		 998int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
999 size_t size, struct ubuf_info *uarg)
cf60af03
YC		 1000{
1001 struct tcp_sock *tp = tcp_sk(sk);
19f6d3f3 1002 struct inet_sock *inet = inet_sk(sk);
ba615f67 1003 struct sockaddr *uaddr = msg->msg_name;
cf60af03
YC		 1004 int err, flags;
1005
5a542133
KI		 1006 if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
1007 TFO_CLIENT_ENABLE) ||
ba615f67
WW		 1008 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1009 uaddr->sa_family == AF_UNSPEC))
cf60af03 1010 return -EOPNOTSUPP;
00db4124 1011 if (tp->fastopen_req)
cf60af03
YC		 1012 return -EALREADY; /* Another Fast Open is in progress */
1013
1014 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1015 sk->sk_allocation);
51456b29 1016 if (unlikely(!tp->fastopen_req))
cf60af03
YC		 1017 return -ENOBUFS;
1018 tp->fastopen_req->data = msg;
f5ddcbbb 1019 tp->fastopen_req->size = size;
f859a448 1020 tp->fastopen_req->uarg = uarg;
cf60af03 1021
08e39c0d 1022 if (inet_test_bit(DEFER_CONNECT, sk)) {
19f6d3f3
WW		 1023 err = tcp_connect(sk);
1024 /* Same failure procedure as in tcp_v4/6_connect */
1025 if (err) {
1026 tcp_set_state(sk, TCP_CLOSE);
1027 inet->inet_dport = 0;
1028 sk->sk_route_caps = 0;
1029 }
1030 }
cf60af03 1031 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
ba615f67 1032 err = __inet_stream_connect(sk->sk_socket, uaddr,
3979ad7e 1033 msg->msg_namelen, flags, 1);
7db92362
WW		 1034 /* fastopen_req could already be freed in __inet_stream_connect
1035 * if the connection times out or gets rst
1036 */
1037 if (tp->fastopen_req) {
1038 *copied = tp->fastopen_req->copied;
1039 tcp_free_fastopen_req(tp);
08e39c0d 1040 inet_clear_bit(DEFER_CONNECT, sk);
7db92362 1041 }
cf60af03
YC		 1042 return err;
1043}
1044
306b13eb 1045int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1da177e4 1046{
1da177e4 1047 struct tcp_sock *tp = tcp_sk(sk);
f214f915 1048 struct ubuf_info *uarg = NULL;
1da177e4 1049 struct sk_buff *skb;
c14ac945 1050 struct sockcm_cookie sockc;
57be5bda
AV		 1051 int flags, err, copied = 0;
1052 int mss_now = 0, size_goal, copied_syn = 0;
1a991488 1053 int process_backlog = 0;
270a1c3d 1054 int zc = 0;
1da177e4
LT		 1055 long timeo;
1056
1da177e4 1057 flags = msg->msg_flags;
f214f915 1058
eb315a7d 1059 if ((flags & MSG_ZEROCOPY) && size) {
eb315a7d
PB		 1060 if (msg->msg_ubuf) {
1061 uarg = msg->msg_ubuf;
270a1c3d
DH		 1062 if (sk->sk_route_caps & NETIF_F_SG)
1063 zc = MSG_ZEROCOPY;
eb315a7d 1064 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
eea96a3e 1065 skb = tcp_write_queue_tail(sk);
eb315a7d
PB		 1066 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
1067 if (!uarg) {
1068 err = -ENOBUFS;
1069 goto out_err;
1070 }
270a1c3d
DH		 1071 if (sk->sk_route_caps & NETIF_F_SG)
1072 zc = MSG_ZEROCOPY;
1073 else
e7d2b510 1074 uarg_to_msgzc(uarg)->zerocopy = 0;
eb315a7d 1075 }
270a1c3d
DH		 1076 } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
1077 if (sk->sk_route_caps & NETIF_F_SG)
1078 zc = MSG_SPLICE_PAGES;
f214f915
WB		 1079 }
1080
08e39c0d
ED		 1081 if (unlikely(flags & MSG_FASTOPEN ||
1082 inet_test_bit(DEFER_CONNECT, sk)) &&
16ae6aa1 1083 !tp->repair) {
f859a448 1084 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
cf60af03
YC		 1085 if (err == -EINPROGRESS && copied_syn > 0)
1086 goto out;
1087 else if (err)
1088 goto out_err;
cf60af03
YC		 1089 }
1090
1da177e4
LT		 1091 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1092
d7722e85
SHY		 1093 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1094
8336886f
JC		 1095 /* Wait for a connection to finish. One exception is TCP Fast Open
1096 * (passive side) where data is allowed to be sent before a connection
1097 * is fully established.
1098 */
1099 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1100 !tcp_passive_fastopen(sk)) {
686a5624
YM		 1101 err = sk_stream_wait_connect(sk, &timeo);
1102 if (err != 0)
cf60af03 1103 goto do_error;
8336886f 1104 }
1da177e4 1105
c0e88ff0
PE		 1106 if (unlikely(tp->repair)) {
1107 if (tp->repair_queue == TCP_RECV_QUEUE) {
1108 copied = tcp_send_rcvq(sk, msg, size);
5924f17a 1109 goto out_nopush;
c0e88ff0
PE		 1110 }
1111
1112 err = -EINVAL;
1113 if (tp->repair_queue == TCP_NO_QUEUE)
1114 goto out_err;
1115
1116 /* 'common' sending to sendq */
1117 }
1118
657a0667 1119 sockcm_init(&sockc, sk);
c14ac945
SHY		 1120 if (msg->msg_controllen) {
1121 err = sock_cmsg_send(sk, msg, &sockc);
1122 if (unlikely(err)) {
1123 err = -EINVAL;
1124 goto out_err;
1125 }
1126 }
1127
1da177e4 1128 /* This should be in poll */
9cd3e072 1129 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4 1130
1da177e4 1131 /* Ok commence sending. */
1da177e4
LT		 1132 copied = 0;
1133
d41a69f1
ED		 1134restart:
1135 mss_now = tcp_send_mss(sk, &size_goal, flags);
1136
1da177e4
LT		 1137 err = -EPIPE;
1138 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
79d8665b 1139 goto do_error;
1da177e4 1140
01e97e65 1141 while (msg_data_left(msg)) {
270a1c3d 1142 ssize_t copy = 0;
1da177e4 1143
57be5bda 1144 skb = tcp_write_queue_tail(sk);
65ec6097
ED		 1145 if (skb)
1146 copy = size_goal - skb->len;
1da177e4 1147
c134ecb8 1148 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
3613b3db
ED		 1149 bool first_skb;
1150
1da177e4 1151new_segment:
57be5bda 1152 if (!sk_stream_memory_free(sk))
afb83012 1153 goto wait_for_space;
1da177e4 1154
1a991488
ED		 1155 if (unlikely(process_backlog >= 16)) {
1156 process_backlog = 0;
1157 if (sk_flush_backlog(sk))
1158 goto restart;
d4011239 1159 }
75c119af 1160 first_skb = tcp_rtx_and_write_queues_empty(sk);
5882efff 1161 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
f8dd3b8d 1162 first_skb);
57be5bda 1163 if (!skb)
afb83012 1164 goto wait_for_space;
1da177e4 1165
1a991488 1166 process_backlog++;
1da177e4 1167
04d8825c 1168 tcp_skb_entail(sk, skb);
57be5bda 1169 copy = size_goal;
9d186cac 1170
57be5bda 1171 /* All packets are restored as if they have
d3edd06e 1172 * already been sent. skb_mstamp_ns isn't set to
57be5bda
AV		 1173 * avoid wrong rtt estimation.
1174 */
1175 if (tp->repair)
1176 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1177 }
1da177e4 1178
57be5bda 1179 /* Try to append data to the end of skb. */
01e97e65
AV		 1180 if (copy > msg_data_left(msg))
1181 copy = msg_data_left(msg);
57be5bda 1182
270a1c3d 1183 if (zc == 0) {
57be5bda
AV		 1184 bool merge = true;
1185 int i = skb_shinfo(skb)->nr_frags;
1186 struct page_frag *pfrag = sk_page_frag(sk);
1187
1188 if (!sk_page_frag_refill(sk, pfrag))
afb83012 1189 goto wait_for_space;
ef015786 1190
57be5bda
AV		 1191 if (!skb_can_coalesce(skb, i, pfrag->page,
1192 pfrag->offset)) {
a86a0661 1193 if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
57be5bda
AV		 1194 tcp_mark_push(tp, skb);
1195 goto new_segment;
1da177e4 1196 }
57be5bda 1197 merge = false;
1da177e4
LT		 1198 }
1199
57be5bda
AV		 1200 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1201
eb315a7d
PB		 1202 if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
1203 if (tcp_downgrade_zcopy_pure(sk, skb))
1204 goto wait_for_space;
1205 skb_zcopy_downgrade_managed(skb);
1206 }
849b425c
ED		 1207
1208 copy = tcp_wmem_schedule(sk, copy);
1209 if (!copy)
afb83012 1210 goto wait_for_space;
1da177e4 1211
57be5bda
AV		 1212 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1213 pfrag->page,
1214 pfrag->offset,
1215 copy);
1216 if (err)
1217 goto do_error;
1da177e4 1218
57be5bda
AV		 1219 /* Update the skb. */
1220 if (merge) {
1221 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1222 } else {
1223 skb_fill_page_desc(skb, i, pfrag->page,
1224 pfrag->offset, copy);
4e33e346 1225 page_ref_inc(pfrag->page);
4ed2d765 1226 }
57be5bda 1227 pfrag->offset += copy;
270a1c3d 1228 } else if (zc == MSG_ZEROCOPY) {
9b65b17d
TA		 1229 /* First append to a fragless skb builds initial
1230 * pure zerocopy skb
1231 */
1232 if (!skb->len)
1233 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1234
1235 if (!skb_zcopy_pure(skb)) {
849b425c
ED		 1236 copy = tcp_wmem_schedule(sk, copy);
1237 if (!copy)
9b65b17d
TA		 1238 goto wait_for_space;
1239 }
358ed624 1240
f214f915 1241 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
111856c7
WB		 1242 if (err == -EMSGSIZE || err == -EEXIST) {
1243 tcp_mark_push(tp, skb);
f214f915 1244 goto new_segment;
111856c7 1245 }
f214f915
WB		 1246 if (err < 0)
1247 goto do_error;
1248 copy = err;
270a1c3d
DH		 1249 } else if (zc == MSG_SPLICE_PAGES) {
1250 /* Splice in data if we can; copy if we can't. */
1251 if (tcp_downgrade_zcopy_pure(sk, skb))
1252 goto wait_for_space;
1253 copy = tcp_wmem_schedule(sk, copy);
1254 if (!copy)
1255 goto wait_for_space;
1256
1257 err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
1258 sk->sk_allocation);
1259 if (err < 0) {
1260 if (err == -EMSGSIZE) {
1261 tcp_mark_push(tp, skb);
1262 goto new_segment;
1263 }
1264 goto do_error;
1265 }
1266 copy = err;
1267
1268 if (!(flags & MSG_NO_SHARED_FRAGS))
1269 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1270
1271 sk_wmem_queued_add(sk, copy);
1272 sk_mem_charge(sk, copy);
57be5bda
AV		 1273 }
1274
1275 if (!copied)
1276 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1277
0f317464 1278 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
57be5bda
AV		 1279 TCP_SKB_CB(skb)->end_seq += copy;
1280 tcp_skb_pcount_set(skb, 0);
1da177e4 1281
57be5bda 1282 copied += copy;
01e97e65 1283 if (!msg_data_left(msg)) {
c134ecb8
MKL		 1284 if (unlikely(flags & MSG_EOR))
1285 TCP_SKB_CB(skb)->eor = 1;
57be5bda
AV		 1286 goto out;
1287 }
1da177e4 1288
65ec6097 1289 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1da177e4
LT		 1290 continue;
1291
57be5bda
AV		 1292 if (forced_push(tp)) {
1293 tcp_mark_push(tp, skb);
1294 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1295 } else if (skb == tcp_send_head(sk))
1296 tcp_push_one(sk, mss_now);
1297 continue;
1298
afb83012 1299wait_for_space:
57be5bda 1300 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
72bf4f17 1301 tcp_remove_empty_skb(sk);
57be5bda
AV		 1302 if (copied)
1303 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1304 TCP_NAGLE_PUSH, size_goal);
1da177e4 1305
686a5624
YM		 1306 err = sk_stream_wait_memory(sk, &timeo);
1307 if (err != 0)
57be5bda 1308 goto do_error;
1da177e4 1309
57be5bda 1310 mss_now = tcp_send_mss(sk, &size_goal, flags);
1da177e4
LT		 1311 }
1312
1313out:
ad02c4f5 1314 if (copied) {
4e8cc228 1315 tcp_tx_timestamp(sk, sockc.tsflags);
f54b3111 1316 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
ad02c4f5 1317 }
5924f17a 1318out_nopush:
a7533584
PB		 1319 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1320 if (uarg && !msg->msg_ubuf)
1321 net_zcopy_put(uarg);
cf60af03 1322 return copied + copied_syn;
1da177e4 1323
fdfc5c85 1324do_error:
27728ba8 1325 tcp_remove_empty_skb(sk);
1da177e4 1326
cf60af03 1327 if (copied + copied_syn)
1da177e4
LT		 1328 goto out;
1329out_err:
a7533584
PB		 1330 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1331 if (uarg && !msg->msg_ubuf)
1332 net_zcopy_put_abort(uarg, true);
1da177e4 1333 err = sk_stream_error(sk, flags, err);
ce5ec440 1334 /* make sure we wake any epoll edge trigger waiter */
216808c6 1335 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
ce5ec440 1336 sk->sk_write_space(sk);
b0f71bd3
FY		 1337 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1338 }
1da177e4
LT		 1339 return err;
1340}
774c4673 1341EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
306b13eb
TH		 1342
1343int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1344{
1345 int ret;
1346
1347 lock_sock(sk);
1348 ret = tcp_sendmsg_locked(sk, msg, size);
1349 release_sock(sk);
1350
1351 return ret;
1352}
4bc2f18b 1353EXPORT_SYMBOL(tcp_sendmsg);
1da177e4 1354
1d7e4538
DH		 1355void tcp_splice_eof(struct socket *sock)
1356{
1357 struct sock *sk = sock->sk;
1358 struct tcp_sock *tp = tcp_sk(sk);
1359 int mss_now, size_goal;
1360
1361 if (!tcp_write_queue_tail(sk))
1362 return;
1363
1364 lock_sock(sk);
1365 mss_now = tcp_send_mss(sk, &size_goal, 0);
1366 tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);
1367 release_sock(sk);
1368}
1369EXPORT_SYMBOL_GPL(tcp_splice_eof);
1370
1da177e4
LT		 1371/*
1372 * Handle reading urgent data. BSD has very simple semantics for
1373 * this, no blocking and very strange errors 8)
1374 */
1375
377f0a08 1376static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1da177e4
LT		 1377{
1378 struct tcp_sock *tp = tcp_sk(sk);
1379
1380 /* No URG data to read. */
1381 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1382 tp->urg_data == TCP_URG_READ)
1383 return -EINVAL; /* Yes this is right ! */
1384
1385 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1386 return -ENOTCONN;
1387
1388 if (tp->urg_data & TCP_URG_VALID) {
1389 int err = 0;
1390 char c = tp->urg_data;
1391
1392 if (!(flags & MSG_PEEK))
7b6a893a 1393 WRITE_ONCE(tp->urg_data, TCP_URG_READ);
1da177e4
LT		 1394
1395 /* Read urgent data. */
1396 msg->msg_flags |= MSG_OOB;
1397
1398 if (len > 0) {
1399 if (!(flags & MSG_TRUNC))
7eab8d9e 1400 err = memcpy_to_msg(msg, &c, 1);
1da177e4
LT		 1401 len = 1;
1402 } else
1403 msg->msg_flags |= MSG_TRUNC;
1404
1405 return err ? -EFAULT : len;
1406 }
1407
1408 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1409 return 0;
1410
1411 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and
1412 * the available implementations agree in this case:
1413 * this call should never block, independent of the
1414 * blocking state of the socket.
1415 * Mike <pall@rz.uni-karlsruhe.de>
1416 */
1417 return -EAGAIN;
1418}
1419
c0e88ff0
PE		 1420static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1421{
1422 struct sk_buff *skb;
1423 int copied = 0, err = 0;
1424
75c119af
ED		 1425 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1426 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1427 if (err)
1428 return err;
1429 copied += skb->len;
1430 }
1431
c0e88ff0 1432 skb_queue_walk(&sk->sk_write_queue, skb) {
51f3d02b 1433 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
c0e88ff0
PE		 1434 if (err)
1435 break;
1436
1437 copied += skb->len;
1438 }
1439
1440 return err ?: copied;
1441}
1442
1da177e4
LT		 1443/* Clean up the receive buffer for full frames taken by the user,
1444 * then send an ACK if necessary. COPIED is the number of bytes
1445 * tcp_recvmsg has given to the user so far, it speeds up the
1446 * calculation of whether or not we must ACK for the sake of
1447 * a window update.
1448 */
e5c6de5f 1449void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1da177e4
LT		 1450{
1451 struct tcp_sock *tp = tcp_sk(sk);
a2a385d6 1452 bool time_to_ack = false;
1da177e4 1453
463c84b9
ACM		 1454 if (inet_csk_ack_scheduled(sk)) {
1455 const struct inet_connection_sock *icsk = inet_csk(sk);
b6b6d653
ED		 1456
1457 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */
463c84b9 1458 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1da177e4
LT		 1459 /*
1460 * If this read emptied read buffer, we send ACK, if
1461 * connection is not bidirectional, user drained
1462 * receive buffer and there was a small segment
1463 * in queue.
1464 */
1ef9696c
AK		 1465 (copied > 0 &&
1466 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1467 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
31954cd8 1468 !inet_csk_in_pingpong_mode(sk))) &&
1ef9696c 1469 !atomic_read(&sk->sk_rmem_alloc)))
a2a385d6 1470 time_to_ack = true;
1da177e4
LT		 1471 }
1472
1473 /* We send an ACK if we can now advertise a non-zero window
1474 * which has been raised "significantly".
1475 *
1476 * Even if window raised up to infinity, do not send window open ACK
1477 * in states, where we will not receive more. It is useless.
1478 */
1479 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1480 __u32 rcv_window_now = tcp_receive_window(tp);
1481
1482 /* Optimize, __tcp_select_window() is not cheap. */
1483 if (2*rcv_window_now <= tp->window_clamp) {
1484 __u32 new_window = __tcp_select_window(sk);
1485
1486 /* Send ACK now, if this read freed lots of space
1487 * in our buffer. Certainly, new_window is new window.
1488 * We can advertise it now, if it is not less than current one.
1489 * "Lots" means "at least twice" here.
1490 */
1491 if (new_window && new_window >= 2 * rcv_window_now)
a2a385d6 1492 time_to_ack = true;
1da177e4
LT		 1493 }
1494 }
1495 if (time_to_ack)
1496 tcp_send_ack(sk);
1497}
1498
c457985a
CW		 1499void tcp_cleanup_rbuf(struct sock *sk, int copied)
1500{
1501 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1502 struct tcp_sock *tp = tcp_sk(sk);
1503
1504 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1505 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1506 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1507 __tcp_cleanup_rbuf(sk, copied);
1508}
1509
3df684c1
ED		 1510static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
1511{
f35f8219 1512 __skb_unlink(skb, &sk->sk_receive_queue);
3df684c1
ED		 1513 if (likely(skb->destructor == sock_rfree)) {
1514 sock_rfree(skb);
1515 skb->destructor = NULL;
1516 skb->sk = NULL;
68822bdf 1517 return skb_attempt_defer_free(skb);
3df684c1 1518 }
f35f8219 1519 __kfree_skb(skb);
3df684c1
ED		 1520}
1521
3f92a64e 1522struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1da177e4
LT		 1523{
1524 struct sk_buff *skb;
1525 u32 offset;
1526
f26845b4 1527 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1da177e4 1528 offset = seq - TCP_SKB_CB(skb)->seq;
9d691539
ED		 1529 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1530 pr_err_once("%s: found a SYN, please report !\n", __func__);
1da177e4 1531 offset--;
9d691539 1532 }
e11ecddf 1533 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1da177e4
LT		 1534 *off = offset;
1535 return skb;
1536 }
f26845b4
ED		 1537 /* This looks weird, but this can happen if TCP collapsing
1538 * splitted a fat GRO packet, while we released socket lock
1539 * in skb_splice_bits()
1540 */
3df684c1 1541 tcp_eat_recv_skb(sk, skb);
1da177e4
LT		 1542 }
1543 return NULL;
1544}
3f92a64e 1545EXPORT_SYMBOL(tcp_recv_skb);
1da177e4
LT		 1546
1547/*
1548 * This routine provides an alternative to tcp_recvmsg() for routines
1549 * that would like to handle copying from skbuffs directly in 'sendfile'
1550 * fashion.
1551 * Note:
1552 * - It is assumed that the socket was locked by the caller.
1553 * - The routine does not block.
1554 * - At present, there is no support for reading OOB data
1555 * or for 'peeking' the socket using this routine
1556 * (although both would be easy to implement).
1557 */
1558int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1559 sk_read_actor_t recv_actor)
1560{
1561 struct sk_buff *skb;
1562 struct tcp_sock *tp = tcp_sk(sk);
1563 u32 seq = tp->copied_seq;
1564 u32 offset;
1565 int copied = 0;
1566
1567 if (sk->sk_state == TCP_LISTEN)
1568 return -ENOTCONN;
1569 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1570 if (offset < skb->len) {
374e7b59
OP		 1571 int used;
1572 size_t len;
1da177e4
LT		 1573
1574 len = skb->len - offset;
1575 /* Stop reading if we hit a patch of urgent data */
b96c51bd 1576 if (unlikely(tp->urg_data)) {
1da177e4
LT		 1577 u32 urg_offset = tp->urg_seq - seq;
1578 if (urg_offset < len)
1579 len = urg_offset;
1580 if (!len)
1581 break;
1582 }
1583 used = recv_actor(desc, skb, offset, len);
ff905b1e 1584 if (used <= 0) {
ddb61a57
JA		 1585 if (!copied)
1586 copied = used;
1587 break;
1da177e4 1588 }
e3d5ea2c
ED		 1589 if (WARN_ON_ONCE(used > len))
1590 used = len;
1591 seq += used;
1592 copied += used;
1593 offset += used;
1594
02275a2e 1595 /* If recv_actor drops the lock (e.g. TCP splice
293ad604
OP		 1596 * receive) the skb pointer might be invalid when
1597 * getting here: tcp_collapse might have deleted it
1598 * while aggregating skbs from the socket queue.
1599 */
02275a2e
WT		 1600 skb = tcp_recv_skb(sk, seq - 1, &offset);
1601 if (!skb)
1da177e4 1602 break;
02275a2e
WT		 1603 /* TCP coalescing might have appended data to the skb.
1604 * Try to splice more frags
1605 */
1606 if (offset + 1 != skb->len)
1607 continue;
1da177e4 1608 }
e11ecddf 1609 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
3df684c1 1610 tcp_eat_recv_skb(sk, skb);
1da177e4
LT		 1611 ++seq;
1612 break;
1613 }
3df684c1 1614 tcp_eat_recv_skb(sk, skb);
1da177e4
LT		 1615 if (!desc->count)
1616 break;
7db48e98 1617 WRITE_ONCE(tp->copied_seq, seq);
1da177e4 1618 }
7db48e98 1619 WRITE_ONCE(tp->copied_seq, seq);
1da177e4
LT		 1620
1621 tcp_rcv_space_adjust(sk);
1622
1623 /* Clean up data we have read: This will do ACK frames. */
f26845b4
ED		 1624 if (copied > 0) {
1625 tcp_recv_skb(sk, seq, &offset);
0e4b4992 1626 tcp_cleanup_rbuf(sk, copied);
f26845b4 1627 }
1da177e4
LT		 1628 return copied;
1629}
4bc2f18b 1630EXPORT_SYMBOL(tcp_read_sock);
1da177e4 1631
965b57b4 1632int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
04919bed 1633{
04919bed
CW		 1634 struct sk_buff *skb;
1635 int copied = 0;
04919bed
CW		 1636
1637 if (sk->sk_state == TCP_LISTEN)
1638 return -ENOTCONN;
1639
9b7177b1 1640 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
db4192a7
CW		 1641 u8 tcp_flags;
1642 int used;
04919bed 1643
db4192a7
CW		 1644 __skb_unlink(skb, &sk->sk_receive_queue);
1645 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1646 tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1647 used = recv_actor(sk, skb);
db4192a7
CW		 1648 if (used < 0) {
1649 if (!copied)
1650 copied = used;
1651 break;
1652 }
db4192a7 1653 copied += used;
04919bed 1654
9b7177b1 1655 if (tcp_flags & TCPHDR_FIN)
db4192a7 1656 break;
04919bed 1657 }
04919bed
CW		 1658 return copied;
1659}
1660EXPORT_SYMBOL(tcp_read_skb);
1661
3f92a64e
JK		 1662void tcp_read_done(struct sock *sk, size_t len)
1663{
1664 struct tcp_sock *tp = tcp_sk(sk);
1665 u32 seq = tp->copied_seq;
1666 struct sk_buff *skb;
1667 size_t left;
1668 u32 offset;
1669
1670 if (sk->sk_state == TCP_LISTEN)
1671 return;
1672
1673 left = len;
1674 while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1675 int used;
1676
1677 used = min_t(size_t, skb->len - offset, left);
1678 seq += used;
1679 left -= used;
1680
1681 if (skb->len > offset + used)
1682 break;
1683
1684 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1685 tcp_eat_recv_skb(sk, skb);
1686 ++seq;
1687 break;
1688 }
1689 tcp_eat_recv_skb(sk, skb);
1690 }
1691 WRITE_ONCE(tp->copied_seq, seq);
1692
1693 tcp_rcv_space_adjust(sk);
1694
1695 /* Clean up data we have read: This will do ACK frames. */
1696 if (left != len)
1697 tcp_cleanup_rbuf(sk, len - left);
1698}
1699EXPORT_SYMBOL(tcp_read_done);
1700
32035585
TH		 1701int tcp_peek_len(struct socket *sock)
1702{
1703 return tcp_inq(sock->sk);
1704}
1705EXPORT_SYMBOL(tcp_peek_len);
1706
d1361840
ED		 1707/* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
1708int tcp_set_rcvlowat(struct sock *sk, int val)
1709{
dfa2f048 1710 int space, cap;
867f816b
SHY		 1711
1712 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1713 cap = sk->sk_rcvbuf >> 1;
1714 else
02739545 1715 cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
867f816b 1716 val = min(val, cap);
eac66402 1717 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
03f45c88
ED		 1718
1719 /* Check if we need to signal EPOLLIN right now */
1720 tcp_data_ready(sk);
1721
d1361840
ED		 1722 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1723 return 0;
1724
dfa2f048
ED		 1725 space = tcp_space_from_win(sk, val);
1726 if (space > sk->sk_rcvbuf) {
1727 WRITE_ONCE(sk->sk_rcvbuf, space);
f410cbea 1728 WRITE_ONCE(tcp_sk(sk)->window_clamp, val);
d1361840
ED		 1729 }
1730 return 0;
1731}
1732EXPORT_SYMBOL(tcp_set_rcvlowat);
1733
892bfd3d
FW		 1734void tcp_update_recv_tstamps(struct sk_buff *skb,
1735 struct scm_timestamping_internal *tss)
7eeba170
AR		 1736{
1737 if (skb->tstamp)
1738 tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1739 else
1740 tss->ts[0] = (struct timespec64) {0};
1741
1742 if (skb_hwtstamps(skb)->hwtstamp)
1743 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1744 else
1745 tss->ts[2] = (struct timespec64) {0};
1746}
1747
05255b82 1748#ifdef CONFIG_MMU
350f6bbc 1749static const struct vm_operations_struct tcp_vm_ops = {
05255b82
ED		 1750};
1751
93ab6cc6
ED		 1752int tcp_mmap(struct file *file, struct socket *sock,
1753 struct vm_area_struct *vma)
1754{
05255b82
ED		 1755 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1756 return -EPERM;
1c71222e 1757 vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC);
05255b82 1758
3e4e28c5 1759 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1c71222e 1760 vm_flags_set(vma, VM_MIXEDMAP);
05255b82
ED		 1761
1762 vma->vm_ops = &tcp_vm_ops;
1763 return 0;
1764}
1765EXPORT_SYMBOL(tcp_mmap);
1766
7fba5309
AR		 1767static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
1768 u32 *offset_frag)
1769{
1770 skb_frag_t *frag;
1771
70701b83
AR		 1772 if (unlikely(offset_skb >= skb->len))
1773 return NULL;
1774
7fba5309
AR		 1775 offset_skb -= skb_headlen(skb);
1776 if ((int)offset_skb < 0 || skb_has_frag_list(skb))
1777 return NULL;
1778
1779 frag = skb_shinfo(skb)->frags;
1780 while (offset_skb) {
1781 if (skb_frag_size(frag) > offset_skb) {
1782 *offset_frag = offset_skb;
1783 return frag;
1784 }
1785 offset_skb -= skb_frag_size(frag);
1786 ++frag;
1787 }
1788 *offset_frag = 0;
1789 return frag;
1790}
1791
98917cf0
AR		 1792static bool can_map_frag(const skb_frag_t *frag)
1793{
577e4432
ED		 1794 struct page *page;
1795
1796 if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag))
1797 return false;
1798
1799 page = skb_frag_page(frag);
1800
1801 if (PageCompound(page) || page->mapping)
1802 return false;
1803
1804 return true;
98917cf0
AR		 1805}
1806
1807static int find_next_mappable_frag(const skb_frag_t *frag,
1808 int remaining_in_skb)
1809{
1810 int offset = 0;
1811
1812 if (likely(can_map_frag(frag)))
1813 return 0;
1814
1815 while (offset < remaining_in_skb && !can_map_frag(frag)) {
1816 offset += skb_frag_size(frag);
1817 ++frag;
1818 }
1819 return offset;
1820}
1821
0c3936d3
AR		 1822static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,
1823 struct tcp_zerocopy_receive *zc,
1824 struct sk_buff *skb, u32 offset)
1825{
1826 u32 frag_offset, partial_frag_remainder = 0;
1827 int mappable_offset;
1828 skb_frag_t *frag;
1829
1830 /* worst case: skip to next skb. try to improve on this case below */
1831 zc->recv_skip_hint = skb->len - offset;
1832
1833 /* Find the frag containing this offset (and how far into that frag) */
1834 frag = skb_advance_to_frag(skb, offset, &frag_offset);
1835 if (!frag)
1836 return;
1837
1838 if (frag_offset) {
1839 struct skb_shared_info *info = skb_shinfo(skb);
1840
1841 /* We read part of the last frag, must recvmsg() rest of skb. */
1842 if (frag == &info->frags[info->nr_frags - 1])
1843 return;
1844
1845 /* Else, we must at least read the remainder in this frag. */
1846 partial_frag_remainder = skb_frag_size(frag) - frag_offset;
1847 zc->recv_skip_hint -= partial_frag_remainder;
1848 ++frag;
1849 }
1850
1851 /* partial_frag_remainder: If part way through a frag, must read rest.
1852 * mappable_offset: Bytes till next mappable frag, *not* counting bytes
1853 * in partial_frag_remainder.
1854 */
1855 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);
1856 zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
1857}
1858
f21a3c48 1859static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
ec095263 1860 int flags, struct scm_timestamping_internal *tss,
f21a3c48
AR		 1861 int *cmsg_flags);
1862static int receive_fallback_to_copy(struct sock *sk,
7eeba170
AR		 1863 struct tcp_zerocopy_receive *zc, int inq,
1864 struct scm_timestamping_internal *tss)
f21a3c48
AR		 1865{
1866 unsigned long copy_address = (unsigned long)zc->copybuf_address;
f21a3c48 1867 struct msghdr msg = {};
7eeba170 1868 int err;
f21a3c48
AR		 1869
1870 zc->length = 0;
1871 zc->recv_skip_hint = 0;
1872
1873 if (copy_address != zc->copybuf_address)
1874 return -EINVAL;
1875
9fd7874c
JA		 1876 err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq,
1877 &msg.msg_iter);
f21a3c48
AR		 1878 if (err)
1879 return err;
1880
ec095263 1881 err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT,
7eeba170 1882 tss, &zc->msg_flags);
f21a3c48
AR		 1883 if (err < 0)
1884 return err;
1885
1886 zc->copybuf_len = err;
0c3936d3
AR		 1887 if (likely(zc->copybuf_len)) {
1888 struct sk_buff *skb;
1889 u32 offset;
1890
1891 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);
1892 if (skb)
1893 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
1894 }
f21a3c48
AR		 1895 return 0;
1896}
1897
18fb76ed
AR		 1898static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
1899 struct sk_buff *skb, u32 copylen,
1900 u32 *offset, u32 *seq)
1901{
1902 unsigned long copy_address = (unsigned long)zc->copybuf_address;
1903 struct msghdr msg = {};
18fb76ed
AR		 1904 int err;
1905
1906 if (copy_address != zc->copybuf_address)
1907 return -EINVAL;
1908
9fd7874c
JA		 1909 err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen,
1910 &msg.msg_iter);
18fb76ed
AR		 1911 if (err)
1912 return err;
1913 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen);
1914 if (err)
1915 return err;
1916 zc->recv_skip_hint -= copylen;
1917 *offset += copylen;
1918 *seq += copylen;
1919 return (__s32)copylen;
1920}
1921
7eeba170
AR		 1922static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
1923 struct sock *sk,
1924 struct sk_buff *skb,
1925 u32 *seq,
1926 s32 copybuf_len,
1927 struct scm_timestamping_internal *tss)
18fb76ed
AR		 1928{
1929 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
1930
1931 if (!copylen)
1932 return 0;
1933 /* skb is null if inq < PAGE_SIZE. */
7eeba170 1934 if (skb) {
18fb76ed 1935 offset = *seq - TCP_SKB_CB(skb)->seq;
7eeba170 1936 } else {
18fb76ed 1937 skb = tcp_recv_skb(sk, *seq, &offset);
7eeba170
AR		 1938 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1939 tcp_update_recv_tstamps(skb, tss);
1940 zc->msg_flags |= TCP_CMSG_TS;
1941 }
1942 }
18fb76ed
AR		 1943
1944 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
1945 seq);
1946 return zc->copybuf_len < 0 ? 0 : copylen;
1947}
1948
94ab9eb9
AR		 1949static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma,
1950 struct page **pending_pages,
1951 unsigned long pages_remaining,
1952 unsigned long *address,
1953 u32 *length,
1954 u32 *seq,
1955 struct tcp_zerocopy_receive *zc,
1956 u32 total_bytes_to_map,
1957 int err)
1958{
1959 /* At least one page did not map. Try zapping if we skipped earlier. */
1960 if (err == -EBUSY &&
1961 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
1962 u32 maybe_zap_len;
1963
1964 maybe_zap_len = total_bytes_to_map - /* All bytes to map */
1965 *length + /* Mapped or pending */
1966 (pages_remaining * PAGE_SIZE); /* Failed map. */
e9adcfec 1967 zap_page_range_single(vma, *address, maybe_zap_len, NULL);
94ab9eb9
AR		 1968 err = 0;
1969 }
1970
1971 if (!err) {
1972 unsigned long leftover_pages = pages_remaining;
1973 int bytes_mapped;
1974
e9adcfec 1975 /* We called zap_page_range_single, try to reinsert. */
94ab9eb9
AR		 1976 err = vm_insert_pages(vma, *address,
1977 pending_pages,
1978 &pages_remaining);
1979 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining);
1980 *seq += bytes_mapped;
1981 *address += bytes_mapped;
1982 }
1983 if (err) {
1984 /* Either we were unable to zap, OR we zapped, retried an
1985 * insert, and still had an issue. Either ways, pages_remaining
1986 * is the number of pages we were unable to map, and we unroll
1987 * some state we speculatively touched before.
1988 */
1989 const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
1990
1991 *length -= bytes_not_mapped;
1992 zc->recv_skip_hint += bytes_not_mapped;
1993 }
1994 return err;
1995}
1996
3763a24c
AR		 1997static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
1998 struct page **pages,
94ab9eb9
AR		 1999 unsigned int pages_to_map,
2000 unsigned long *address,
2001 u32 *length,
3763a24c 2002 u32 *seq,
94ab9eb9
AR		 2003 struct tcp_zerocopy_receive *zc,
2004 u32 total_bytes_to_map)
3763a24c
AR		 2005{
2006 unsigned long pages_remaining = pages_to_map;
94ab9eb9
AR		 2007 unsigned int pages_mapped;
2008 unsigned int bytes_mapped;
2009 int err;
3763a24c 2010
94ab9eb9
AR		 2011 err = vm_insert_pages(vma, *address, pages, &pages_remaining);
2012 pages_mapped = pages_to_map - (unsigned int)pages_remaining;
2013 bytes_mapped = PAGE_SIZE * pages_mapped;
3763a24c
AR		 2014 /* Even if vm_insert_pages fails, it may have partially succeeded in
2015 * mapping (some but not all of the pages).
2016 */
2017 *seq += bytes_mapped;
94ab9eb9
AR		 2018 *address += bytes_mapped;
2019
2020 if (likely(!err))
2021 return 0;
2022
2023 /* Error: maybe zap and retry + rollback state for failed inserts. */
2024 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped,
2025 pages_remaining, address, length, seq, zc, total_bytes_to_map,
2026 err);
3763a24c
AR		 2027}
2028
3c5a2fd0 2029#define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS)
7eeba170
AR		 2030static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
2031 struct tcp_zerocopy_receive *zc,
2032 struct scm_timestamping_internal *tss)
2033{
2034 unsigned long msg_control_addr;
2035 struct msghdr cmsg_dummy;
2036
2037 msg_control_addr = (unsigned long)zc->msg_control;
c39ef213 2038 cmsg_dummy.msg_control_user = (void __user *)msg_control_addr;
7eeba170
AR		 2039 cmsg_dummy.msg_controllen =
2040 (__kernel_size_t)zc->msg_controllen;
2041 cmsg_dummy.msg_flags = in_compat_syscall()
2042 ? MSG_CMSG_COMPAT : 0;
a6f8ee58 2043 cmsg_dummy.msg_control_is_user = true;
7eeba170
AR		 2044 zc->msg_flags = 0;
2045 if (zc->msg_control == msg_control_addr &&
2046 zc->msg_controllen == cmsg_dummy.msg_controllen) {
2047 tcp_recv_timestamp(&cmsg_dummy, sk, tss);
2048 zc->msg_control = (__u64)
c39ef213 2049 ((uintptr_t)cmsg_dummy.msg_control_user);
7eeba170
AR		 2050 zc->msg_controllen =
2051 (__u64)cmsg_dummy.msg_controllen;
2052 zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
2053 }
2054}
2055
7a7f0946
AR		 2056static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
2057 unsigned long address,
2058 bool *mmap_locked)
2059{
350f6bbc 2060 struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
7a7f0946 2061
7a7f0946 2062 if (vma) {
350f6bbc 2063 if (vma->vm_ops != &tcp_vm_ops) {
7a7f0946
AR		 2064 vma_end_read(vma);
2065 return NULL;
2066 }
2067 *mmap_locked = false;
2068 return vma;
2069 }
2070
2071 mmap_read_lock(mm);
2072 vma = vma_lookup(mm, address);
350f6bbc 2073 if (!vma || vma->vm_ops != &tcp_vm_ops) {
7a7f0946
AR		 2074 mmap_read_unlock(mm);
2075 return NULL;
2076 }
2077 *mmap_locked = true;
2078 return vma;
2079}
2080
94ab9eb9 2081#define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32
05255b82 2082static int tcp_zerocopy_receive(struct sock *sk,
7eeba170
AR		 2083 struct tcp_zerocopy_receive *zc,
2084 struct scm_timestamping_internal *tss)
05255b82 2085{
94ab9eb9 2086 u32 length = 0, offset, vma_len, avail_len, copylen = 0;
05255b82 2087 unsigned long address = (unsigned long)zc->address;
94ab9eb9 2088 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE];
18fb76ed
AR		 2089 s32 copybuf_len = zc->copybuf_len;
2090 struct tcp_sock *tp = tcp_sk(sk);
05255b82 2091 const skb_frag_t *frags = NULL;
94ab9eb9 2092 unsigned int pages_to_map = 0;
05255b82
ED		 2093 struct vm_area_struct *vma;
2094 struct sk_buff *skb = NULL;
18fb76ed 2095 u32 seq = tp->copied_seq;
94ab9eb9 2096 u32 total_bytes_to_map;
18fb76ed 2097 int inq = tcp_inq(sk);
7a7f0946 2098 bool mmap_locked;
93ab6cc6
ED		 2099 int ret;
2100
18fb76ed 2101 zc->copybuf_len = 0;
7eeba170 2102 zc->msg_flags = 0;
18fb76ed 2103
05255b82 2104 if (address & (PAGE_SIZE - 1) || address != zc->address)
93ab6cc6
ED		 2105 return -EINVAL;
2106
93ab6cc6 2107 if (sk->sk_state == TCP_LISTEN)
05255b82 2108 return -ENOTCONN;
93ab6cc6
ED		 2109
2110 sock_rps_record_flow(sk);
2111
f21a3c48 2112 if (inq && inq <= copybuf_len)
7eeba170 2113 return receive_fallback_to_copy(sk, zc, inq, tss);
f21a3c48 2114
936ced41
AR		 2115 if (inq < PAGE_SIZE) {
2116 zc->length = 0;
2117 zc->recv_skip_hint = inq;
2118 if (!inq && sock_flag(sk, SOCK_DONE))
2119 return -EIO;
2120 return 0;
2121 }
2122
7a7f0946
AR		 2123 vma = find_tcp_vma(current->mm, address, &mmap_locked);
2124 if (!vma)
e776af60 2125 return -EINVAL;
7a7f0946 2126
18fb76ed
AR		 2127 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
2128 avail_len = min_t(u32, vma_len, inq);
94ab9eb9
AR		 2129 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1);
2130 if (total_bytes_to_map) {
2131 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
e9adcfec
MK		 2132 zap_page_range_single(vma, address, total_bytes_to_map,
2133 NULL);
94ab9eb9 2134 zc->length = total_bytes_to_map;
8f2b0293
SHY		 2135 zc->recv_skip_hint = 0;
2136 } else {
18fb76ed
AR		 2137 zc->length = avail_len;
2138 zc->recv_skip_hint = avail_len;
8f2b0293 2139 }
05255b82
ED		 2140 ret = 0;
2141 while (length + PAGE_SIZE <= zc->length) {
98917cf0 2142 int mappable_offset;
94ab9eb9 2143 struct page *page;
98917cf0 2144
05255b82 2145 if (zc->recv_skip_hint < PAGE_SIZE) {
7fba5309
AR		 2146 u32 offset_frag;
2147
05255b82 2148 if (skb) {
0e627190
AR		 2149 if (zc->recv_skip_hint > 0)
2150 break;
05255b82
ED		 2151 skb = skb->next;
2152 offset = seq - TCP_SKB_CB(skb)->seq;
2153 } else {
2154 skb = tcp_recv_skb(sk, seq, &offset);
2155 }
7eeba170
AR		 2156
2157 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2158 tcp_update_recv_tstamps(skb, tss);
2159 zc->msg_flags |= TCP_CMSG_TS;
2160 }
05255b82 2161 zc->recv_skip_hint = skb->len - offset;
7fba5309
AR		 2162 frags = skb_advance_to_frag(skb, offset, &offset_frag);
2163 if (!frags || offset_frag)
05255b82 2164 break;
93ab6cc6 2165 }
789762ce 2166
98917cf0
AR		 2167 mappable_offset = find_next_mappable_frag(frags,
2168 zc->recv_skip_hint);
2169 if (mappable_offset) {
2170 zc->recv_skip_hint = mappable_offset;
05255b82 2171 break;
789762ce 2172 }
94ab9eb9
AR		 2173 page = skb_frag_page(frags);
2174 prefetchw(page);
2175 pages[pages_to_map++] = page;
05255b82 2176 length += PAGE_SIZE;
05255b82
ED		 2177 zc->recv_skip_hint -= PAGE_SIZE;
2178 frags++;
94ab9eb9
AR		 2179 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE ||
2180 zc->recv_skip_hint < PAGE_SIZE) {
2181 /* Either full batch, or we're about to go to next skb
2182 * (and we cannot unroll failed ops across skbs).
2183 */
2184 ret = tcp_zerocopy_vm_insert_batch(vma, pages,
2185 pages_to_map,
2186 &address, &length,
2187 &seq, zc,
2188 total_bytes_to_map);
3763a24c
AR		 2189 if (ret)
2190 goto out;
94ab9eb9 2191 pages_to_map = 0;
3763a24c
AR		 2192 }
2193 }
94ab9eb9
AR		 2194 if (pages_to_map) {
2195 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map,
2196 &address, &length, &seq,
2197 zc, total_bytes_to_map);
93ab6cc6 2198 }
93ab6cc6 2199out:
7a7f0946
AR		 2200 if (mmap_locked)
2201 mmap_read_unlock(current->mm);
2202 else
2203 vma_end_read(vma);
18fb76ed
AR		 2204 /* Try to copy straggler data. */
2205 if (!ret)
7eeba170 2206 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
18fb76ed
AR		 2207
2208 if (length + copylen) {
7db48e98 2209 WRITE_ONCE(tp->copied_seq, seq);
05255b82
ED		 2210 tcp_rcv_space_adjust(sk);
2211
2212 /* Clean up data we have read: This will do ACK frames. */
2213 tcp_recv_skb(sk, seq, &offset);
18fb76ed 2214 tcp_cleanup_rbuf(sk, length + copylen);
05255b82
ED		 2215 ret = 0;
2216 if (length == zc->length)
2217 zc->recv_skip_hint = 0;
2218 } else {
2219 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
2220 ret = -EIO;
2221 }
2222 zc->length = length;
93ab6cc6
ED		 2223 return ret;
2224}
05255b82 2225#endif
93ab6cc6 2226
98aaa913 2227/* Similar to __sock_recv_timestamp, but does not require an skb */
892bfd3d
FW		 2228void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
2229 struct scm_timestamping_internal *tss)
98aaa913 2230{
887feae3 2231 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
98aaa913
MM		 2232 bool has_timestamping = false;
2233
2234 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
2235 if (sock_flag(sk, SOCK_RCVTSTAMP)) {
2236 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
887feae3 2237 if (new_tstamp) {
df1b4ba9
AB		 2238 struct __kernel_timespec kts = {
2239 .tv_sec = tss->ts[0].tv_sec,
2240 .tv_nsec = tss->ts[0].tv_nsec,
2241 };
887feae3
DD		 2242 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
2243 sizeof(kts), &kts);
2244 } else {
df1b4ba9
AB		 2245 struct __kernel_old_timespec ts_old = {
2246 .tv_sec = tss->ts[0].tv_sec,
2247 .tv_nsec = tss->ts[0].tv_nsec,
2248 };
887feae3 2249 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
9718475e 2250 sizeof(ts_old), &ts_old);
887feae3 2251 }
98aaa913 2252 } else {
887feae3 2253 if (new_tstamp) {
df1b4ba9
AB		 2254 struct __kernel_sock_timeval stv = {
2255 .tv_sec = tss->ts[0].tv_sec,
2256 .tv_usec = tss->ts[0].tv_nsec / 1000,
2257 };
887feae3
DD		 2258 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
2259 sizeof(stv), &stv);
2260 } else {
df1b4ba9
AB		 2261 struct __kernel_old_timeval tv = {
2262 .tv_sec = tss->ts[0].tv_sec,
2263 .tv_usec = tss->ts[0].tv_nsec / 1000,
2264 };
887feae3
DD		 2265 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
2266 sizeof(tv), &tv);
2267 }
98aaa913
MM		 2268 }
2269 }
2270
e3390b30 2271 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_SOFTWARE)
98aaa913
MM		 2272 has_timestamping = true;
2273 else
9718475e 2274 tss->ts[0] = (struct timespec64) {0};
98aaa913
MM		 2275 }
2276
2277 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
e3390b30 2278 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_RAW_HARDWARE)
98aaa913
MM		 2279 has_timestamping = true;
2280 else
9718475e 2281 tss->ts[2] = (struct timespec64) {0};
98aaa913
MM		 2282 }
2283
2284 if (has_timestamping) {
9718475e
DD		 2285 tss->ts[1] = (struct timespec64) {0};
2286 if (sock_flag(sk, SOCK_TSTAMP_NEW))
2287 put_cmsg_scm_timestamping64(msg, tss);
2288 else
2289 put_cmsg_scm_timestamping(msg, tss);
98aaa913
MM		 2290 }
2291}
2292
b75eba76
SHY		 2293static int tcp_inq_hint(struct sock *sk)
2294{
2295 const struct tcp_sock *tp = tcp_sk(sk);
2296 u32 copied_seq = READ_ONCE(tp->copied_seq);
2297 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
2298 int inq;
2299
2300 inq = rcv_nxt - copied_seq;
2301 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
2302 lock_sock(sk);
2303 inq = tp->rcv_nxt - tp->copied_seq;
2304 release_sock(sk);
2305 }
6466e715
SHY		 2306 /* After receiving a FIN, tell the user-space to continue reading
2307 * by returning a non-zero inq.
2308 */
2309 if (inq == 0 && sock_flag(sk, SOCK_DONE))
2310 inq = 1;
b75eba76
SHY		 2311 return inq;
2312}
2313
1da177e4
LT		 2314/*
2315 * This routine copies from a sock struct into the user buffer.
2316 *
2317 * Technical note: in 2.3 we work on _locked_ socket, so that
2318 * tricks with *seq access order and skb->users are not required.
2319 * Probably, code can be easily improved even more.
2320 */
2321
2cd81161 2322static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
ec095263 2323 int flags, struct scm_timestamping_internal *tss,
2cd81161 2324 int *cmsg_flags)
1da177e4
LT		 2325{
2326 struct tcp_sock *tp = tcp_sk(sk);
2327 int copied = 0;
2328 u32 peek_seq;
2329 u32 *seq;
2330 unsigned long used;
2cd81161 2331 int err;
1da177e4
LT		 2332 int target; /* Read at least this many bytes */
2333 long timeo;
dfbafc99 2334 struct sk_buff *skb, *last;
05ea4916 2335 u32 peek_offset = 0;
77527313 2336 u32 urg_hole = 0;
1da177e4 2337
1da177e4
LT		 2338 err = -ENOTCONN;
2339 if (sk->sk_state == TCP_LISTEN)
2340 goto out;
2341
f94fd25c 2342 if (tp->recvmsg_inq) {
925bba24 2343 *cmsg_flags = TCP_CMSG_INQ;
f94fd25c
JA		 2344 msg->msg_get_inq = 1;
2345 }
ec095263 2346 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1da177e4
LT		 2347
2348 /* Urgent data needs to be handled specially. */
2349 if (flags & MSG_OOB)
2350 goto recv_urg;
2351
c0e88ff0
PE		 2352 if (unlikely(tp->repair)) {
2353 err = -EPERM;
2354 if (!(flags & MSG_PEEK))
2355 goto out;
2356
2357 if (tp->repair_queue == TCP_SEND_QUEUE)
2358 goto recv_sndq;
2359
2360 err = -EINVAL;
2361 if (tp->repair_queue == TCP_NO_QUEUE)
2362 goto out;
2363
2364 /* 'common' recv queue MSG_PEEK-ing */
2365 }
2366
1da177e4
LT		 2367 seq = &tp->copied_seq;
2368 if (flags & MSG_PEEK) {
05ea4916
JM		 2369 peek_offset = max(sk_peek_offset(sk, flags), 0);
2370 peek_seq = tp->copied_seq + peek_offset;
1da177e4
LT		 2371 seq = &peek_seq;
2372 }
2373
2374 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2375
2376 do {
1da177e4
LT		 2377 u32 offset;
2378
2379 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
b96c51bd 2380 if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
1da177e4
LT		 2381 if (copied)
2382 break;
2383 if (signal_pending(current)) {
2384 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2385 break;
2386 }
2387 }
2388
2389 /* Next get a buffer. */
2390
dfbafc99 2391 last = skb_peek_tail(&sk->sk_receive_queue);
91521944 2392 skb_queue_walk(&sk->sk_receive_queue, skb) {
dfbafc99 2393 last = skb;
1da177e4
LT		 2394 /* Now that we have two receive queues this
2395 * shouldn't happen.
2396 */
d792c100 2397 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
e56b8ce3 2398 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2af6fd8b
JP		 2399 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2400 flags))
1da177e4 2401 break;
d792c100 2402
1da177e4 2403 offset = *seq - TCP_SKB_CB(skb)->seq;
9d691539
ED		 2404 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2405 pr_err_once("%s: found a SYN, please report !\n", __func__);
1da177e4 2406 offset--;
9d691539 2407 }
1da177e4
LT		 2408 if (offset < skb->len)
2409 goto found_ok_skb;
e11ecddf 2410 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1da177e4 2411 goto found_fin_ok;
2af6fd8b 2412 WARN(!(flags & MSG_PEEK),
e56b8ce3 2413 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2af6fd8b 2414 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
91521944 2415 }
1da177e4
LT		 2416
2417 /* Well, if we have backlog, try to process it now yet. */
2418
9ed498c6 2419 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
1da177e4
LT		 2420 break;
2421
2422 if (copied) {
8bd172b7
ED		 2423 if (!timeo ||
2424 sk->sk_err ||
1da177e4
LT		 2425 sk->sk_state == TCP_CLOSE ||
2426 (sk->sk_shutdown & RCV_SHUTDOWN) ||
518a09ef 2427 signal_pending(current))
1da177e4
LT		 2428 break;
2429 } else {
2430 if (sock_flag(sk, SOCK_DONE))
2431 break;
2432
2433 if (sk->sk_err) {
2434 copied = sock_error(sk);
2435 break;
2436 }
2437
2438 if (sk->sk_shutdown & RCV_SHUTDOWN)
2439 break;
2440
2441 if (sk->sk_state == TCP_CLOSE) {
c47078d6
ED		 2442 /* This occurs when user tries to read
2443 * from never connected socket.
2444 */
2445 copied = -ENOTCONN;
1da177e4
LT		 2446 break;
2447 }
2448
2449 if (!timeo) {
2450 copied = -EAGAIN;
2451 break;
2452 }
2453
2454 if (signal_pending(current)) {
2455 copied = sock_intr_errno(timeo);
2456 break;
2457 }
2458 }
2459
1da177e4
LT		 2460 if (copied >= target) {
2461 /* Do not sleep, just process backlog. */
93afcfd1 2462 __sk_flush_backlog(sk);
dfbafc99 2463 } else {
29fbc26e 2464 tcp_cleanup_rbuf(sk, copied);
419ce133
PA		 2465 err = sk_wait_data(sk, &timeo, last);
2466 if (err < 0) {
2467 err = copied ? : err;
2468 goto out;
2469 }
dfbafc99 2470 }
1da177e4 2471
77527313 2472 if ((flags & MSG_PEEK) &&
05ea4916 2473 (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {
e87cc472
JP		 2474 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2475 current->comm,
2476 task_pid_nr(current));
05ea4916 2477 peek_seq = tp->copied_seq + peek_offset;
1da177e4
LT		 2478 }
2479 continue;
2480
fdb8b298 2481found_ok_skb:
1da177e4
LT		 2482 /* Ok so how much can we use? */
2483 used = skb->len - offset;
2484 if (len < used)
2485 used = len;
2486
2487 /* Do we have urgent data here? */
b96c51bd 2488 if (unlikely(tp->urg_data)) {
1da177e4
LT		 2489 u32 urg_offset = tp->urg_seq - *seq;
2490 if (urg_offset < used) {
2491 if (!urg_offset) {
2492 if (!sock_flag(sk, SOCK_URGINLINE)) {
7db48e98 2493 WRITE_ONCE(*seq, *seq + 1);
77527313 2494 urg_hole++;
1da177e4
LT		 2495 offset++;
2496 used--;
2497 if (!used)
2498 goto skip_copy;
2499 }
2500 } else
2501 used = urg_offset;
2502 }
2503 }
2504
2505 if (!(flags & MSG_TRUNC)) {
51f3d02b 2506 err = skb_copy_datagram_msg(skb, offset, msg, used);
7bced397
DW		 2507 if (err) {
2508 /* Exception. Bailout! */
2509 if (!copied)
2510 copied = -EFAULT;
2511 break;
1da177e4
LT		 2512 }
2513 }
2514
7db48e98 2515 WRITE_ONCE(*seq, *seq + used);
1da177e4
LT		 2516 copied += used;
2517 len -= used;
05ea4916
JM		 2518 if (flags & MSG_PEEK)
2519 sk_peek_offset_fwd(sk, used);
2520 else
2521 sk_peek_offset_bwd(sk, used);
1da177e4
LT		 2522 tcp_rcv_space_adjust(sk);
2523
2524skip_copy:
b96c51bd 2525 if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
7b6a893a 2526 WRITE_ONCE(tp->urg_data, 0);
31770e34
FW		 2527 tcp_fast_path_check(sk);
2528 }
1da177e4 2529
98aaa913 2530 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2cd81161 2531 tcp_update_recv_tstamps(skb, tss);
925bba24 2532 *cmsg_flags |= TCP_CMSG_TS;
98aaa913 2533 }
cc4de047
KL		 2534
2535 if (used + offset < skb->len)
2536 continue;
2537
e11ecddf 2538 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1da177e4 2539 goto found_fin_ok;
7bced397 2540 if (!(flags & MSG_PEEK))
3df684c1 2541 tcp_eat_recv_skb(sk, skb);
1da177e4
LT		 2542 continue;
2543
fdb8b298 2544found_fin_ok:
1da177e4 2545 /* Process the FIN. */
7db48e98 2546 WRITE_ONCE(*seq, *seq + 1);
7bced397 2547 if (!(flags & MSG_PEEK))
3df684c1 2548 tcp_eat_recv_skb(sk, skb);
1da177e4
LT		 2549 break;
2550 } while (len > 0);
2551
1da177e4
LT		 2552 /* According to UNIX98, msg_name/msg_namelen are ignored
2553 * on connected socket. I was just happy when found this 8) --ANK
2554 */
2555
2556 /* Clean up data we have read: This will do ACK frames. */
0e4b4992 2557 tcp_cleanup_rbuf(sk, copied);
1da177e4
LT		 2558 return copied;
2559
2560out:
1da177e4
LT		 2561 return err;
2562
2563recv_urg:
377f0a08 2564 err = tcp_recv_urg(sk, msg, len, flags);
1da177e4 2565 goto out;
c0e88ff0
PE		 2566
2567recv_sndq:
2568 err = tcp_peek_sndq(sk, msg, len);
2569 goto out;
1da177e4 2570}
2cd81161 2571
ec095263
OH		 2572int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
2573 int *addr_len)
2cd81161 2574{
f94fd25c 2575 int cmsg_flags = 0, ret;
2cd81161
AR		 2576 struct scm_timestamping_internal tss;
2577
2578 if (unlikely(flags & MSG_ERRQUEUE))
2579 return inet_recv_error(sk, msg, len, addr_len);
2580
2581 if (sk_can_busy_loop(sk) &&
2582 skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2583 sk->sk_state == TCP_ESTABLISHED)
ec095263 2584 sk_busy_loop(sk, flags & MSG_DONTWAIT);
2cd81161
AR		 2585
2586 lock_sock(sk);
ec095263 2587 ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags);
2cd81161
AR		 2588 release_sock(sk);
2589
f94fd25c 2590 if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) {
925bba24 2591 if (cmsg_flags & TCP_CMSG_TS)
2cd81161 2592 tcp_recv_timestamp(msg, sk, &tss);
f94fd25c
JA		 2593 if (msg->msg_get_inq) {
2594 msg->msg_inq = tcp_inq_hint(sk);
2595 if (cmsg_flags & TCP_CMSG_INQ)
2596 put_cmsg(msg, SOL_TCP, TCP_CM_INQ,
2597 sizeof(msg->msg_inq), &msg->msg_inq);
2cd81161
AR		 2598 }
2599 }
2600 return ret;
2601}
4bc2f18b 2602EXPORT_SYMBOL(tcp_recvmsg);
1da177e4 2603
490d5046
IJ		 2604void tcp_set_state(struct sock *sk, int state)
2605{
2606 int oldstate = sk->sk_state;
2607
d4487491
LB		 2608 /* We defined a new enum for TCP states that are exported in BPF
2609 * so as not force the internal TCP states to be frozen. The
2610 * following checks will detect if an internal state value ever
2611 * differs from the BPF value. If this ever happens, then we will
2612 * need to remap the internal value to the BPF value before calling
2613 * tcp_call_bpf_2arg.
2614 */
2615 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2616 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2617 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2618 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2619 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2620 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2621 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2622 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2623 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2624 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2625 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2626 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
91051f00 2627 BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE);
d4487491
LB		 2628 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2629
97a19caf
YS		 2630 /* bpf uapi header bpf.h defines an anonymous enum with values
2631 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux
2632 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON.
2633 * But clang built vmlinux does not have this enum in DWARF
2634 * since clang removes the above code before generating IR/debuginfo.
2635 * Let us explicitly emit the type debuginfo to ensure the
2636 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF
2637 * regardless of which compiler is used.
2638 */
2639 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
2640
d4487491
LB		 2641 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2642 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
e8fce239 2643
490d5046
IJ		 2644 switch (state) {
2645 case TCP_ESTABLISHED:
2646 if (oldstate != TCP_ESTABLISHED)
81cc8a75 2647 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
490d5046
IJ		 2648 break;
2649
2650 case TCP_CLOSE:
2651 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
81cc8a75 2652 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
490d5046
IJ		 2653
2654 sk->sk_prot->unhash(sk);
2655 if (inet_csk(sk)->icsk_bind_hash &&
2656 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
ab1e0a13 2657 inet_put_port(sk);
a8eceea8 2658 fallthrough;
490d5046 2659 default:
5a5f3a8d 2660 if (oldstate == TCP_ESTABLISHED)
74688e48 2661 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
490d5046
IJ		 2662 }
2663
2664 /* Change state AFTER socket is unhashed to avoid closed
2665 * socket sitting in hash tables.
2666 */
563e0bb0 2667 inet_sk_state_store(sk, state);
490d5046
IJ		 2668}
2669EXPORT_SYMBOL_GPL(tcp_set_state);
2670
1da177e4
LT		 2671/*
2672 * State processing on a close. This implements the state shift for
2673 * sending our FIN frame. Note that we only send a FIN for some
2674 * states. A shutdown() may have already sent the FIN, or we may be
2675 * closed.
2676 */
2677
9b5b5cff 2678static const unsigned char new_state[16] = {
1da177e4 2679 /* current state: new state: action: */
0980c1e3
ED		 2680 [0 /* (Invalid) */] = TCP_CLOSE,
2681 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2682 [TCP_SYN_SENT] = TCP_CLOSE,
2683 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2684 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
2685 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
2686 [TCP_TIME_WAIT] = TCP_CLOSE,
2687 [TCP_CLOSE] = TCP_CLOSE,
2688 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
2689 [TCP_LAST_ACK] = TCP_LAST_ACK,
2690 [TCP_LISTEN] = TCP_CLOSE,
2691 [TCP_CLOSING] = TCP_CLOSING,
2692 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
1da177e4
LT		 2693};
2694
2695static int tcp_close_state(struct sock *sk)
2696{
2697 int next = (int)new_state[sk->sk_state];
2698 int ns = next & TCP_STATE_MASK;
2699
2700 tcp_set_state(sk, ns);
2701
2702 return next & TCP_ACTION_FIN;
2703}
2704
2705/*
2706 * Shutdown the sending side of a connection. Much like close except
1f29b058 2707 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1da177e4
LT		 2708 */
2709
2710void tcp_shutdown(struct sock *sk, int how)
2711{
2712 /* We need to grab some memory, and put together a FIN,
2713 * and then put it into the queue to be sent.
2714 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2715 */
2716 if (!(how & SEND_SHUTDOWN))
2717 return;
2718
2719 /* If we've already sent a FIN, or it's a closed state, skip this. */
2720 if ((1 << sk->sk_state) &
2721 (TCPF_ESTABLISHED | TCPF_SYN_SENT |
94062790 2722 TCPF_CLOSE_WAIT)) {
1da177e4
LT		 2723 /* Clear out any half completed packets. FIN if needed. */
2724 if (tcp_close_state(sk))
2725 tcp_send_fin(sk);
2726 }
2727}
4bc2f18b 2728EXPORT_SYMBOL(tcp_shutdown);
1da177e4 2729
19757ceb
ED		 2730int tcp_orphan_count_sum(void)
2731{
2732 int i, total = 0;
2733
2734 for_each_possible_cpu(i)
2735 total += per_cpu(tcp_orphan_count, i);
2736
2737 return max(total, 0);
2738}
2739
2740static int tcp_orphan_cache;
2741static struct timer_list tcp_orphan_timer;
2742#define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
2743
2744static void tcp_orphan_update(struct timer_list *unused)
2745{
2746 WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
2747 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
2748}
2749
2750static bool tcp_too_many_orphans(int shift)
2751{
47e6ab24
KI		 2752 return READ_ONCE(tcp_orphan_cache) << shift >
2753 READ_ONCE(sysctl_tcp_max_orphans);
19757ceb
ED		 2754}
2755
dda4d96a
ED		 2756static bool tcp_out_of_memory(const struct sock *sk)
2757{
2758 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
2759 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
2760 return true;
2761 return false;
2762}
2763
2764bool tcp_check_oom(const struct sock *sk, int shift)
efcdbf24
AS		 2765{
2766 bool too_many_orphans, out_of_socket_memory;
2767
19757ceb 2768 too_many_orphans = tcp_too_many_orphans(shift);
efcdbf24
AS		 2769 out_of_socket_memory = tcp_out_of_memory(sk);
2770
e87cc472
JP		 2771 if (too_many_orphans)
2772 net_info_ratelimited("too many orphaned sockets\n");
2773 if (out_of_socket_memory)
2774 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
efcdbf24
AS		 2775 return too_many_orphans || out_of_socket_memory;
2776}
2777
77c3c956 2778void __tcp_close(struct sock *sk, long timeout)
1da177e4
LT		 2779{
2780 struct sk_buff *skb;
2781 int data_was_unread = 0;
75c2d907 2782 int state;
1da177e4 2783
e14cadfd 2784 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
1da177e4
LT		 2785
2786 if (sk->sk_state == TCP_LISTEN) {
2787 tcp_set_state(sk, TCP_CLOSE);
2788
2789 /* Special case. */
0a5578cf 2790 inet_csk_listen_stop(sk);
1da177e4
LT		 2791
2792 goto adjudge_to_death;
2793 }
2794
2795 /* We need to flush the recv. buffs. We do this only on the
2796 * descriptor close, not protocol-sourced closes, because the
2797 * reader process may not have drained the data yet!
2798 */
2799 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
e11ecddf
ED		 2800 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2801
2802 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2803 len--;
1da177e4
LT		 2804 data_was_unread += len;
2805 __kfree_skb(skb);
2806 }
2807
565b7b2d
KK		 2808 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2809 if (sk->sk_state == TCP_CLOSE)
2810 goto adjudge_to_death;
2811
65bb723c
GR		 2812 /* As outlined in RFC 2525, section 2.17, we send a RST here because
2813 * data was lost. To witness the awful effects of the old behavior of
2814 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2815 * GET in an FTP client, suspend the process, wait for the client to
2816 * advertise a zero window, then kill -9 the FTP client, wheee...
2817 * Note: timeout is always zero in such a case.
1da177e4 2818 */
ee995283
PE		 2819 if (unlikely(tcp_sk(sk)->repair)) {
2820 sk->sk_prot->disconnect(sk, 0);
2821 } else if (data_was_unread) {
1da177e4 2822 /* Unread data was tossed, zap the connection. */
6aef70a8 2823 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
1da177e4 2824 tcp_set_state(sk, TCP_CLOSE);
5691276b
JX		 2825 tcp_send_active_reset(sk, sk->sk_allocation,
2826 SK_RST_REASON_NOT_SPECIFIED);
1da177e4
LT		 2827 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2828 /* Check zero linger _after_ checking for unread data. */
2829 sk->sk_prot->disconnect(sk, 0);
6aef70a8 2830 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
1da177e4
LT		 2831 } else if (tcp_close_state(sk)) {
2832 /* We FIN if the application ate all the data before
2833 * zapping the connection.
2834 */
2835
2836 /* RED-PEN. Formally speaking, we have broken TCP state
2837 * machine. State transitions:
2838 *
2839 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
94062790 2840 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult)
1da177e4
LT		 2841 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2842 *
2843 * are legal only when FIN has been sent (i.e. in window),
2844 * rather than queued out of window. Purists blame.
2845 *
2846 * F.e. "RFC state" is ESTABLISHED,
2847 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2848 *
2849 * The visible declinations are that sometimes
2850 * we enter time-wait state, when it is not required really
2851 * (harmless), do not send active resets, when they are
2852 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2853 * they look as CLOSING or LAST_ACK for Linux)
2854 * Probably, I missed some more holelets.
2855 * --ANK
8336886f
JC		 2856 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2857 * in a single packet! (May consider it later but will
2858 * probably need API support or TCP_CORK SYN-ACK until
2859 * data is written and socket is closed.)
1da177e4
LT		 2860 */
2861 tcp_send_fin(sk);
2862 }
2863
2864 sk_stream_wait_close(sk, timeout);
2865
2866adjudge_to_death:
75c2d907
HX		 2867 state = sk->sk_state;
2868 sock_hold(sk);
2869 sock_orphan(sk);
75c2d907 2870
1da177e4
LT		 2871 local_bh_disable();
2872 bh_lock_sock(sk);
8873c064
ED		 2873 /* remove backlog if any, without releasing ownership. */
2874 __release_sock(sk);
1da177e4 2875
19757ceb 2876 this_cpu_inc(tcp_orphan_count);
eb4dea58 2877
75c2d907
HX		 2878 /* Have we already been destroyed by a softirq or backlog? */
2879 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2880 goto out;
1da177e4
LT		 2881
2882 /* This is a (useful) BSD violating of the RFC. There is a
2883 * problem with TCP as specified in that the other end could
2884 * keep a socket open forever with no application left this end.
b10bd54c 2885 * We use a 1 minute timeout (about the same as BSD) then kill
1da177e4
LT		 2886 * our end. If they send after that then tough - BUT: long enough
2887 * that we won't make the old 4*rto = almost no time - whoops
2888 * reset mistake.
2889 *
2890 * Nope, it was not mistake. It is really desired behaviour
2891 * f.e. on http servers, when such sockets are useless, but
2892 * consume significant resources. Let's do it with special
2893 * linger2 option. --ANK
2894 */
2895
2896 if (sk->sk_state == TCP_FIN_WAIT2) {
2897 struct tcp_sock *tp = tcp_sk(sk);
a81722dd 2898 if (READ_ONCE(tp->linger2) < 0) {
1da177e4 2899 tcp_set_state(sk, TCP_CLOSE);
5691276b
JX		 2900 tcp_send_active_reset(sk, GFP_ATOMIC,
2901 SK_RST_REASON_NOT_SPECIFIED);
02a1d6e7 2902 __NET_INC_STATS(sock_net(sk),
de0744af 2903 LINUX_MIB_TCPABORTONLINGER);
1da177e4 2904 } else {
463c84b9 2905 const int tmo = tcp_fin_time(sk);
1da177e4
LT		 2906
2907 if (tmo > TCP_TIMEWAIT_LEN) {
52499afe
DM		 2908 inet_csk_reset_keepalive_timer(sk,
2909 tmo - TCP_TIMEWAIT_LEN);
1da177e4 2910 } else {
1da177e4
LT		 2911 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2912 goto out;
2913 }
2914 }
2915 }
2916 if (sk->sk_state != TCP_CLOSE) {
efcdbf24 2917 if (tcp_check_oom(sk, 0)) {
1da177e4 2918 tcp_set_state(sk, TCP_CLOSE);
5691276b
JX		 2919 tcp_send_active_reset(sk, GFP_ATOMIC,
2920 SK_RST_REASON_NOT_SPECIFIED);
02a1d6e7 2921 __NET_INC_STATS(sock_net(sk),
de0744af 2922 LINUX_MIB_TCPABORTONMEMORY);
4ee806d5
DS		 2923 } else if (!check_net(sock_net(sk))) {
2924 /* Not possible to send reset; just close */
2925 tcp_set_state(sk, TCP_CLOSE);
1da177e4
LT		 2926 }
2927 }
1da177e4 2928
8336886f 2929 if (sk->sk_state == TCP_CLOSE) {
d983ea6f
ED		 2930 struct request_sock *req;
2931
2932 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
2933 lockdep_sock_is_held(sk));
8336886f
JC		 2934 /* We could get here with a non-NULL req if the socket is
2935 * aborted (e.g., closed with unread data) before 3WHS
2936 * finishes.
2937 */
00db4124 2938 if (req)
8336886f 2939 reqsk_fastopen_remove(sk, req, false);
0a5578cf 2940 inet_csk_destroy_sock(sk);
8336886f 2941 }
1da177e4
LT		 2942 /* Otherwise, socket is reprieved until protocol close. */
2943
2944out:
2945 bh_unlock_sock(sk);
2946 local_bh_enable();
77c3c956
PA		 2947}
2948
2949void tcp_close(struct sock *sk, long timeout)
2950{
2951 lock_sock(sk);
2952 __tcp_close(sk, timeout);
8873c064 2953 release_sock(sk);
151c9c72
ED		 2954 if (!sk->sk_net_refcnt)
2955 inet_csk_clear_xmit_timers_sync(sk);
1da177e4
LT		 2956 sock_put(sk);
2957}
4bc2f18b 2958EXPORT_SYMBOL(tcp_close);
1da177e4
LT		 2959
2960/* These states need RST on ABORT according to RFC793 */
2961
a2a385d6 2962static inline bool tcp_need_reset(int state)
1da177e4
LT		 2963{
2964 return (1 << state) &
2965 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
a7150e38 2966 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1da177e4
LT		 2967}
2968
75c119af
ED		 2969static void tcp_rtx_queue_purge(struct sock *sk)
2970{
2971 struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
2972
2bec445f 2973 tcp_sk(sk)->highest_sack = NULL;
75c119af
ED		 2974 while (p) {
2975 struct sk_buff *skb = rb_to_skb(p);
2976
2977 p = rb_next(p);
2978 /* Since we are deleting whole queue, no need to
2979 * list_del(&skb->tcp_tsorted_anchor)
2980 */
2981 tcp_rtx_queue_unlink(skb, sk);
03271f3a 2982 tcp_wmem_free_skb(sk, skb);
75c119af
ED		 2983 }
2984}
2985
ac3f09ba
ED		 2986void tcp_write_queue_purge(struct sock *sk)
2987{
2988 struct sk_buff *skb;
2989
2990 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
2991 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
2992 tcp_skb_tsorted_anchor_cleanup(skb);
03271f3a 2993 tcp_wmem_free_skb(sk, skb);
ac3f09ba 2994 }
75c119af 2995 tcp_rtx_queue_purge(sk);
ac3f09ba 2996 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
ac3f09ba 2997 tcp_clear_all_retrans_hints(tcp_sk(sk));
bffd168c 2998 tcp_sk(sk)->packets_out = 0;
04c03114 2999 inet_csk(sk)->icsk_backoff = 0;
ac3f09ba
ED		 3000}
3001
1da177e4
LT		 3002int tcp_disconnect(struct sock *sk, int flags)
3003{
3004 struct inet_sock *inet = inet_sk(sk);
463c84b9 3005 struct inet_connection_sock *icsk = inet_csk(sk);
1da177e4 3006 struct tcp_sock *tp = tcp_sk(sk);
1da177e4 3007 int old_state = sk->sk_state;
0f317464 3008 u32 seq;
1da177e4
LT		 3009
3010 if (old_state != TCP_CLOSE)
3011 tcp_set_state(sk, TCP_CLOSE);
3012
3013 /* ABORT function of RFC793 */
3014 if (old_state == TCP_LISTEN) {
0a5578cf 3015 inet_csk_listen_stop(sk);
ee995283 3016 } else if (unlikely(tp->repair)) {
e13ec3da 3017 WRITE_ONCE(sk->sk_err, ECONNABORTED);
1da177e4
LT		 3018 } else if (tcp_need_reset(old_state) ||
3019 (tp->snd_nxt != tp->write_seq &&
3020 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
caa20d9a 3021 /* The last check adjusts for discrepancy of Linux wrt. RFC
1da177e4
LT		 3022 * states
3023 */
5691276b 3024 tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_NOT_SPECIFIED);
e13ec3da 3025 WRITE_ONCE(sk->sk_err, ECONNRESET);
a7150e38 3026 } else if (old_state == TCP_SYN_SENT)
e13ec3da 3027 WRITE_ONCE(sk->sk_err, ECONNRESET);
1da177e4
LT		 3028
3029 tcp_clear_xmit_timers(sk);
3030 __skb_queue_purge(&sk->sk_receive_queue);
7db48e98 3031 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
7b6a893a 3032 WRITE_ONCE(tp->urg_data, 0);
05ea4916 3033 sk_set_peek_off(sk, -1);
fe067e8a 3034 tcp_write_queue_purge(sk);
cf1ef3f0 3035 tcp_fastopen_active_disable_ofo_check(sk);
9f5afeae 3036 skb_rbtree_purge(&tp->out_of_order_queue);
1da177e4 3037
c720c7e8 3038 inet->inet_dport = 0;
1da177e4 3039
e0833d1f 3040 inet_bhash2_reset_saddr(sk);
1da177e4 3041
e14cadfd 3042 WRITE_ONCE(sk->sk_shutdown, 0);
1da177e4 3043 sock_reset_flag(sk, SOCK_DONE);
740b0f18 3044 tp->srtt_us = 0;
b9e2e689 3045 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
3f6c65d6 3046 tp->rcv_rtt_last_tsecr = 0;
0f317464
ED		 3047
3048 seq = tp->write_seq + tp->max_window + 2;
3049 if (!seq)
3050 seq = 1;
3051 WRITE_ONCE(tp->write_seq, seq);
3052
463c84b9 3053 icsk->icsk_backoff = 0;
6687e988 3054 icsk->icsk_probes_out = 0;
9d9b1ee0 3055 icsk->icsk_probes_tstamp = 0;
6a408147 3056 icsk->icsk_rto = TCP_TIMEOUT_INIT;
ca584ba0 3057 icsk->icsk_rto_min = TCP_RTO_MIN;
2b8ee4f0 3058 icsk->icsk_delack_max = TCP_DELACK_MAX;
0b6a05c1 3059 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
40570375 3060 tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
1da177e4 3061 tp->snd_cwnd_cnt = 0;
f4ce91ce
NC		 3062 tp->is_cwnd_limited = 0;
3063 tp->max_packets_out = 0;
1fdf475a 3064 tp->window_clamp = 0;
2fbdd562 3065 tp->delivered = 0;
e21db6f6 3066 tp->delivered_ce = 0;
ce69e563
CP		 3067 if (icsk->icsk_ca_ops->release)
3068 icsk->icsk_ca_ops->release(sk);
3069 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
8919a9b3 3070 icsk->icsk_ca_initialized = 0;
6687e988 3071 tcp_set_ca_state(sk, TCP_CA_Open);
d4761754 3072 tp->is_sack_reneg = 0;
1da177e4 3073 tcp_clear_retrans(tp);
c13c48c0 3074 tp->total_retrans = 0;
463c84b9 3075 inet_csk_delack_init(sk);
499350a5
WW		 3076 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
3077 * issue in __tcp_select_window()
3078 */
3079 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
b40b4f79 3080 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
1da177e4 3081 __sk_dst_reset(sk);
8f905c0e 3082 dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL));
17c3060b 3083 tcp_saved_syn_free(tp);
5d9f4262 3084 tp->compressed_ack = 0;
784f8344
ED		 3085 tp->segs_in = 0;
3086 tp->segs_out = 0;
ba113c3a 3087 tp->bytes_sent = 0;
e858faf5
CP		 3088 tp->bytes_acked = 0;
3089 tp->bytes_received = 0;
fb31c9b9 3090 tp->bytes_retrans = 0;
db7ffee6
ED		 3091 tp->data_segs_in = 0;
3092 tp->data_segs_out = 0;
7788174e
YC		 3093 tp->duplicate_sack[0].start_seq = 0;
3094 tp->duplicate_sack[0].end_seq = 0;
7e10b655 3095 tp->dsack_dups = 0;
7ec65372 3096 tp->reord_seen = 0;
5c701549
ED		 3097 tp->retrans_out = 0;
3098 tp->sacked_out = 0;
3099 tp->tlp_high_seq = 0;
3100 tp->last_oow_ack_time = 0;
29c1c446 3101 tp->plb_rehash = 0;
6cda8b74
ED		 3102 /* There's a bubble in the pipe until at least the first ACK. */
3103 tp->app_limited = ~0U;
300b655d 3104 tp->rate_app_limited = 1;
792c4354
ED		 3105 tp->rack.mstamp = 0;
3106 tp->rack.advanced = 0;
3107 tp->rack.reo_wnd_steps = 1;
3108 tp->rack.last_delivered = 0;
3109 tp->rack.reo_wnd_persist = 0;
3110 tp->rack.dsack_seen = 0;
6bcdc40d
ED		 3111 tp->syn_data_acked = 0;
3112 tp->rx_opt.saw_tstamp = 0;
3113 tp->rx_opt.dsack = 0;
3114 tp->rx_opt.num_sacks = 0;
f9af2dbb 3115 tp->rcv_ooopack = 0;
6cda8b74 3116
1da177e4 3117
7db92362
WW		 3118 /* Clean up fastopen related fields */
3119 tcp_free_fastopen_req(tp);
08e39c0d 3120 inet_clear_bit(DEFER_CONNECT, sk);
48027478 3121 tp->fastopen_client_fail = 0;
7db92362 3122
c720c7e8 3123 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
1da177e4 3124
9b42d55a
LR		 3125 if (sk->sk_frag.page) {
3126 put_page(sk->sk_frag.page);
3127 sk->sk_frag.page = NULL;
3128 sk->sk_frag.offset = 0;
3129 }
e3ae2365 3130 sk_error_report(sk);
a01512b1 3131 return 0;
1da177e4 3132}
4bc2f18b 3133EXPORT_SYMBOL(tcp_disconnect);
1da177e4 3134
a2a385d6 3135static inline bool tcp_can_repair_sock(const struct sock *sk)
ee995283 3136{
cb388e7e 3137 return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
319b0534 3138 (sk->sk_state != TCP_LISTEN);
ee995283
PE		 3139}
3140
d38d2b00 3141static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
b1ed4c4f
AV		 3142{
3143 struct tcp_repair_window opt;
3144
3145 if (!tp->repair)
3146 return -EPERM;
3147
3148 if (len != sizeof(opt))
3149 return -EINVAL;
3150
d38d2b00 3151 if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
b1ed4c4f
AV		 3152 return -EFAULT;
3153
3154 if (opt.max_window < opt.snd_wnd)
3155 return -EINVAL;
3156
3157 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
3158 return -EINVAL;
3159
3160 if (after(opt.rcv_wup, tp->rcv_nxt))
3161 return -EINVAL;
3162
3163 tp->snd_wl1 = opt.snd_wl1;
3164 tp->snd_wnd = opt.snd_wnd;
3165 tp->max_window = opt.max_window;
3166
3167 tp->rcv_wnd = opt.rcv_wnd;
3168 tp->rcv_wup = opt.rcv_wup;
3169
3170 return 0;
3171}
3172
d38d2b00
CH		 3173static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
3174 unsigned int len)
b139ba4e 3175{
15e56515 3176 struct tcp_sock *tp = tcp_sk(sk);
de248a75 3177 struct tcp_repair_opt opt;
d3c48151 3178 size_t offset = 0;
b139ba4e 3179
de248a75 3180 while (len >= sizeof(opt)) {
d3c48151 3181 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
b139ba4e
PE		 3182 return -EFAULT;
3183
d3c48151 3184 offset += sizeof(opt);
de248a75 3185 len -= sizeof(opt);
b139ba4e 3186
de248a75
PE		 3187 switch (opt.opt_code) {
3188 case TCPOPT_MSS:
3189 tp->rx_opt.mss_clamp = opt.opt_val;
15e56515 3190 tcp_mtup_init(sk);
b139ba4e 3191 break;
de248a75 3192 case TCPOPT_WINDOW:
bc26ccd8
AV		 3193 {
3194 u16 snd_wscale = opt.opt_val & 0xFFFF;
3195 u16 rcv_wscale = opt.opt_val >> 16;
3196
589c49cb 3197 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
bc26ccd8 3198 return -EFBIG;
b139ba4e 3199
bc26ccd8
AV		 3200 tp->rx_opt.snd_wscale = snd_wscale;
3201 tp->rx_opt.rcv_wscale = rcv_wscale;
3202 tp->rx_opt.wscale_ok = 1;
3203 }
b139ba4e 3204 break;
b139ba4e 3205 case TCPOPT_SACK_PERM:
de248a75
PE		 3206 if (opt.opt_val != 0)
3207 return -EINVAL;
3208
b139ba4e 3209 tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
b139ba4e
PE		 3210 break;
3211 case TCPOPT_TIMESTAMP:
de248a75
PE		 3212 if (opt.opt_val != 0)
3213 return -EINVAL;
3214
b139ba4e
PE		 3215 tp->rx_opt.tstamp_ok = 1;
3216 break;
3217 }
3218 }
3219
3220 return 0;
3221}
3222
a842fe14
ED		 3223DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
3224EXPORT_SYMBOL(tcp_tx_delay_enabled);
3225
3226static void tcp_enable_tx_delay(void)
3227{
3228 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
3229 static int __tcp_tx_delay_enabled = 0;
3230
3231 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
3232 static_branch_enable(&tcp_tx_delay_enabled);
3233 pr_info("TCP_TX_DELAY enabled\n");
3234 }
3235 }
3236}
3237
db10538a
CH		 3238/* When set indicates to always queue non-full frames. Later the user clears
3239 * this option and we transmit any pending partial frames in the queue. This is
3240 * meant to be used alongside sendfile() to get properly filled frames when the
3241 * user (for example) must write out headers with a write() call first and then
3242 * use sendfile to send out the data parts.
3243 *
3244 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
3245 * TCP_NODELAY.
3246 */
6fadaa56 3247void __tcp_sock_set_cork(struct sock *sk, bool on)
db10538a
CH		 3248{
3249 struct tcp_sock *tp = tcp_sk(sk);
3250
3251 if (on) {
3252 tp->nonagle |= TCP_NAGLE_CORK;
3253 } else {
3254 tp->nonagle &= ~TCP_NAGLE_CORK;
3255 if (tp->nonagle & TCP_NAGLE_OFF)
3256 tp->nonagle |= TCP_NAGLE_PUSH;
3257 tcp_push_pending_frames(sk);
3258 }
3259}
3260
3261void tcp_sock_set_cork(struct sock *sk, bool on)
3262{
3263 lock_sock(sk);
3264 __tcp_sock_set_cork(sk, on);
3265 release_sock(sk);
3266}
3267EXPORT_SYMBOL(tcp_sock_set_cork);
3268
12abc5ee
CH		 3269/* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
3270 * remembered, but it is not activated until cork is cleared.
3271 *
3272 * However, when TCP_NODELAY is set we make an explicit push, which overrides
3273 * even TCP_CORK for currently queued segments.
3274 */
6fadaa56 3275void __tcp_sock_set_nodelay(struct sock *sk, bool on)
12abc5ee
CH		 3276{
3277 if (on) {
3278 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
3279 tcp_push_pending_frames(sk);
3280 } else {
3281 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
3282 }
3283}
3284
3285void tcp_sock_set_nodelay(struct sock *sk)
3286{
3287 lock_sock(sk);
3288 __tcp_sock_set_nodelay(sk, true);
3289 release_sock(sk);
3290}
3291EXPORT_SYMBOL(tcp_sock_set_nodelay);
3292
ddd061b8
CH		 3293static void __tcp_sock_set_quickack(struct sock *sk, int val)
3294{
3295 if (!val) {
3296 inet_csk_enter_pingpong_mode(sk);
3297 return;
3298 }
3299
3300 inet_csk_exit_pingpong_mode(sk);
3301 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
3302 inet_csk_ack_scheduled(sk)) {
3303 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
3304 tcp_cleanup_rbuf(sk, 1);
3305 if (!(val & 1))
3306 inet_csk_enter_pingpong_mode(sk);
3307 }
3308}
3309
3310void tcp_sock_set_quickack(struct sock *sk, int val)
3311{
3312 lock_sock(sk);
3313 __tcp_sock_set_quickack(sk, val);
3314 release_sock(sk);
3315}
3316EXPORT_SYMBOL(tcp_sock_set_quickack);
3317
557eadfc
CH		 3318int tcp_sock_set_syncnt(struct sock *sk, int val)
3319{
3320 if (val < 1 || val > MAX_TCP_SYNCNT)
3321 return -EINVAL;
3322
3a037f0f 3323 WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val);
557eadfc
CH		 3324 return 0;
3325}
3326EXPORT_SYMBOL(tcp_sock_set_syncnt);
3327
d58f2e15 3328int tcp_sock_set_user_timeout(struct sock *sk, int val)
c488aead 3329{
d58f2e15
ED		 3330 /* Cap the max time in ms TCP will retry or probe the window
3331 * before giving up and aborting (ETIMEDOUT) a connection.
3332 */
3333 if (val < 0)
3334 return -EINVAL;
3335
26023e91 3336 WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val);
d58f2e15 3337 return 0;
c488aead
CH		 3338}
3339EXPORT_SYMBOL(tcp_sock_set_user_timeout);
3340
aad4a0a9 3341int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
71c48eb8
CH		 3342{
3343 struct tcp_sock *tp = tcp_sk(sk);
3344
3345 if (val < 1 || val > MAX_TCP_KEEPIDLE)
3346 return -EINVAL;
3347
4164245c
ED		 3348 /* Paired with WRITE_ONCE() in keepalive_time_when() */
3349 WRITE_ONCE(tp->keepalive_time, val * HZ);
71c48eb8
CH		 3350 if (sock_flag(sk, SOCK_KEEPOPEN) &&
3351 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
3352 u32 elapsed = keepalive_time_elapsed(tp);
3353
3354 if (tp->keepalive_time > elapsed)
3355 elapsed = tp->keepalive_time - elapsed;
3356 else
3357 elapsed = 0;
3358 inet_csk_reset_keepalive_timer(sk, elapsed);
3359 }
3360
3361 return 0;
3362}
3363
3364int tcp_sock_set_keepidle(struct sock *sk, int val)
3365{
3366 int err;
3367
3368 lock_sock(sk);
aad4a0a9 3369 err = tcp_sock_set_keepidle_locked(sk, val);
71c48eb8
CH		 3370 release_sock(sk);
3371 return err;
3372}
3373EXPORT_SYMBOL(tcp_sock_set_keepidle);
3374
d41ecaac
CH		 3375int tcp_sock_set_keepintvl(struct sock *sk, int val)
3376{
3377 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3378 return -EINVAL;
3379
5ecf9d4f 3380 WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ);
d41ecaac
CH		 3381 return 0;
3382}
3383EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3384
480aeb96
CH		 3385int tcp_sock_set_keepcnt(struct sock *sk, int val)
3386{
3387 if (val < 1 || val > MAX_TCP_KEEPCNT)
3388 return -EINVAL;
3389
6e5e1de6
ED		 3390 /* Paired with READ_ONCE() in keepalive_probes() */
3391 WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val);
480aeb96
CH		 3392 return 0;
3393}
3394EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3395
cb811109
P		 3396int tcp_set_window_clamp(struct sock *sk, int val)
3397{
3398 struct tcp_sock *tp = tcp_sk(sk);
3399
3400 if (!val) {
3401 if (sk->sk_state != TCP_CLOSE)
3402 return -EINVAL;
f410cbea 3403 WRITE_ONCE(tp->window_clamp, 0);
cb811109 3404 } else {
58d3aade
PA		 3405 u32 new_rcv_ssthresh, old_window_clamp = tp->window_clamp;
3406 u32 new_window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
3407 SOCK_MIN_RCVBUF / 2 : val;
3408
3409 if (new_window_clamp == old_window_clamp)
3410 return 0;
3411
f410cbea 3412 WRITE_ONCE(tp->window_clamp, new_window_clamp);
58d3aade
PA		 3413 if (new_window_clamp < old_window_clamp) {
3414 /* need to apply the reserved mem provisioning only
3415 * when shrinking the window clamp
3416 */
3417 __tcp_adjust_rcv_ssthresh(sk, tp->window_clamp);
3418
3419 } else {
3420 new_rcv_ssthresh = min(tp->rcv_wnd, tp->window_clamp);
3421 tp->rcv_ssthresh = max(new_rcv_ssthresh,
3422 tp->rcv_ssthresh);
3423 }
cb811109
P		 3424 }
3425 return 0;
3426}
3427
1da177e4
LT		 3428/*
3429 * Socket option code for TCP.
3430 */
0c751f70
MKL		 3431int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3432 sockptr_t optval, unsigned int optlen)
1da177e4
LT		 3433{
3434 struct tcp_sock *tp = tcp_sk(sk);
463c84b9 3435 struct inet_connection_sock *icsk = inet_csk(sk);
1e579caa 3436 struct net *net = sock_net(sk);
1da177e4
LT		 3437 int val;
3438 int err = 0;
3439
e56fb50f
WAS		 3440 /* These are data/string values, all the others are ints */
3441 switch (optname) {
3442 case TCP_CONGESTION: {
5f8ef48d
SH		 3443 char name[TCP_CA_NAME_MAX];
3444
3445 if (optlen < 1)
3446 return -EINVAL;
3447
d38d2b00 3448 val = strncpy_from_sockptr(name, optval,
4fdb78d3 3449 min_t(long, TCP_CA_NAME_MAX-1, optlen));
5f8ef48d
SH		 3450 if (val < 0)
3451 return -EFAULT;
3452 name[val] = 0;
3453
cb388e7e 3454 sockopt_lock_sock(sk);
84e5a0f2 3455 err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
cb388e7e
MKL		 3456 sockopt_ns_capable(sock_net(sk)->user_ns,
3457 CAP_NET_ADMIN));
3458 sockopt_release_sock(sk);
5f8ef48d
SH		 3459 return err;
3460 }
734942cc
DW		 3461 case TCP_ULP: {
3462 char name[TCP_ULP_NAME_MAX];
3463
3464 if (optlen < 1)
3465 return -EINVAL;
3466
d38d2b00 3467 val = strncpy_from_sockptr(name, optval,
734942cc
DW		 3468 min_t(long, TCP_ULP_NAME_MAX - 1,
3469 optlen));
3470 if (val < 0)
3471 return -EFAULT;
3472 name[val] = 0;
3473
cb388e7e 3474 sockopt_lock_sock(sk);
734942cc 3475 err = tcp_set_ulp(sk, name);
cb388e7e 3476 sockopt_release_sock(sk);
734942cc
DW		 3477 return err;
3478 }
1fba70e5 3479 case TCP_FASTOPEN_KEY: {
0f1ce023
JB		 3480 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3481 __u8 *backup_key = NULL;
1fba70e5 3482
0f1ce023
JB		 3483 /* Allow a backup key as well to facilitate key rotation
3484 * First key is the active one.
3485 */
3486 if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3487 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
1fba70e5
YC		 3488 return -EINVAL;
3489
d38d2b00 3490 if (copy_from_sockptr(key, optval, optlen))
1fba70e5
YC		 3491 return -EFAULT;
3492
0f1ce023
JB		 3493 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3494 backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3495
438ac880 3496 return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
1fba70e5 3497 }
e56fb50f
WAS		 3498 default:
3499 /* fallthru */
3500 break;
ccbd6a5a 3501 }
5f8ef48d 3502
1da177e4
LT		 3503 if (optlen < sizeof(int))
3504 return -EINVAL;
3505
d38d2b00 3506 if (copy_from_sockptr(&val, optval, sizeof(val)))
1da177e4
LT		 3507 return -EFAULT;
3508
d44fd4a7
ED		 3509 /* Handle options that can be set without locking the socket. */
3510 switch (optname) {
3511 case TCP_SYNCNT:
3512 return tcp_sock_set_syncnt(sk, val);
d58f2e15
ED		 3513 case TCP_USER_TIMEOUT:
3514 return tcp_sock_set_user_timeout(sk, val);
6fd70a6b
ED		 3515 case TCP_KEEPINTVL:
3516 return tcp_sock_set_keepintvl(sk, val);
84485080
ED		 3517 case TCP_KEEPCNT:
3518 return tcp_sock_set_keepcnt(sk, val);
a81722dd
ED		 3519 case TCP_LINGER2:
3520 if (val < 0)
3521 WRITE_ONCE(tp->linger2, -1);
3522 else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3523 WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);
3524 else
3525 WRITE_ONCE(tp->linger2, val * HZ);
3526 return 0;
6e97ba55
ED		 3527 case TCP_DEFER_ACCEPT:
3528 /* Translate value in seconds to number of retransmits */
3529 WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept,
3530 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3531 TCP_RTO_MAX / HZ));
3532 return 0;
d44fd4a7
ED		 3533 }
3534
cb388e7e 3535 sockopt_lock_sock(sk);
1da177e4
LT		 3536
3537 switch (optname) {
3538 case TCP_MAXSEG:
3539 /* Values greater than interface MTU won't take effect. However
3540 * at the point when this call is done we typically don't yet
a777f715
RC		 3541 * know which interface is going to be used
3542 */
cfc62d87 3543 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
1da177e4
LT		 3544 err = -EINVAL;
3545 break;
3546 }
3547 tp->rx_opt.user_mss = val;
3548 break;
3549
3550 case TCP_NODELAY:
12abc5ee 3551 __tcp_sock_set_nodelay(sk, val);
1da177e4
LT		 3552 break;
3553
36e31b0a
AP		 3554 case TCP_THIN_LINEAR_TIMEOUTS:
3555 if (val < 0 || val > 1)
3556 err = -EINVAL;
3557 else
3558 tp->thin_lto = val;
3559 break;
3560
7e380175
AP		 3561 case TCP_THIN_DUPACK:
3562 if (val < 0 || val > 1)
3563 err = -EINVAL;
7e380175
AP		 3564 break;
3565
ee995283
PE		 3566 case TCP_REPAIR:
3567 if (!tcp_can_repair_sock(sk))
3568 err = -EPERM;
31048d7a 3569 else if (val == TCP_REPAIR_ON) {
ee995283
PE		 3570 tp->repair = 1;
3571 sk->sk_reuse = SK_FORCE_REUSE;
3572 tp->repair_queue = TCP_NO_QUEUE;
31048d7a 3573 } else if (val == TCP_REPAIR_OFF) {
ee995283
PE		 3574 tp->repair = 0;
3575 sk->sk_reuse = SK_NO_REUSE;
3576 tcp_send_window_probe(sk);
31048d7a 3577 } else if (val == TCP_REPAIR_OFF_NO_WP) {
ee995283
PE		 3578 tp->repair = 0;
3579 sk->sk_reuse = SK_NO_REUSE;
ee995283
PE		 3580 } else
3581 err = -EINVAL;
3582
3583 break;
3584
3585 case TCP_REPAIR_QUEUE:
3586 if (!tp->repair)
3587 err = -EPERM;
bf2acc94 3588 else if ((unsigned int)val < TCP_QUEUES_NR)
ee995283
PE		 3589 tp->repair_queue = val;
3590 else
3591 err = -EINVAL;
3592 break;
3593
3594 case TCP_QUEUE_SEQ:
8811f4a9 3595 if (sk->sk_state != TCP_CLOSE) {
ee995283 3596 err = -EPERM;
8811f4a9
ED		 3597 } else if (tp->repair_queue == TCP_SEND_QUEUE) {
3598 if (!tcp_rtx_queue_empty(sk))
3599 err = -EPERM;
3600 else
3601 WRITE_ONCE(tp->write_seq, val);
3602 } else if (tp->repair_queue == TCP_RECV_QUEUE) {
3603 if (tp->rcv_nxt != tp->copied_seq) {
3604 err = -EPERM;
3605 } else {
3606 WRITE_ONCE(tp->rcv_nxt, val);
3607 WRITE_ONCE(tp->copied_seq, val);
3608 }
3609 } else {
ee995283 3610 err = -EINVAL;
8811f4a9 3611 }
ee995283
PE		 3612 break;
3613
b139ba4e
PE		 3614 case TCP_REPAIR_OPTIONS:
3615 if (!tp->repair)
3616 err = -EINVAL;
0c175da7 3617 else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
d38d2b00 3618 err = tcp_repair_options_est(sk, optval, optlen);
b139ba4e
PE		 3619 else
3620 err = -EPERM;
3621 break;
3622
1da177e4 3623 case TCP_CORK:
db10538a 3624 __tcp_sock_set_cork(sk, val);
1da177e4
LT		 3625 break;
3626
3627 case TCP_KEEPIDLE:
aad4a0a9 3628 err = tcp_sock_set_keepidle_locked(sk, val);
1da177e4 3629 break;
cd8ae852 3630 case TCP_SAVE_SYN:
267cf9fa
MKL		 3631 /* 0: disable, 1: enable, 2: start from ether_header */
3632 if (val < 0 || val > 2)
cd8ae852
ED		 3633 err = -EINVAL;
3634 else
3635 tp->save_syn = val;
3636 break;
3637
1da177e4 3638 case TCP_WINDOW_CLAMP:
cb811109 3639 err = tcp_set_window_clamp(sk, val);
1da177e4
LT		 3640 break;
3641
3642 case TCP_QUICKACK:
ddd061b8 3643 __tcp_sock_set_quickack(sk, val);
1da177e4
LT		 3644 break;
3645
faadfaba 3646 case TCP_AO_REPAIR:
965c00e4
DS		 3647 if (!tcp_can_repair_sock(sk)) {
3648 err = -EPERM;
3649 break;
3650 }
faadfaba
DS		 3651 err = tcp_ao_set_repair(sk, optval, optlen);
3652 break;
4954f17d
DS		 3653#ifdef CONFIG_TCP_AO
3654 case TCP_AO_ADD_KEY:
3655 case TCP_AO_DEL_KEY:
3656 case TCP_AO_INFO: {
3657 /* If this is the first TCP-AO setsockopt() on the socket,
faadfaba
DS		 3658 * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR
3659 * in any state.
4954f17d 3660 */
faadfaba
DS		 3661 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
3662 goto ao_parse;
3663 if (rcu_dereference_protected(tcp_sk(sk)->ao_info,
4954f17d 3664 lockdep_sock_is_held(sk)))
faadfaba
DS		 3665 goto ao_parse;
3666 if (tp->repair)
3667 goto ao_parse;
3668 err = -EISCONN;
3669 break;
3670ao_parse:
3671 err = tp->af_specific->ao_parse(sk, optname, optval, optlen);
4954f17d
DS		 3672 break;
3673 }
3674#endif
cfb6eeb4
YH		 3675#ifdef CONFIG_TCP_MD5SIG
3676 case TCP_MD5SIG:
8917a777 3677 case TCP_MD5SIG_EXT:
d38d2b00 3678 err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
cfb6eeb4
YH		 3679 break;
3680#endif
8336886f
JC		 3681 case TCP_FASTOPEN:
3682 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
dfea2aa6 3683 TCPF_LISTEN))) {
43713848 3684 tcp_fastopen_init_key_once(net);
dfea2aa6 3685
0536fcc0 3686 fastopen_queue_tune(sk, val);
dfea2aa6 3687 } else {
8336886f 3688 err = -EINVAL;
dfea2aa6 3689 }
8336886f 3690 break;
19f6d3f3
WW		 3691 case TCP_FASTOPEN_CONNECT:
3692 if (val > 1 || val < 0) {
3693 err = -EINVAL;
5a542133
KI		 3694 } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
3695 TFO_CLIENT_ENABLE) {
19f6d3f3
WW		 3696 if (sk->sk_state == TCP_CLOSE)
3697 tp->fastopen_connect = val;
3698 else
3699 err = -EINVAL;
3700 } else {
3701 err = -EOPNOTSUPP;
3702 }
3703 break;
71c02379
CP		 3704 case TCP_FASTOPEN_NO_COOKIE:
3705 if (val > 1 || val < 0)
3706 err = -EINVAL;
3707 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3708 err = -EINVAL;
3709 else
3710 tp->fastopen_no_cookie = val;
3711 break;
93be6ce0 3712 case TCP_TIMESTAMP:
614e8316 3713 if (!tp->repair) {
93be6ce0 3714 err = -EPERM;
614e8316
ED		 3715 break;
3716 }
3717 /* val is an opaque field,
3718 * and low order bit contains usec_ts enable bit.
3719 * Its a best effort, and we do not care if user makes an error.
3720 */
3721 tp->tcp_usec_ts = val & 1;
3722 WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));
93be6ce0 3723 break;
b1ed4c4f
AV		 3724 case TCP_REPAIR_WINDOW:
3725 err = tcp_repair_set_window(tp, optval, optlen);
3726 break;
c9bee3b7 3727 case TCP_NOTSENT_LOWAT:
1aeb87bc 3728 WRITE_ONCE(tp->notsent_lowat, val);
c9bee3b7
ED		 3729 sk->sk_write_space(sk);
3730 break;
b75eba76
SHY		 3731 case TCP_INQ:
3732 if (val > 1 || val < 0)
3733 err = -EINVAL;
3734 else
3735 tp->recvmsg_inq = val;
3736 break;
a842fe14
ED		 3737 case TCP_TX_DELAY:
3738 if (val)
3739 tcp_enable_tx_delay();
348b81b6 3740 WRITE_ONCE(tp->tcp_tx_delay, val);
a842fe14 3741 break;
1da177e4
LT		 3742 default:
3743 err = -ENOPROTOOPT;
3744 break;
3ff50b79
SH		 3745 }
3746
cb388e7e 3747 sockopt_release_sock(sk);
1da177e4
LT		 3748 return err;
3749}
3750
a7b75c5a 3751int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
b7058842 3752 unsigned int optlen)
3fdadf7d 3753{
cf533ea5 3754 const struct inet_connection_sock *icsk = inet_csk(sk);
3fdadf7d
DM		 3755
3756 if (level != SOL_TCP)
f49cd2f4
KI		 3757 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
3758 return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
3759 optval, optlen);
a7b75c5a 3760 return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3fdadf7d 3761}
4bc2f18b 3762EXPORT_SYMBOL(tcp_setsockopt);
3fdadf7d 3763
efd90174
FY		 3764static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3765 struct tcp_info *info)
3766{
3767 u64 stats[__TCP_CHRONO_MAX], total = 0;
3768 enum tcp_chrono i;
3769
3770 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
3771 stats[i] = tp->chrono_stat[i - 1];
3772 if (i == tp->chrono_type)
628174cc 3773 stats[i] += tcp_jiffies32 - tp->chrono_start;
efd90174
FY		 3774 stats[i] *= USEC_PER_SEC / HZ;
3775 total += stats[i];
3776 }
3777
3778 info->tcpi_busy_time = total;
3779 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
3780 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
3781}
3782
1da177e4 3783/* Return information about state of tcp endpoint in API format. */
0df48c26 3784void tcp_get_info(struct sock *sk, struct tcp_info *info)
1da177e4 3785{
35ac838a 3786 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
463c84b9 3787 const struct inet_connection_sock *icsk = inet_csk(sk);
76a9ebe8 3788 unsigned long rate;
0263598c 3789 u32 now;
ff5d7497 3790 u64 rate64;
67db3e4b 3791 bool slow;
1da177e4
LT		 3792
3793 memset(info, 0, sizeof(*info));
35ac838a
CG		 3794 if (sk->sk_type != SOCK_STREAM)
3795 return;
1da177e4 3796
986ffdfd 3797 info->tcpi_state = inet_sk_state_load(sk);
00fd38d9 3798
ccbf3bfa
ED		 3799 /* Report meaningful fields for all TCP states, including listeners */
3800 rate = READ_ONCE(sk->sk_pacing_rate);
76a9ebe8 3801 rate64 = (rate != ~0UL) ? rate : ~0ULL;
f522a5fc 3802 info->tcpi_pacing_rate = rate64;
ccbf3bfa
ED		 3803
3804 rate = READ_ONCE(sk->sk_max_pacing_rate);
76a9ebe8 3805 rate64 = (rate != ~0UL) ? rate : ~0ULL;
f522a5fc 3806 info->tcpi_max_pacing_rate = rate64;
ccbf3bfa
ED		 3807
3808 info->tcpi_reordering = tp->reordering;
40570375 3809 info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);
ccbf3bfa
ED		 3810
3811 if (info->tcpi_state == TCP_LISTEN) {
3812 /* listeners aliased fields :
3813 * tcpi_unacked -> Number of children ready for accept()
3814 * tcpi_sacked -> max backlog
3815 */
288efe86 3816 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
099ecf59 3817 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
ccbf3bfa
ED		 3818 return;
3819 }
b369e7fd
ED		 3820
3821 slow = lock_sock_fast(sk);
3822
6687e988 3823 info->tcpi_ca_state = icsk->icsk_ca_state;
463c84b9 3824 info->tcpi_retransmits = icsk->icsk_retransmits;
6687e988 3825 info->tcpi_probes = icsk->icsk_probes_out;
463c84b9 3826 info->tcpi_backoff = icsk->icsk_backoff;
1da177e4
LT		 3827
3828 if (tp->rx_opt.tstamp_ok)
3829 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
e60402d0 3830 if (tcp_is_sack(tp))
1da177e4
LT		 3831 info->tcpi_options |= TCPI_OPT_SACK;
3832 if (tp->rx_opt.wscale_ok) {
3833 info->tcpi_options |= TCPI_OPT_WSCALE;
3834 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
3835 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
e905a9ed 3836 }
1da177e4 3837
b5c5693b 3838 if (tp->ecn_flags & TCP_ECN_OK)
1da177e4 3839 info->tcpi_options |= TCPI_OPT_ECN;
b5c5693b
ED		 3840 if (tp->ecn_flags & TCP_ECN_SEEN)
3841 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
6f73601e
YC		 3842 if (tp->syn_data_acked)
3843 info->tcpi_options |= TCPI_OPT_SYN_DATA;
a77a0f5c
ED		 3844 if (tp->tcp_usec_ts)
3845 info->tcpi_options |= TCPI_OPT_USEC_TS;
1da177e4 3846
463c84b9 3847 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
95b9a87c
DM		 3848 info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato,
3849 tcp_delack_max(sk)));
c1b4a7e6 3850 info->tcpi_snd_mss = tp->mss_cache;
463c84b9 3851 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
1da177e4 3852
ccbf3bfa
ED		 3853 info->tcpi_unacked = tp->packets_out;
3854 info->tcpi_sacked = tp->sacked_out;
3855
1da177e4
LT		 3856 info->tcpi_lost = tp->lost_out;
3857 info->tcpi_retrans = tp->retrans_out;
1da177e4 3858
d635fbe2 3859 now = tcp_jiffies32;
1da177e4 3860 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
463c84b9 3861 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
1da177e4
LT		 3862 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
3863
d83d8461 3864 info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
1da177e4 3865 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
740b0f18
ED		 3866 info->tcpi_rtt = tp->srtt_us >> 3;
3867 info->tcpi_rttvar = tp->mdev_us >> 2;
1da177e4 3868 info->tcpi_snd_ssthresh = tp->snd_ssthresh;
1da177e4 3869 info->tcpi_advmss = tp->advmss;
1da177e4 3870
645f4c6f 3871 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
1da177e4
LT		 3872 info->tcpi_rcv_space = tp->rcvq_space.space;
3873
3874 info->tcpi_total_retrans = tp->total_retrans;
977cb0ec 3875
f522a5fc
ED		 3876 info->tcpi_bytes_acked = tp->bytes_acked;
3877 info->tcpi_bytes_received = tp->bytes_received;
67db3e4b 3878 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
efd90174 3879 tcp_get_info_chrono_stats(tp, info);
67db3e4b 3880
2efd055c 3881 info->tcpi_segs_out = tp->segs_out;
0307a0b7
ED		 3882
3883 /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */
3884 info->tcpi_segs_in = READ_ONCE(tp->segs_in);
3885 info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);
cd9b2660 3886
cd9b2660 3887 info->tcpi_min_rtt = tcp_min_rtt(tp);
a44d6eac 3888 info->tcpi_data_segs_out = tp->data_segs_out;
eb8329e0
YC		 3889
3890 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
0263598c
WW		 3891 rate64 = tcp_compute_delivery_rate(tp);
3892 if (rate64)
f522a5fc 3893 info->tcpi_delivery_rate = rate64;
feb5f2ec
YC		 3894 info->tcpi_delivered = tp->delivered;
3895 info->tcpi_delivered_ce = tp->delivered_ce;
ba113c3a 3896 info->tcpi_bytes_sent = tp->bytes_sent;
fb31c9b9 3897 info->tcpi_bytes_retrans = tp->bytes_retrans;
7e10b655 3898 info->tcpi_dsack_dups = tp->dsack_dups;
7ec65372 3899 info->tcpi_reord_seen = tp->reord_seen;
f9af2dbb 3900 info->tcpi_rcv_ooopack = tp->rcv_ooopack;
8f7baad7 3901 info->tcpi_snd_wnd = tp->snd_wnd;
71fc7047
MAQ		 3902 info->tcpi_rcv_wnd = tp->rcv_wnd;
3903 info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;
48027478 3904 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
3868ab0f
A		 3905
3906 info->tcpi_total_rto = tp->total_rto;
3907 info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;
3908 info->tcpi_total_rto_time = tp->total_rto_time;
2a7c8d29
ED		 3909 if (tp->rto_stamp)
3910 info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;
3868ab0f 3911
b369e7fd 3912 unlock_sock_fast(sk, slow);
1da177e4 3913}
1da177e4
LT		 3914EXPORT_SYMBOL_GPL(tcp_get_info);
3915
984988aa
WW		 3916static size_t tcp_opt_stats_get_size(void)
3917{
3918 return
3919 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */
3920 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
3921 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
3922 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
3923 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
3924 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */
3925 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
3926 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */
3927 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */
3928 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */
3929 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
3930 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
3931 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
3932 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */
3933 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
3934 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */
3935 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
ba113c3a 3936 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
fb31c9b9 3937 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
7e10b655 3938 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
7ec65372 3939 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
e8bd8fca 3940 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
32efcc06 3941 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
e08ab0b3 3942 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
48040793 3943 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */
e7ed11ee 3944 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */
29c1c446 3945 nla_total_size(sizeof(u32)) + /* TCP_NLA_REHASH */
984988aa
WW		 3946 0;
3947}
3948
e7ed11ee
YS		 3949/* Returns TTL or hop limit of an incoming packet from skb. */
3950static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb)
3951{
3952 if (skb->protocol == htons(ETH_P_IP))
3953 return ip_hdr(skb)->ttl;
3954 else if (skb->protocol == htons(ETH_P_IPV6))
3955 return ipv6_hdr(skb)->hop_limit;
3956 else
3957 return 0;
3958}
3959
48040793 3960struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
e7ed11ee
YS		 3961 const struct sk_buff *orig_skb,
3962 const struct sk_buff *ack_skb)
1c885808
FY		 3963{
3964 const struct tcp_sock *tp = tcp_sk(sk);
3965 struct sk_buff *stats;
3966 struct tcp_info info;
76a9ebe8 3967 unsigned long rate;
bb7c19f9 3968 u64 rate64;
1c885808 3969
984988aa 3970 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
1c885808
FY		 3971 if (!stats)
3972 return NULL;
3973
3974 tcp_get_info_chrono_stats(tp, &info);
3975 nla_put_u64_64bit(stats, TCP_NLA_BUSY,
3976 info.tcpi_busy_time, TCP_NLA_PAD);
3977 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
3978 info.tcpi_rwnd_limited, TCP_NLA_PAD);
3979 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
3980 info.tcpi_sndbuf_limited, TCP_NLA_PAD);
7e98102f
YC		 3981 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
3982 tp->data_segs_out, TCP_NLA_PAD);
3983 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
3984 tp->total_retrans, TCP_NLA_PAD);
bb7c19f9
WW		 3985
3986 rate = READ_ONCE(sk->sk_pacing_rate);
76a9ebe8 3987 rate64 = (rate != ~0UL) ? rate : ~0ULL;
bb7c19f9
WW		 3988 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3989
3990 rate64 = tcp_compute_delivery_rate(tp);
3991 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
3992
40570375 3993 nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp));
bb7c19f9
WW		 3994 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
3995 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
3996
3997 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
3998 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
7156d194 3999 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
feb5f2ec
YC		 4000 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
4001 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
87ecc95d
PJ		 4002
4003 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
be631892 4004 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
feb5f2ec 4005
ba113c3a
WW		 4006 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
4007 TCP_NLA_PAD);
fb31c9b9
WW		 4008 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
4009 TCP_NLA_PAD);
7e10b655 4010 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
7ec65372 4011 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
e8bd8fca 4012 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
32efcc06 4013 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
e08ab0b3
YS		 4014 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
4015 max_t(int, 0, tp->write_seq - tp->snd_nxt));
48040793
YS		 4016 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
4017 TCP_NLA_PAD);
e7ed11ee
YS		 4018 if (ack_skb)
4019 nla_put_u8(stats, TCP_NLA_TTL,
4020 tcp_skb_ttl_or_hop_limit(ack_skb));
ba113c3a 4021
29c1c446 4022 nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash);
1c885808
FY		 4023 return stats;
4024}
4025
273b7f0f
MKL		 4026int do_tcp_getsockopt(struct sock *sk, int level,
4027 int optname, sockptr_t optval, sockptr_t optlen)
1da177e4 4028{
295f7324 4029 struct inet_connection_sock *icsk = inet_csk(sk);
1da177e4 4030 struct tcp_sock *tp = tcp_sk(sk);
6fa25166 4031 struct net *net = sock_net(sk);
1da177e4
LT		 4032 int val, len;
4033
34704ef0 4034 if (copy_from_sockptr(&len, optlen, sizeof(int)))
1da177e4
LT		 4035 return -EFAULT;
4036
1da177e4
LT		 4037 if (len < 0)
4038 return -EINVAL;
4039
716edc97
GI		 4040 len = min_t(unsigned int, len, sizeof(int));
4041
1da177e4
LT		 4042 switch (optname) {
4043 case TCP_MAXSEG:
c1b4a7e6 4044 val = tp->mss_cache;
34dfde4a
CZ		 4045 if (tp->rx_opt.user_mss &&
4046 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
1da177e4 4047 val = tp->rx_opt.user_mss;
5e6a3ce6
PE		 4048 if (tp->repair)
4049 val = tp->rx_opt.mss_clamp;
1da177e4
LT		 4050 break;
4051 case TCP_NODELAY:
4052 val = !!(tp->nonagle&TCP_NAGLE_OFF);
4053 break;
4054 case TCP_CORK:
4055 val = !!(tp->nonagle&TCP_NAGLE_CORK);
4056 break;
4057 case TCP_KEEPIDLE:
df19a626 4058 val = keepalive_time_when(tp) / HZ;
1da177e4
LT		 4059 break;
4060 case TCP_KEEPINTVL:
df19a626 4061 val = keepalive_intvl_when(tp) / HZ;
1da177e4
LT		 4062 break;
4063 case TCP_KEEPCNT:
df19a626 4064 val = keepalive_probes(tp);
1da177e4
LT		 4065 break;
4066 case TCP_SYNCNT:
3a037f0f 4067 val = READ_ONCE(icsk->icsk_syn_retries) ? :
20a3b1c0 4068 READ_ONCE(net->ipv4.sysctl_tcp_syn_retries);
1da177e4
LT		 4069 break;
4070 case TCP_LINGER2:
9df5335c 4071 val = READ_ONCE(tp->linger2);
1da177e4 4072 if (val >= 0)
39e24435 4073 val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
1da177e4
LT		 4074 break;
4075 case TCP_DEFER_ACCEPT:
ae488c74
ED		 4076 val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept);
4077 val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ,
4078 TCP_RTO_MAX / HZ);
1da177e4
LT		 4079 break;
4080 case TCP_WINDOW_CLAMP:
f410cbea 4081 val = READ_ONCE(tp->window_clamp);
1da177e4
LT		 4082 break;
4083 case TCP_INFO: {
4084 struct tcp_info info;
4085
34704ef0 4086 if (copy_from_sockptr(&len, optlen, sizeof(int)))
1da177e4
LT		 4087 return -EFAULT;
4088
4089 tcp_get_info(sk, &info);
4090
4091 len = min_t(unsigned int, len, sizeof(info));
34704ef0 4092 if (copy_to_sockptr(optlen, &len, sizeof(int)))
1da177e4 4093 return -EFAULT;
34704ef0 4094 if (copy_to_sockptr(optval, &info, len))
1da177e4
LT		 4095 return -EFAULT;
4096 return 0;
4097 }
6e9250f5
ED		 4098 case TCP_CC_INFO: {
4099 const struct tcp_congestion_ops *ca_ops;
4100 union tcp_cc_info info;
4101 size_t sz = 0;
4102 int attr;
4103
34704ef0 4104 if (copy_from_sockptr(&len, optlen, sizeof(int)))
6e9250f5
ED		 4105 return -EFAULT;
4106
4107 ca_ops = icsk->icsk_ca_ops;
4108 if (ca_ops && ca_ops->get_info)
4109 sz = ca_ops->get_info(sk, ~0U, &attr, &info);
4110
4111 len = min_t(unsigned int, len, sz);
34704ef0 4112 if (copy_to_sockptr(optlen, &len, sizeof(int)))
6e9250f5 4113 return -EFAULT;
34704ef0 4114 if (copy_to_sockptr(optval, &info, len))
6e9250f5
ED		 4115 return -EFAULT;
4116 return 0;
4117 }
1da177e4 4118 case TCP_QUICKACK:
31954cd8 4119 val = !inet_csk_in_pingpong_mode(sk);
1da177e4 4120 break;
5f8ef48d
SH		 4121
4122 case TCP_CONGESTION:
34704ef0 4123 if (copy_from_sockptr(&len, optlen, sizeof(int)))
5f8ef48d
SH		 4124 return -EFAULT;
4125 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
34704ef0 4126 if (copy_to_sockptr(optlen, &len, sizeof(int)))
5f8ef48d 4127 return -EFAULT;
34704ef0 4128 if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len))
5f8ef48d
SH		 4129 return -EFAULT;
4130 return 0;
e56fb50f 4131
734942cc 4132 case TCP_ULP:
34704ef0 4133 if (copy_from_sockptr(&len, optlen, sizeof(int)))
734942cc
DW		 4134 return -EFAULT;
4135 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
d97af30f 4136 if (!icsk->icsk_ulp_ops) {
34704ef0
MKL		 4137 len = 0;
4138 if (copy_to_sockptr(optlen, &len, sizeof(int)))
d97af30f
DW		 4139 return -EFAULT;
4140 return 0;
4141 }
34704ef0 4142 if (copy_to_sockptr(optlen, &len, sizeof(int)))
734942cc 4143 return -EFAULT;
34704ef0 4144 if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
734942cc
DW		 4145 return -EFAULT;
4146 return 0;
4147
1fba70e5 4148 case TCP_FASTOPEN_KEY: {
f19008e6
JB		 4149 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
4150 unsigned int key_len;
1fba70e5 4151
34704ef0 4152 if (copy_from_sockptr(&len, optlen, sizeof(int)))
1fba70e5
YC		 4153 return -EFAULT;
4154
f19008e6
JB		 4155 key_len = tcp_fastopen_get_cipher(net, icsk, key) *
4156 TCP_FASTOPEN_KEY_LENGTH;
0f1ce023 4157 len = min_t(unsigned int, len, key_len);
34704ef0 4158 if (copy_to_sockptr(optlen, &len, sizeof(int)))
1fba70e5 4159 return -EFAULT;
34704ef0 4160 if (copy_to_sockptr(optval, key, len))
1fba70e5
YC		 4161 return -EFAULT;
4162 return 0;
4163 }
3c0fef0b
JH		 4164 case TCP_THIN_LINEAR_TIMEOUTS:
4165 val = tp->thin_lto;
4166 break;
4a7f6009 4167
3c0fef0b 4168 case TCP_THIN_DUPACK:
4a7f6009 4169 val = 0;
3c0fef0b 4170 break;
dca43c75 4171
ee995283
PE		 4172 case TCP_REPAIR:
4173 val = tp->repair;
4174 break;
4175
4176 case TCP_REPAIR_QUEUE:
4177 if (tp->repair)
4178 val = tp->repair_queue;
4179 else
4180 return -EINVAL;
4181 break;
4182
b1ed4c4f
AV		 4183 case TCP_REPAIR_WINDOW: {
4184 struct tcp_repair_window opt;
4185
34704ef0 4186 if (copy_from_sockptr(&len, optlen, sizeof(int)))
b1ed4c4f
AV		 4187 return -EFAULT;
4188
4189 if (len != sizeof(opt))
4190 return -EINVAL;
4191
4192 if (!tp->repair)
4193 return -EPERM;
4194
4195 opt.snd_wl1 = tp->snd_wl1;
4196 opt.snd_wnd = tp->snd_wnd;
4197 opt.max_window = tp->max_window;
4198 opt.rcv_wnd = tp->rcv_wnd;
4199 opt.rcv_wup = tp->rcv_wup;
4200
34704ef0 4201 if (copy_to_sockptr(optval, &opt, len))
b1ed4c4f
AV		 4202 return -EFAULT;
4203 return 0;
4204 }
ee995283
PE		 4205 case TCP_QUEUE_SEQ:
4206 if (tp->repair_queue == TCP_SEND_QUEUE)
4207 val = tp->write_seq;
4208 else if (tp->repair_queue == TCP_RECV_QUEUE)
4209 val = tp->rcv_nxt;
4210 else
4211 return -EINVAL;
4212 break;
4213
dca43c75 4214 case TCP_USER_TIMEOUT:
26023e91 4215 val = READ_ONCE(icsk->icsk_user_timeout);
dca43c75 4216 break;
1536e285
KN		 4217
4218 case TCP_FASTOPEN:
70f360dd 4219 val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen);
1536e285
KN		 4220 break;
4221
19f6d3f3
WW		 4222 case TCP_FASTOPEN_CONNECT:
4223 val = tp->fastopen_connect;
4224 break;
4225
71c02379
CP		 4226 case TCP_FASTOPEN_NO_COOKIE:
4227 val = tp->fastopen_no_cookie;
4228 break;
4229
a842fe14 4230 case TCP_TX_DELAY:
348b81b6 4231 val = READ_ONCE(tp->tcp_tx_delay);
a842fe14
ED		 4232 break;
4233
93be6ce0 4234 case TCP_TIMESTAMP:
614e8316
ED		 4235 val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);
4236 if (tp->tcp_usec_ts)
4237 val |= 1;
4238 else
4239 val &= ~1;
93be6ce0 4240 break;
c9bee3b7 4241 case TCP_NOTSENT_LOWAT:
1aeb87bc 4242 val = READ_ONCE(tp->notsent_lowat);
c9bee3b7 4243 break;
b75eba76
SHY		 4244 case TCP_INQ:
4245 val = tp->recvmsg_inq;
4246 break;
cd8ae852
ED		 4247 case TCP_SAVE_SYN:
4248 val = tp->save_syn;
4249 break;
4250 case TCP_SAVED_SYN: {
34704ef0 4251 if (copy_from_sockptr(&len, optlen, sizeof(int)))
cd8ae852
ED		 4252 return -EFAULT;
4253
d51bbff2 4254 sockopt_lock_sock(sk);
cd8ae852 4255 if (tp->saved_syn) {
70a217f1 4256 if (len < tcp_saved_syn_len(tp->saved_syn)) {
34704ef0
MKL		 4257 len = tcp_saved_syn_len(tp->saved_syn);
4258 if (copy_to_sockptr(optlen, &len, sizeof(int))) {
d51bbff2 4259 sockopt_release_sock(sk);
aea0929e
EM		 4260 return -EFAULT;
4261 }
d51bbff2 4262 sockopt_release_sock(sk);
aea0929e
EM		 4263 return -EINVAL;
4264 }
70a217f1 4265 len = tcp_saved_syn_len(tp->saved_syn);
34704ef0 4266 if (copy_to_sockptr(optlen, &len, sizeof(int))) {
d51bbff2 4267 sockopt_release_sock(sk);
cd8ae852
ED		 4268 return -EFAULT;
4269 }
34704ef0 4270 if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
d51bbff2 4271 sockopt_release_sock(sk);
cd8ae852
ED		 4272 return -EFAULT;
4273 }
4274 tcp_saved_syn_free(tp);
d51bbff2 4275 sockopt_release_sock(sk);
cd8ae852 4276 } else {
d51bbff2 4277 sockopt_release_sock(sk);
cd8ae852 4278 len = 0;
34704ef0 4279 if (copy_to_sockptr(optlen, &len, sizeof(int)))
cd8ae852
ED		 4280 return -EFAULT;
4281 }
4282 return 0;
4283 }
05255b82
ED		 4284#ifdef CONFIG_MMU
4285 case TCP_ZEROCOPY_RECEIVE: {
7eeba170 4286 struct scm_timestamping_internal tss;
e0fecb28 4287 struct tcp_zerocopy_receive zc = {};
05255b82
ED		 4288 int err;
4289
34704ef0 4290 if (copy_from_sockptr(&len, optlen, sizeof(int)))
05255b82 4291 return -EFAULT;
2107d45f
AR		 4292 if (len < 0 ||
4293 len < offsetofend(struct tcp_zerocopy_receive, length))
05255b82 4294 return -EINVAL;
3c5a2fd0 4295 if (unlikely(len > sizeof(zc))) {
34704ef0
MKL		 4296 err = check_zeroed_sockptr(optval, sizeof(zc),
4297 len - sizeof(zc));
3c5a2fd0
AR		 4298 if (err < 1)
4299 return err == 0 ? -EINVAL : err;
c8856c05 4300 len = sizeof(zc);
34704ef0 4301 if (copy_to_sockptr(optlen, &len, sizeof(int)))
0b7f41f6
AR		 4302 return -EFAULT;
4303 }
34704ef0 4304 if (copy_from_sockptr(&zc, optval, len))
05255b82 4305 return -EFAULT;
3c5a2fd0
AR		 4306 if (zc.reserved)
4307 return -EINVAL;
4308 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS))
4309 return -EINVAL;
d51bbff2 4310 sockopt_lock_sock(sk);
7eeba170 4311 err = tcp_zerocopy_receive(sk, &zc, &tss);
9cacf81f
SF		 4312 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4313 &zc, &len, err);
d51bbff2 4314 sockopt_release_sock(sk);
7eeba170
AR		 4315 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
4316 goto zerocopy_rcv_cmsg;
c8856c05 4317 switch (len) {
7eeba170
AR		 4318 case offsetofend(struct tcp_zerocopy_receive, msg_flags):
4319 goto zerocopy_rcv_cmsg;
4320 case offsetofend(struct tcp_zerocopy_receive, msg_controllen):
4321 case offsetofend(struct tcp_zerocopy_receive, msg_control):
4322 case offsetofend(struct tcp_zerocopy_receive, flags):
4323 case offsetofend(struct tcp_zerocopy_receive, copybuf_len):
4324 case offsetofend(struct tcp_zerocopy_receive, copybuf_address):
33946518
AR		 4325 case offsetofend(struct tcp_zerocopy_receive, err):
4326 goto zerocopy_rcv_sk_err;
c8856c05
AR		 4327 case offsetofend(struct tcp_zerocopy_receive, inq):
4328 goto zerocopy_rcv_inq;
4329 case offsetofend(struct tcp_zerocopy_receive, length):
4330 default:
4331 goto zerocopy_rcv_out;
4332 }
7eeba170
AR		 4333zerocopy_rcv_cmsg:
4334 if (zc.msg_flags & TCP_CMSG_TS)
4335 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);
4336 else
4337 zc.msg_flags = 0;
33946518
AR		 4338zerocopy_rcv_sk_err:
4339 if (!err)
4340 zc.err = sock_error(sk);
c8856c05
AR		 4341zerocopy_rcv_inq:
4342 zc.inq = tcp_inq_hint(sk);
4343zerocopy_rcv_out:
34704ef0 4344 if (!err && copy_to_sockptr(optval, &zc, len))
05255b82
ED		 4345 err = -EFAULT;
4346 return err;
4347 }
4348#endif
faadfaba 4349 case TCP_AO_REPAIR:
965c00e4
DS		 4350 if (!tcp_can_repair_sock(sk))
4351 return -EPERM;
faadfaba 4352 return tcp_ao_get_repair(sk, optval, optlen);
ef84703a
DS		 4353 case TCP_AO_GET_KEYS:
4354 case TCP_AO_INFO: {
4355 int err;
4356
4357 sockopt_lock_sock(sk);
4358 if (optname == TCP_AO_GET_KEYS)
4359 err = tcp_ao_get_mkts(sk, optval, optlen);
4360 else
4361 err = tcp_ao_get_sock_info(sk, optval, optlen);
4362 sockopt_release_sock(sk);
4363
4364 return err;
4365 }
c084ebd7
MBN		 4366 case TCP_IS_MPTCP:
4367 val = 0;
4368 break;
1da177e4
LT		 4369 default:
4370 return -ENOPROTOOPT;
3ff50b79 4371 }
1da177e4 4372
34704ef0 4373 if (copy_to_sockptr(optlen, &len, sizeof(int)))
1da177e4 4374 return -EFAULT;
34704ef0 4375 if (copy_to_sockptr(optval, &val, len))
1da177e4
LT		 4376 return -EFAULT;
4377 return 0;
4378}
4379
9cacf81f
SF		 4380bool tcp_bpf_bypass_getsockopt(int level, int optname)
4381{
4382 /* TCP do_tcp_getsockopt has optimized getsockopt implementation
4383 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4384 */
4385 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4386 return true;
4387
4388 return false;
4389}
4390EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt);
4391
3fdadf7d
DM		 4392int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
4393 int __user *optlen)
4394{
4395 struct inet_connection_sock *icsk = inet_csk(sk);
4396
4397 if (level != SOL_TCP)
f49cd2f4
KI		 4398 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4399 return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
4400 optval, optlen);
34704ef0
MKL		 4401 return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
4402 USER_SOCKPTR(optlen));
3fdadf7d 4403}
4bc2f18b 4404EXPORT_SYMBOL(tcp_getsockopt);
3fdadf7d 4405
cfb6eeb4 4406#ifdef CONFIG_TCP_MD5SIG
8c73b263
DS		 4407int tcp_md5_sigpool_id = -1;
4408EXPORT_SYMBOL_GPL(tcp_md5_sigpool_id);
cfb6eeb4 4409
8c73b263 4410int tcp_md5_alloc_sigpool(void)
cfb6eeb4 4411{
8c73b263
DS		 4412 size_t scratch_size;
4413 int ret;
71cea17e 4414
8c73b263
DS		 4415 scratch_size = sizeof(union tcp_md5sum_block) + sizeof(struct tcphdr);
4416 ret = tcp_sigpool_alloc_ahash("md5", scratch_size);
4417 if (ret >= 0) {
4418 /* As long as any md5 sigpool was allocated, the return
4419 * id would stay the same. Re-write the id only for the case
4420 * when previously all MD5 keys were deleted and this call
4421 * allocates the first MD5 key, which may return a different
4422 * sigpool id than was used previously.
4423 */
4424 WRITE_ONCE(tcp_md5_sigpool_id, ret); /* Avoids the compiler potentially being smart here */
4425 return 0;
cfb6eeb4 4426 }
8c73b263 4427 return ret;
cfb6eeb4 4428}
35790c04 4429
8c73b263 4430void tcp_md5_release_sigpool(void)
cfb6eeb4 4431{
8c73b263 4432 tcp_sigpool_release(READ_ONCE(tcp_md5_sigpool_id));
35790c04 4433}
cfb6eeb4 4434
8c73b263 4435void tcp_md5_add_sigpool(void)
49a72dfb 4436{
8c73b263 4437 tcp_sigpool_get(READ_ONCE(tcp_md5_sigpool_id));
49a72dfb 4438}
49a72dfb 4439
8c73b263
DS		 4440int tcp_md5_hash_key(struct tcp_sigpool *hp,
4441 const struct tcp_md5sig_key *key)
49a72dfb 4442{
e6ced831 4443 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
49a72dfb
AL		 4444 struct scatterlist sg;
4445
6a2febec 4446 sg_init_one(&sg, key->key, keylen);
8c73b263 4447 ahash_request_set_crypt(hp->req, &sg, NULL, keylen);
e6ced831 4448
8c73b263
DS		 4449 /* We use data_race() because tcp_md5_do_add() might change
4450 * key->key under us
4451 */
4452 return data_race(crypto_ahash_update(hp->req));
49a72dfb 4453}
49a72dfb
AL		 4454EXPORT_SYMBOL(tcp_md5_hash_key);
4455
7bbb765b 4456/* Called with rcu_read_lock() */
1330b6ef
JK		 4457enum skb_drop_reason
4458tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4459 const void *saddr, const void *daddr,
0a3a8090 4460 int family, int l3index, const __u8 *hash_location)
7bbb765b 4461{
0a3a8090 4462 /* This gets called for each TCP segment that has TCP-MD5 option.
7bbb765b
DS		 4463 * We have 3 drop cases:
4464 * o No MD5 hash and one expected.
4465 * o MD5 hash and we're not expecting one.
4466 * o MD5 hash and its wrong.
4467 */
e9d9da91 4468 const struct tcp_sock *tp = tcp_sk(sk);
0a3a8090 4469 struct tcp_md5sig_key *key;
7bbb765b 4470 u8 newhash[16];
0a3a8090 4471 int genhash;
7bbb765b 4472
0a3a8090 4473 key = tcp_md5_do_lookup(sk, l3index, saddr, family);
7bbb765b 4474
0a3a8090 4475 if (!key && hash_location) {
7bbb765b 4476 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
2717b5ad 4477 tcp_hash_fail("Unexpected MD5 Hash found", family, skb, "");
1330b6ef 4478 return SKB_DROP_REASON_TCP_MD5UNEXPECTED;
7bbb765b
DS		 4479 }
4480
e62d2e11
ED		 4481 /* Check the signature.
4482 * To support dual stack listeners, we need to handle
4483 * IPv4-mapped case.
4484 */
4485 if (family == AF_INET)
0a3a8090 4486 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
e62d2e11 4487 else
0a3a8090 4488 genhash = tp->af_specific->calc_md5_hash(newhash, key,
e62d2e11 4489 NULL, skb);
7bbb765b 4490 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
7bbb765b
DS		 4491 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
4492 if (family == AF_INET) {
2717b5ad
DS		 4493 tcp_hash_fail("MD5 Hash failed", AF_INET, skb, "%s L3 index %d",
4494 genhash ? "tcp_v4_calc_md5_hash failed"
4495 : "", l3index);
7bbb765b 4496 } else {
2717b5ad
DS		 4497 if (genhash) {
4498 tcp_hash_fail("MD5 Hash failed",
4499 AF_INET6, skb, "L3 index %d",
4500 l3index);
4501 } else {
4502 tcp_hash_fail("MD5 Hash mismatch",
4503 AF_INET6, skb, "L3 index %d",
4504 l3index);
4505 }
7bbb765b 4506 }
1330b6ef 4507 return SKB_DROP_REASON_TCP_MD5FAILURE;
7bbb765b 4508 }
1330b6ef 4509 return SKB_NOT_DROPPED_YET;
7bbb765b
DS		 4510}
4511EXPORT_SYMBOL(tcp_inbound_md5_hash);
4512
cfb6eeb4
YH		 4513#endif
4514
4ac02bab
AK		 4515void tcp_done(struct sock *sk)
4516{
d983ea6f 4517 struct request_sock *req;
8336886f 4518
cab209e5
ED		 4519 /* We might be called with a new socket, after
4520 * inet_csk_prepare_forced_close() has been called
4521 * so we can not use lockdep_sock_is_held(sk)
4522 */
4523 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
8336886f 4524
5a5f3a8d 4525 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
c10d9310 4526 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
4ac02bab
AK		 4527
4528 tcp_set_state(sk, TCP_CLOSE);
4529 tcp_clear_xmit_timers(sk);
00db4124 4530 if (req)
8336886f 4531 reqsk_fastopen_remove(sk, req, false);
4ac02bab 4532
e14cadfd 4533 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
4ac02bab
AK		 4534
4535 if (!sock_flag(sk, SOCK_DEAD))
4536 sk->sk_state_change(sk);
4537 else
4538 inet_csk_destroy_sock(sk);
4539}
4540EXPORT_SYMBOL_GPL(tcp_done);
4541
c1e64e29
LC		 4542int tcp_abort(struct sock *sk, int err)
4543{
af9784d0 4544 int state = inet_sk_state_load(sk);
07f6f4a3 4545
af9784d0
ED		 4546 if (state == TCP_NEW_SYN_RECV) {
4547 struct request_sock *req = inet_reqsk(sk);
4548
4549 local_bh_disable();
4550 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
4551 local_bh_enable();
4552 return 0;
4553 }
4554 if (state == TCP_TIME_WAIT) {
4555 struct inet_timewait_sock *tw = inet_twsk(sk);
4556
4557 refcount_inc(&tw->tw_refcnt);
4558 local_bh_disable();
4559 inet_twsk_deschedule_put(tw);
4560 local_bh_enable();
4561 return 0;
c1e64e29
LC		 4562 }
4563
4ddbcb88
AG		 4564 /* BPF context ensures sock locking. */
4565 if (!has_current_bpf_ctx())
4566 /* Don't race with userspace socket closes such as tcp_close. */
4567 lock_sock(sk);
c1e64e29 4568
2010b93e
LC		 4569 if (sk->sk_state == TCP_LISTEN) {
4570 tcp_set_state(sk, TCP_CLOSE);
4571 inet_csk_listen_stop(sk);
4572 }
4573
c1e64e29
LC		 4574 /* Don't race with BH socket closes such as inet_csk_listen_stop. */
4575 local_bh_disable();
4576 bh_lock_sock(sk);
4577
4578 if (!sock_flag(sk, SOCK_DEAD)) {
e13ec3da 4579 WRITE_ONCE(sk->sk_err, err);
c1e64e29
LC		 4580 /* This barrier is coupled with smp_rmb() in tcp_poll() */
4581 smp_wmb();
e3ae2365 4582 sk_error_report(sk);
c1e64e29 4583 if (tcp_need_reset(sk->sk_state))
5691276b
JX		 4584 tcp_send_active_reset(sk, GFP_ATOMIC,
4585 SK_RST_REASON_NOT_SPECIFIED);
c1e64e29
LC		 4586 tcp_done(sk);
4587 }
4588
4589 bh_unlock_sock(sk);
4590 local_bh_enable();
e05836ac 4591 tcp_write_queue_purge(sk);
4ddbcb88
AG		 4592 if (!has_current_bpf_ctx())
4593 release_sock(sk);
c1e64e29
LC		 4594 return 0;
4595}
4596EXPORT_SYMBOL_GPL(tcp_abort);
4597
5f8ef48d 4598extern struct tcp_congestion_ops tcp_reno;
1da177e4
LT		 4599
4600static __initdata unsigned long thash_entries;
4601static int __init set_thash_entries(char *str)
4602{
413c27d8
EZ		 4603 ssize_t ret;
4604
1da177e4
LT		 4605 if (!str)
4606 return 0;
413c27d8
EZ		 4607
4608 ret = kstrtoul(str, 0, &thash_entries);
4609 if (ret)
4610 return 0;
4611
1da177e4
LT		 4612 return 1;
4613}
4614__setup("thash_entries=", set_thash_entries);
4615
47d7a88c 4616static void __init tcp_init_mem(void)
4acb4190 4617{
b66e91cc
ED		 4618 unsigned long limit = nr_free_buffer_pages() / 16;
4619
4acb4190 4620 limit = max(limit, 128UL);
b66e91cc
ED		 4621 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */
4622 sysctl_tcp_mem[1] = limit; /* 6.25 % */
4623 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */
4acb4190
GC		 4624}
4625
d5fed5ad
CL		 4626static void __init tcp_struct_check(void)
4627{
4628 /* TX read-mostly hotpath cache lines */
4629 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window);
4630 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh);
4631 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering);
4632 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat);
4633 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs);
4634 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, lost_skb_hint);
4635 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint);
4636 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_tx, 40);
4637
4638 /* TXRX read-mostly hotpath cache lines */
4639 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset);
4640 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd);
4641 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache);
4642 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd);
4643 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out);
4644 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out);
4645 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out);
119ff048
ED		 4646 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio);
4647 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_txrx, 32);
d5fed5ad
CL		 4648
4649 /* RX read-mostly hotpath cache lines */
4650 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
4651 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rcv_tstamp);
4652 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1);
4653 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq);
4654 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us);
4655 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out);
4656 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss);
4657 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data);
4658 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost);
4659 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min);
4660 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue);
4661 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
4662 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 69);
4663
4664 /* TX read-write hotpath cache lines */
4665 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
4666 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
4667 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
4668 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
4669 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
4670 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
4671 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
4672 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq);
4673 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime);
4674 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us);
4675 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
d5fed5ad
CL		 4676 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
4677 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
4678 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
4679 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
d2c3a7eb 4680 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 89);
d5fed5ad
CL		 4681
4682 /* TXRX read-write hotpath cache lines */
4683 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags);
d2c3a7eb
ED		 4684 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache);
4685 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp);
d5fed5ad
CL		 4686 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt);
4687 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt);
4688 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una);
4689 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp);
4690 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
4691 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
4692 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
4693 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
4694 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
4695 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
4696 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd);
4697 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
9b9fd458
ED		 4698
4699 /* 32bit arches with 8byte alignment on u64 fields might need padding
4700 * before tcp_clock_cache.
4701 */
4702 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92 + 4);
d5fed5ad
CL		 4703
4704 /* RX read-write hotpath cache lines */
4705 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
4706 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
4707 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
4708 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
4709 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
4710 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
4711 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
4712 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
4713 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
4714 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
4715 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
4716 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
4717 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
4718 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
4719 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_rx, 99);
4720}
4721
1da177e4
LT		 4722void __init tcp_init(void)
4723{
b49960a0 4724 int max_rshare, max_wshare, cnt;
b2d3ea4a 4725 unsigned long limit;
074b8517 4726 unsigned int i;
1da177e4 4727
3b4929f6 4728 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
b2d3ea4a 4729 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
c593642c 4730 sizeof_field(struct sk_buff, cb));
1da177e4 4731
d5fed5ad
CL		 4732 tcp_struct_check();
4733
908c7f19 4734 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
19757ceb
ED		 4735
4736 timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
4737 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
4738
27da6d37
MKL		 4739 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
4740 thash_entries, 21, /* one slot per 2 MB*/
4741 0, 64 * 1024);
6e04e021
ACM		 4742 tcp_hashinfo.bind_bucket_cachep =
4743 kmem_cache_create("tcp_bind_bucket",
4744 sizeof(struct inet_bind_bucket), 0,
990c74e3
VA		 4745 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4746 SLAB_ACCOUNT,
4747 NULL);
28044fc1
JK		 4748 tcp_hashinfo.bind2_bucket_cachep =
4749 kmem_cache_create("tcp_bind2_bucket",
4750 sizeof(struct inet_bind2_bucket), 0,
4751 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4752 SLAB_ACCOUNT,
4753 NULL);
1da177e4 4754
1da177e4
LT		 4755 /* Size and allocate the main established and bind bucket
4756 * hash tables.
4757 *
4758 * The methodology is similar to that of the buffer cache.
4759 */
6e04e021 4760 tcp_hashinfo.ehash =
1da177e4 4761 alloc_large_system_hash("TCP established",
0f7ff927 4762 sizeof(struct inet_ehash_bucket),
1da177e4 4763 thash_entries,
fd90b29d 4764 17, /* one slot per 128 KB of memory */
9e950efa 4765 0,
1da177e4 4766 NULL,
f373b53b 4767 &tcp_hashinfo.ehash_mask,
31fe62b9 4768 0,
0ccfe618 4769 thash_entries ? 0 : 512 * 1024);
05dbc7b5 4770 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3ab5aee7 4771 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
05dbc7b5 4772
230140cf
ED		 4773 if (inet_ehash_locks_alloc(&tcp_hashinfo))
4774 panic("TCP: failed to alloc ehash_locks");
6e04e021 4775 tcp_hashinfo.bhash =
593d1ebe 4776 alloc_large_system_hash("TCP bind",
28044fc1 4777 2 * sizeof(struct inet_bind_hashbucket),
f373b53b 4778 tcp_hashinfo.ehash_mask + 1,
fd90b29d 4779 17, /* one slot per 128 KB of memory */
9e950efa 4780 0,
6e04e021 4781 &tcp_hashinfo.bhash_size,
1da177e4 4782 NULL,
31fe62b9 4783 0,
1da177e4 4784 64 * 1024);
074b8517 4785 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
28044fc1 4786 tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
6e04e021
ACM		 4787 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
4788 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
4789 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
28044fc1
JK		 4790 spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
4791 INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
1da177e4
LT		 4792 }
4793
d1e5e640 4794 tcp_hashinfo.pernet = false;
c5ed63d6
ED		 4795
4796 cnt = tcp_hashinfo.ehash_mask + 1;
c5ed63d6 4797 sysctl_tcp_max_orphans = cnt / 2;
1da177e4 4798
a4fe34bf 4799 tcp_init_mem();
c43b874d 4800 /* Set per-socket limits to no more than 1/128 the pressure threshold */
5fb84b14 4801 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
b49960a0
ED		 4802 max_wshare = min(4UL*1024*1024, limit);
4803 max_rshare = min(6UL*1024*1024, limit);
7b4f4b5e 4804
100fdd1f 4805 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
356d1833
ED		 4806 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
4807 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
7b4f4b5e 4808
100fdd1f 4809 init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE;
a337531b
YC		 4810 init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
4811 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
1da177e4 4812
afd46503 4813 pr_info("Hash tables configured (established %u bind %u)\n",
058bd4d2 4814 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
317a76f9 4815
1946e672 4816 tcp_v4_init();
51c5d0c4 4817 tcp_metrics_init();
55d8694f 4818 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
46d3ceab 4819 tcp_tasklet_init();
f870fa0b 4820 mptcp_init();
1da177e4 4821}
Linux 6.x block layer and io_uring tree(s)