2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * Implementation of the Transmission Control Protocol(TCP).
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Mark Evans, <evansmp@uhura.aston.ac.uk>
11 * Corey Minyard <wf-rch!minyard@relay.EU.net>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14 * Linus Torvalds, <torvalds@cs.helsinki.fi>
15 * Alan Cox, <gw4pts@gw4pts.ampr.org>
16 * Matthew Dillon, <dillon@apollo.west.oic.com>
17 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 * Jorge Cwik, <jorge@laser.satlink.net>
22 * Changes: Pedro Roque : Retransmit queue handled by TCP.
23 * : Fragmentation on mtu decrease
24 * : Segment collapse on retransmit
27 * Linus Torvalds : send_delayed_ack
28 * David S. Miller : Charge memory using the right skb
29 * during syn/ack processing.
30 * David S. Miller : Output engine completely rewritten.
31 * Andrea Arcangeli: SYNACK carry ts_recent in tsecr.
32 * Cacophonix Gaul : draft-minshall-nagle-01
33 * J Hadi Salim : ECN support
37 #define pr_fmt(fmt) "TCP: " fmt
41 #include <linux/compiler.h>
42 #include <linux/gfp.h>
43 #include <linux/module.h>
45 /* People can turn this off for buggy TCP's found in printers etc. */
46 int sysctl_tcp_retrans_collapse __read_mostly = 1;
48 /* People can turn this on to work with those rare, broken TCPs that
49 * interpret the window field as a signed quantity.
51 int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
53 /* Default TSQ limit of two TSO segments */
54 int sysctl_tcp_limit_output_bytes __read_mostly = 131072;
56 /* This limits the percentage of the congestion window which we
57 * will allow a single TSO frame to consume. Building TSO frames
58 * which are too large can cause TCP streams to be bursty.
60 int sysctl_tcp_tso_win_divisor __read_mostly = 3;
62 int sysctl_tcp_mtu_probing __read_mostly = 0;
63 int sysctl_tcp_base_mss __read_mostly = TCP_BASE_MSS;
65 /* By default, RFC2861 behavior. */
66 int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
68 unsigned int sysctl_tcp_notsent_lowat __read_mostly = UINT_MAX;
69 EXPORT_SYMBOL(sysctl_tcp_notsent_lowat);
71 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
72 int push_one, gfp_t gfp);
74 /* Account for new data that has been sent to the network. */
75 static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb)
77 struct inet_connection_sock *icsk = inet_csk(sk);
78 struct tcp_sock *tp = tcp_sk(sk);
79 unsigned int prior_packets = tp->packets_out;
81 tcp_advance_send_head(sk, skb);
82 tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
84 tp->packets_out += tcp_skb_pcount(skb);
85 if (!prior_packets || icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
86 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
90 NET_ADD_STATS_BH(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
94 /* SND.NXT, if window was not shrunk.
95 * If window has been shrunk, what should we make? It is not clear at all.
96 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
97 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
98 * invalid. OK, let's make this for now:
100 static inline __u32 tcp_acceptable_seq(const struct sock *sk)
102 const struct tcp_sock *tp = tcp_sk(sk);
104 if (!before(tcp_wnd_end(tp), tp->snd_nxt))
107 return tcp_wnd_end(tp);
110 /* Calculate mss to advertise in SYN segment.
111 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
113 * 1. It is independent of path mtu.
114 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
115 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
116 * attached devices, because some buggy hosts are confused by
118 * 4. We do not make 3, we advertise MSS, calculated from first
119 * hop device mtu, but allow to raise it to ip_rt_min_advmss.
120 * This may be overridden via information stored in routing table.
121 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
122 * probably even Jumbo".
124 static __u16 tcp_advertise_mss(struct sock *sk)
126 struct tcp_sock *tp = tcp_sk(sk);
127 const struct dst_entry *dst = __sk_dst_get(sk);
128 int mss = tp->advmss;
131 unsigned int metric = dst_metric_advmss(dst);
142 /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
143 * This is the first part of cwnd validation mechanism. */
144 static void tcp_cwnd_restart(struct sock *sk, const struct dst_entry *dst)
146 struct tcp_sock *tp = tcp_sk(sk);
147 s32 delta = tcp_time_stamp - tp->lsndtime;
148 u32 restart_cwnd = tcp_init_cwnd(tp, dst);
149 u32 cwnd = tp->snd_cwnd;
151 tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
153 tp->snd_ssthresh = tcp_current_ssthresh(sk);
154 restart_cwnd = min(restart_cwnd, cwnd);
156 while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
158 tp->snd_cwnd = max(cwnd, restart_cwnd);
159 tp->snd_cwnd_stamp = tcp_time_stamp;
160 tp->snd_cwnd_used = 0;
163 /* Congestion state accounting after a packet has been sent. */
164 static void tcp_event_data_sent(struct tcp_sock *tp,
167 struct inet_connection_sock *icsk = inet_csk(sk);
168 const u32 now = tcp_time_stamp;
169 const struct dst_entry *dst = __sk_dst_get(sk);
171 if (sysctl_tcp_slow_start_after_idle &&
172 (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto))
173 tcp_cwnd_restart(sk, __sk_dst_get(sk));
177 /* If it is a reply for ato after last received
178 * packet, enter pingpong mode.
180 if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato &&
181 (!dst || !dst_metric(dst, RTAX_QUICKACK)))
182 icsk->icsk_ack.pingpong = 1;
185 /* Account for an ACK we sent. */
186 static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
188 tcp_dec_quickack_mode(sk, pkts);
189 inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
193 u32 tcp_default_init_rwnd(u32 mss)
195 /* Initial receive window should be twice of TCP_INIT_CWND to
196 * enable proper sending of new unsent data during fast recovery
197 * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a
198 * limit when mss is larger than 1460.
200 u32 init_rwnd = TCP_INIT_CWND * 2;
203 init_rwnd = max((1460 * init_rwnd) / mss, 2U);
207 /* Determine a window scaling and initial window to offer.
208 * Based on the assumption that the given amount of space
209 * will be offered. Store the results in the tp structure.
210 * NOTE: for smooth operation initial space offering should
211 * be a multiple of mss if possible. We assume here that mss >= 1.
212 * This MUST be enforced by all callers.
214 void tcp_select_initial_window(int __space, __u32 mss,
215 __u32 *rcv_wnd, __u32 *window_clamp,
216 int wscale_ok, __u8 *rcv_wscale,
219 unsigned int space = (__space < 0 ? 0 : __space);
221 /* If no clamp set the clamp to the max possible scaled window */
222 if (*window_clamp == 0)
223 (*window_clamp) = (65535 << 14);
224 space = min(*window_clamp, space);
226 /* Quantize space offering to a multiple of mss if possible. */
228 space = (space / mss) * mss;
230 /* NOTE: offering an initial window larger than 32767
231 * will break some buggy TCP stacks. If the admin tells us
232 * it is likely we could be speaking with such a buggy stack
233 * we will truncate our initial window offering to 32K-1
234 * unless the remote has sent us a window scaling option,
235 * which we interpret as a sign the remote TCP is not
236 * misinterpreting the window field as a signed quantity.
238 if (sysctl_tcp_workaround_signed_windows)
239 (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
245 /* Set window scaling on max possible window
246 * See RFC1323 for an explanation of the limit to 14
248 space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max);
249 space = min_t(u32, space, *window_clamp);
250 while (space > 65535 && (*rcv_wscale) < 14) {
256 if (mss > (1 << *rcv_wscale)) {
257 if (!init_rcv_wnd) /* Use default unless specified otherwise */
258 init_rcv_wnd = tcp_default_init_rwnd(mss);
259 *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
262 /* Set the clamp no higher than max representable value */
263 (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp);
265 EXPORT_SYMBOL(tcp_select_initial_window);
267 /* Chose a new window to advertise, update state in tcp_sock for the
268 * socket, and return result with RFC1323 scaling applied. The return
269 * value can be stuffed directly into th->window for an outgoing
272 static u16 tcp_select_window(struct sock *sk)
274 struct tcp_sock *tp = tcp_sk(sk);
275 u32 old_win = tp->rcv_wnd;
276 u32 cur_win = tcp_receive_window(tp);
277 u32 new_win = __tcp_select_window(sk);
279 /* Never shrink the offered window */
280 if (new_win < cur_win) {
281 /* Danger Will Robinson!
282 * Don't update rcv_wup/rcv_wnd here or else
283 * we will not be able to advertise a zero
284 * window in time. --DaveM
286 * Relax Will Robinson.
289 NET_INC_STATS(sock_net(sk),
290 LINUX_MIB_TCPWANTZEROWINDOWADV);
291 new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
293 tp->rcv_wnd = new_win;
294 tp->rcv_wup = tp->rcv_nxt;
296 /* Make sure we do not exceed the maximum possible
299 if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
300 new_win = min(new_win, MAX_TCP_WINDOW);
302 new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
304 /* RFC1323 scaling applied */
305 new_win >>= tp->rx_opt.rcv_wscale;
307 /* If we advertise zero window, disable fast path. */
311 NET_INC_STATS(sock_net(sk),
312 LINUX_MIB_TCPTOZEROWINDOWADV);
313 } else if (old_win == 0) {
314 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
320 /* Packet ECN state for a SYN-ACK */
321 static inline void TCP_ECN_send_synack(const struct tcp_sock *tp, struct sk_buff *skb)
323 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
324 if (!(tp->ecn_flags & TCP_ECN_OK))
325 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
328 /* Packet ECN state for a SYN. */
329 static inline void TCP_ECN_send_syn(struct sock *sk, struct sk_buff *skb)
331 struct tcp_sock *tp = tcp_sk(sk);
334 if (sock_net(sk)->ipv4.sysctl_tcp_ecn == 1) {
335 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
336 tp->ecn_flags = TCP_ECN_OK;
340 static __inline__ void
341 TCP_ECN_make_synack(const struct request_sock *req, struct tcphdr *th)
343 if (inet_rsk(req)->ecn_ok)
347 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
350 static inline void TCP_ECN_send(struct sock *sk, struct sk_buff *skb,
353 struct tcp_sock *tp = tcp_sk(sk);
355 if (tp->ecn_flags & TCP_ECN_OK) {
356 /* Not-retransmitted data segment: set ECT and inject CWR. */
357 if (skb->len != tcp_header_len &&
358 !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
360 if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
361 tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
362 tcp_hdr(skb)->cwr = 1;
363 skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
366 /* ACK or retransmitted segment: clear ECT|CE */
367 INET_ECN_dontxmit(sk);
369 if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
370 tcp_hdr(skb)->ece = 1;
374 /* Constructs common control bits of non-data skb. If SYN/FIN is present,
375 * auto increment end seqno.
377 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
379 struct skb_shared_info *shinfo = skb_shinfo(skb);
381 skb->ip_summed = CHECKSUM_PARTIAL;
384 TCP_SKB_CB(skb)->tcp_flags = flags;
385 TCP_SKB_CB(skb)->sacked = 0;
387 shinfo->gso_segs = 1;
388 shinfo->gso_size = 0;
389 shinfo->gso_type = 0;
391 TCP_SKB_CB(skb)->seq = seq;
392 if (flags & (TCPHDR_SYN | TCPHDR_FIN))
394 TCP_SKB_CB(skb)->end_seq = seq;
397 static inline bool tcp_urg_mode(const struct tcp_sock *tp)
399 return tp->snd_una != tp->snd_up;
402 #define OPTION_SACK_ADVERTISE (1 << 0)
403 #define OPTION_TS (1 << 1)
404 #define OPTION_MD5 (1 << 2)
405 #define OPTION_WSCALE (1 << 3)
406 #define OPTION_FAST_OPEN_COOKIE (1 << 8)
408 struct tcp_out_options {
409 u16 options; /* bit field of OPTION_* */
410 u16 mss; /* 0 to disable */
411 u8 ws; /* window scale, 0 to disable */
412 u8 num_sack_blocks; /* number of SACK blocks to include */
413 u8 hash_size; /* bytes in hash_location */
414 __u8 *hash_location; /* temporary pointer, overloaded */
415 __u32 tsval, tsecr; /* need to include OPTION_TS */
416 struct tcp_fastopen_cookie *fastopen_cookie; /* Fast open cookie */
419 /* Write previously computed TCP options to the packet.
421 * Beware: Something in the Internet is very sensitive to the ordering of
422 * TCP options, we learned this through the hard way, so be careful here.
423 * Luckily we can at least blame others for their non-compliance but from
424 * inter-operability perspective it seems that we're somewhat stuck with
425 * the ordering which we have been using if we want to keep working with
426 * those broken things (not that it currently hurts anybody as there isn't
427 * particular reason why the ordering would need to be changed).
429 * At least SACK_PERM as the first option is known to lead to a disaster
430 * (but it may well be that other scenarios fail similarly).
432 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
433 struct tcp_out_options *opts)
435 u16 options = opts->options; /* mungable copy */
437 if (unlikely(OPTION_MD5 & options)) {
438 *ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
439 (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
440 /* overload cookie hash location */
441 opts->hash_location = (__u8 *)ptr;
445 if (unlikely(opts->mss)) {
446 *ptr++ = htonl((TCPOPT_MSS << 24) |
447 (TCPOLEN_MSS << 16) |
451 if (likely(OPTION_TS & options)) {
452 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
453 *ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
454 (TCPOLEN_SACK_PERM << 16) |
455 (TCPOPT_TIMESTAMP << 8) |
457 options &= ~OPTION_SACK_ADVERTISE;
459 *ptr++ = htonl((TCPOPT_NOP << 24) |
461 (TCPOPT_TIMESTAMP << 8) |
464 *ptr++ = htonl(opts->tsval);
465 *ptr++ = htonl(opts->tsecr);
468 if (unlikely(OPTION_SACK_ADVERTISE & options)) {
469 *ptr++ = htonl((TCPOPT_NOP << 24) |
471 (TCPOPT_SACK_PERM << 8) |
475 if (unlikely(OPTION_WSCALE & options)) {
476 *ptr++ = htonl((TCPOPT_NOP << 24) |
477 (TCPOPT_WINDOW << 16) |
478 (TCPOLEN_WINDOW << 8) |
482 if (unlikely(opts->num_sack_blocks)) {
483 struct tcp_sack_block *sp = tp->rx_opt.dsack ?
484 tp->duplicate_sack : tp->selective_acks;
487 *ptr++ = htonl((TCPOPT_NOP << 24) |
490 (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
491 TCPOLEN_SACK_PERBLOCK)));
493 for (this_sack = 0; this_sack < opts->num_sack_blocks;
495 *ptr++ = htonl(sp[this_sack].start_seq);
496 *ptr++ = htonl(sp[this_sack].end_seq);
499 tp->rx_opt.dsack = 0;
502 if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
503 struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
505 *ptr++ = htonl((TCPOPT_EXP << 24) |
506 ((TCPOLEN_EXP_FASTOPEN_BASE + foc->len) << 16) |
507 TCPOPT_FASTOPEN_MAGIC);
509 memcpy(ptr, foc->val, foc->len);
510 if ((foc->len & 3) == 2) {
511 u8 *align = ((u8 *)ptr) + foc->len;
512 align[0] = align[1] = TCPOPT_NOP;
514 ptr += (foc->len + 3) >> 2;
518 /* Compute TCP options for SYN packets. This is not the final
519 * network wire format yet.
521 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
522 struct tcp_out_options *opts,
523 struct tcp_md5sig_key **md5)
525 struct tcp_sock *tp = tcp_sk(sk);
526 unsigned int remaining = MAX_TCP_OPTION_SPACE;
527 struct tcp_fastopen_request *fastopen = tp->fastopen_req;
529 #ifdef CONFIG_TCP_MD5SIG
530 *md5 = tp->af_specific->md5_lookup(sk, sk);
532 opts->options |= OPTION_MD5;
533 remaining -= TCPOLEN_MD5SIG_ALIGNED;
539 /* We always get an MSS option. The option bytes which will be seen in
540 * normal data packets should timestamps be used, must be in the MSS
541 * advertised. But we subtract them from tp->mss_cache so that
542 * calculations in tcp_sendmsg are simpler etc. So account for this
543 * fact here if necessary. If we don't do this correctly, as a
544 * receiver we won't recognize data packets as being full sized when we
545 * should, and thus we won't abide by the delayed ACK rules correctly.
546 * SACKs don't matter, we never delay an ACK when we have any of those
548 opts->mss = tcp_advertise_mss(sk);
549 remaining -= TCPOLEN_MSS_ALIGNED;
551 if (likely(sysctl_tcp_timestamps && *md5 == NULL)) {
552 opts->options |= OPTION_TS;
553 opts->tsval = TCP_SKB_CB(skb)->when + tp->tsoffset;
554 opts->tsecr = tp->rx_opt.ts_recent;
555 remaining -= TCPOLEN_TSTAMP_ALIGNED;
557 if (likely(sysctl_tcp_window_scaling)) {
558 opts->ws = tp->rx_opt.rcv_wscale;
559 opts->options |= OPTION_WSCALE;
560 remaining -= TCPOLEN_WSCALE_ALIGNED;
562 if (likely(sysctl_tcp_sack)) {
563 opts->options |= OPTION_SACK_ADVERTISE;
564 if (unlikely(!(OPTION_TS & opts->options)))
565 remaining -= TCPOLEN_SACKPERM_ALIGNED;
568 if (fastopen && fastopen->cookie.len >= 0) {
569 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + fastopen->cookie.len;
570 need = (need + 3) & ~3U; /* Align to 32 bits */
571 if (remaining >= need) {
572 opts->options |= OPTION_FAST_OPEN_COOKIE;
573 opts->fastopen_cookie = &fastopen->cookie;
575 tp->syn_fastopen = 1;
579 return MAX_TCP_OPTION_SPACE - remaining;
582 /* Set up TCP options for SYN-ACKs. */
583 static unsigned int tcp_synack_options(struct sock *sk,
584 struct request_sock *req,
585 unsigned int mss, struct sk_buff *skb,
586 struct tcp_out_options *opts,
587 struct tcp_md5sig_key **md5,
588 struct tcp_fastopen_cookie *foc)
590 struct inet_request_sock *ireq = inet_rsk(req);
591 unsigned int remaining = MAX_TCP_OPTION_SPACE;
593 #ifdef CONFIG_TCP_MD5SIG
594 *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req);
596 opts->options |= OPTION_MD5;
597 remaining -= TCPOLEN_MD5SIG_ALIGNED;
599 /* We can't fit any SACK blocks in a packet with MD5 + TS
600 * options. There was discussion about disabling SACK
601 * rather than TS in order to fit in better with old,
602 * buggy kernels, but that was deemed to be unnecessary.
604 ireq->tstamp_ok &= !ireq->sack_ok;
610 /* We always send an MSS option. */
612 remaining -= TCPOLEN_MSS_ALIGNED;
614 if (likely(ireq->wscale_ok)) {
615 opts->ws = ireq->rcv_wscale;
616 opts->options |= OPTION_WSCALE;
617 remaining -= TCPOLEN_WSCALE_ALIGNED;
619 if (likely(ireq->tstamp_ok)) {
620 opts->options |= OPTION_TS;
621 opts->tsval = TCP_SKB_CB(skb)->when;
622 opts->tsecr = req->ts_recent;
623 remaining -= TCPOLEN_TSTAMP_ALIGNED;
625 if (likely(ireq->sack_ok)) {
626 opts->options |= OPTION_SACK_ADVERTISE;
627 if (unlikely(!ireq->tstamp_ok))
628 remaining -= TCPOLEN_SACKPERM_ALIGNED;
631 u32 need = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
632 need = (need + 3) & ~3U; /* Align to 32 bits */
633 if (remaining >= need) {
634 opts->options |= OPTION_FAST_OPEN_COOKIE;
635 opts->fastopen_cookie = foc;
640 return MAX_TCP_OPTION_SPACE - remaining;
643 /* Compute TCP options for ESTABLISHED sockets. This is not the
644 * final wire format yet.
646 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
647 struct tcp_out_options *opts,
648 struct tcp_md5sig_key **md5)
650 struct tcp_skb_cb *tcb = skb ? TCP_SKB_CB(skb) : NULL;
651 struct tcp_sock *tp = tcp_sk(sk);
652 unsigned int size = 0;
653 unsigned int eff_sacks;
657 #ifdef CONFIG_TCP_MD5SIG
658 *md5 = tp->af_specific->md5_lookup(sk, sk);
659 if (unlikely(*md5)) {
660 opts->options |= OPTION_MD5;
661 size += TCPOLEN_MD5SIG_ALIGNED;
667 if (likely(tp->rx_opt.tstamp_ok)) {
668 opts->options |= OPTION_TS;
669 opts->tsval = tcb ? tcb->when + tp->tsoffset : 0;
670 opts->tsecr = tp->rx_opt.ts_recent;
671 size += TCPOLEN_TSTAMP_ALIGNED;
674 eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
675 if (unlikely(eff_sacks)) {
676 const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
677 opts->num_sack_blocks =
678 min_t(unsigned int, eff_sacks,
679 (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
680 TCPOLEN_SACK_PERBLOCK);
681 size += TCPOLEN_SACK_BASE_ALIGNED +
682 opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
689 /* TCP SMALL QUEUES (TSQ)
691 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
692 * to reduce RTT and bufferbloat.
693 * We do this using a special skb destructor (tcp_wfree).
695 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
696 * needs to be reallocated in a driver.
697 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
699 * Since transmit from skb destructor is forbidden, we use a tasklet
700 * to process all sockets that eventually need to send more skbs.
701 * We use one tasklet per cpu, with its own queue of sockets.
704 struct tasklet_struct tasklet;
705 struct list_head head; /* queue of tcp sockets */
707 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
709 static void tcp_tsq_handler(struct sock *sk)
711 if ((1 << sk->sk_state) &
712 (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
713 TCPF_CLOSE_WAIT | TCPF_LAST_ACK))
714 tcp_write_xmit(sk, tcp_current_mss(sk), tcp_sk(sk)->nonagle,
718 * One tasklet per cpu tries to send more skbs.
719 * We run in tasklet context but need to disable irqs when
720 * transferring tsq->head because tcp_wfree() might
721 * interrupt us (non NAPI drivers)
723 static void tcp_tasklet_func(unsigned long data)
725 struct tsq_tasklet *tsq = (struct tsq_tasklet *)data;
728 struct list_head *q, *n;
732 local_irq_save(flags);
733 list_splice_init(&tsq->head, &list);
734 local_irq_restore(flags);
736 list_for_each_safe(q, n, &list) {
737 tp = list_entry(q, struct tcp_sock, tsq_node);
738 list_del(&tp->tsq_node);
740 sk = (struct sock *)tp;
743 if (!sock_owned_by_user(sk)) {
746 /* defer the work to tcp_release_cb() */
747 set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
751 clear_bit(TSQ_QUEUED, &tp->tsq_flags);
756 #define TCP_DEFERRED_ALL ((1UL << TCP_TSQ_DEFERRED) | \
757 (1UL << TCP_WRITE_TIMER_DEFERRED) | \
758 (1UL << TCP_DELACK_TIMER_DEFERRED) | \
759 (1UL << TCP_MTU_REDUCED_DEFERRED))
761 * tcp_release_cb - tcp release_sock() callback
764 * called from release_sock() to perform protocol dependent
765 * actions before socket release.
767 void tcp_release_cb(struct sock *sk)
769 struct tcp_sock *tp = tcp_sk(sk);
770 unsigned long flags, nflags;
772 /* perform an atomic operation only if at least one flag is set */
774 flags = tp->tsq_flags;
775 if (!(flags & TCP_DEFERRED_ALL))
777 nflags = flags & ~TCP_DEFERRED_ALL;
778 } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
780 if (flags & (1UL << TCP_TSQ_DEFERRED))
783 if (flags & (1UL << TCP_WRITE_TIMER_DEFERRED)) {
784 tcp_write_timer_handler(sk);
787 if (flags & (1UL << TCP_DELACK_TIMER_DEFERRED)) {
788 tcp_delack_timer_handler(sk);
791 if (flags & (1UL << TCP_MTU_REDUCED_DEFERRED)) {
792 sk->sk_prot->mtu_reduced(sk);
796 EXPORT_SYMBOL(tcp_release_cb);
798 void __init tcp_tasklet_init(void)
802 for_each_possible_cpu(i) {
803 struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
805 INIT_LIST_HEAD(&tsq->head);
806 tasklet_init(&tsq->tasklet,
813 * Write buffer destructor automatically called from kfree_skb.
814 * We can't xmit new skbs from this context, as we might already
817 void tcp_wfree(struct sk_buff *skb)
819 struct sock *sk = skb->sk;
820 struct tcp_sock *tp = tcp_sk(sk);
822 if (test_and_clear_bit(TSQ_THROTTLED, &tp->tsq_flags) &&
823 !test_and_set_bit(TSQ_QUEUED, &tp->tsq_flags)) {
825 struct tsq_tasklet *tsq;
827 /* Keep a ref on socket.
828 * This last ref will be released in tcp_tasklet_func()
830 atomic_sub(skb->truesize - 1, &sk->sk_wmem_alloc);
832 /* queue this socket to tasklet queue */
833 local_irq_save(flags);
834 tsq = &__get_cpu_var(tsq_tasklet);
835 list_add(&tp->tsq_node, &tsq->head);
836 tasklet_schedule(&tsq->tasklet);
837 local_irq_restore(flags);
843 /* This routine actually transmits TCP packets queued in by
844 * tcp_do_sendmsg(). This is used by both the initial
845 * transmission and possible later retransmissions.
846 * All SKB's seen here are completely headerless. It is our
847 * job to build the TCP header, and pass the packet down to
848 * IP so it can do the same plus pass the packet off to the
851 * We are working here with either a clone of the original
852 * SKB, or a fresh unique copy made by the retransmit engine.
854 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
857 const struct inet_connection_sock *icsk = inet_csk(sk);
858 struct inet_sock *inet;
860 struct tcp_skb_cb *tcb;
861 struct tcp_out_options opts;
862 unsigned int tcp_options_size, tcp_header_size;
863 struct tcp_md5sig_key *md5;
867 BUG_ON(!skb || !tcp_skb_pcount(skb));
870 const struct sk_buff *fclone = skb + 1;
872 skb_mstamp_get(&skb->skb_mstamp);
874 if (unlikely(skb->fclone == SKB_FCLONE_ORIG &&
875 fclone->fclone == SKB_FCLONE_CLONE))
876 NET_INC_STATS_BH(sock_net(sk),
877 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
879 if (unlikely(skb_cloned(skb)))
880 skb = pskb_copy(skb, gfp_mask);
882 skb = skb_clone(skb, gfp_mask);
889 tcb = TCP_SKB_CB(skb);
890 memset(&opts, 0, sizeof(opts));
892 if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
893 tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
895 tcp_options_size = tcp_established_options(sk, skb, &opts,
897 tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
899 if (tcp_packets_in_flight(tp) == 0)
900 tcp_ca_event(sk, CA_EVENT_TX_START);
902 /* if no packet is in qdisc/device queue, then allow XPS to select
905 skb->ooo_okay = sk_wmem_alloc_get(sk) == 0;
907 skb_push(skb, tcp_header_size);
908 skb_reset_transport_header(skb);
912 skb->destructor = tcp_wfree;
913 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
915 /* Build TCP header and checksum it. */
917 th->source = inet->inet_sport;
918 th->dest = inet->inet_dport;
919 th->seq = htonl(tcb->seq);
920 th->ack_seq = htonl(tp->rcv_nxt);
921 *(((__be16 *)th) + 6) = htons(((tcp_header_size >> 2) << 12) |
924 if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
925 /* RFC1323: The window in SYN & SYN/ACK segments
928 th->window = htons(min(tp->rcv_wnd, 65535U));
930 th->window = htons(tcp_select_window(sk));
935 /* The urg_mode check is necessary during a below snd_una win probe */
936 if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
937 if (before(tp->snd_up, tcb->seq + 0x10000)) {
938 th->urg_ptr = htons(tp->snd_up - tcb->seq);
940 } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
941 th->urg_ptr = htons(0xFFFF);
946 tcp_options_write((__be32 *)(th + 1), tp, &opts);
947 if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0))
948 TCP_ECN_send(sk, skb, tcp_header_size);
950 #ifdef CONFIG_TCP_MD5SIG
951 /* Calculate the MD5 hash, as we have all we need now */
953 sk_nocaps_add(sk, NETIF_F_GSO_MASK);
954 tp->af_specific->calc_md5_hash(opts.hash_location,
959 icsk->icsk_af_ops->send_check(sk, skb);
961 if (likely(tcb->tcp_flags & TCPHDR_ACK))
962 tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
964 if (skb->len != tcp_header_size)
965 tcp_event_data_sent(tp, sk);
967 if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
968 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
969 tcp_skb_pcount(skb));
971 err = icsk->icsk_af_ops->queue_xmit(skb, &inet->cork.fl);
972 if (likely(err <= 0))
975 tcp_enter_cwr(sk, 1);
977 return net_xmit_eval(err);
980 /* This routine just queues the buffer for sending.
982 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
983 * otherwise socket can stall.
985 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
987 struct tcp_sock *tp = tcp_sk(sk);
989 /* Advance write_seq and place onto the write_queue. */
990 tp->write_seq = TCP_SKB_CB(skb)->end_seq;
991 skb_header_release(skb);
992 tcp_add_write_queue_tail(sk, skb);
993 sk->sk_wmem_queued += skb->truesize;
994 sk_mem_charge(sk, skb->truesize);
997 /* Initialize TSO segments for a packet. */
998 static void tcp_set_skb_tso_segs(const struct sock *sk, struct sk_buff *skb,
999 unsigned int mss_now)
1001 struct skb_shared_info *shinfo = skb_shinfo(skb);
1003 /* Make sure we own this skb before messing gso_size/gso_segs */
1004 WARN_ON_ONCE(skb_cloned(skb));
1006 if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) {
1007 /* Avoid the costly divide in the normal
1010 shinfo->gso_segs = 1;
1011 shinfo->gso_size = 0;
1012 shinfo->gso_type = 0;
1014 shinfo->gso_segs = DIV_ROUND_UP(skb->len, mss_now);
1015 shinfo->gso_size = mss_now;
1016 shinfo->gso_type = sk->sk_gso_type;
1020 /* When a modification to fackets out becomes necessary, we need to check
1021 * skb is counted to fackets_out or not.
1023 static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb,
1026 struct tcp_sock *tp = tcp_sk(sk);
1028 if (!tp->sacked_out || tcp_is_reno(tp))
1031 if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq))
1032 tp->fackets_out -= decr;
1035 /* Pcount in the middle of the write queue got changed, we need to do various
1036 * tweaks to fix counters
1038 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1040 struct tcp_sock *tp = tcp_sk(sk);
1042 tp->packets_out -= decr;
1044 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1045 tp->sacked_out -= decr;
1046 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1047 tp->retrans_out -= decr;
1048 if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1049 tp->lost_out -= decr;
1051 /* Reno case is special. Sigh... */
1052 if (tcp_is_reno(tp) && decr > 0)
1053 tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1055 tcp_adjust_fackets_out(sk, skb, decr);
1057 if (tp->lost_skb_hint &&
1058 before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1059 (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)))
1060 tp->lost_cnt_hint -= decr;
1062 tcp_verify_left_out(tp);
1065 /* Function to create two new TCP segments. Shrinks the given segment
1066 * to the specified size and appends a new segment with the rest of the
1067 * packet to the list. This won't be called frequently, I hope.
1068 * Remember, these are still headerless SKBs at this point.
1070 int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
1071 unsigned int mss_now)
1073 struct tcp_sock *tp = tcp_sk(sk);
1074 struct sk_buff *buff;
1075 int nsize, old_factor;
1079 if (WARN_ON(len > skb->len))
1082 nsize = skb_headlen(skb) - len;
1086 if (skb_unclone(skb, GFP_ATOMIC))
1089 /* Get a new skb... force flag on. */
1090 buff = sk_stream_alloc_skb(sk, nsize, GFP_ATOMIC);
1092 return -ENOMEM; /* We'll just try again later. */
1094 sk->sk_wmem_queued += buff->truesize;
1095 sk_mem_charge(sk, buff->truesize);
1096 nlen = skb->len - len - nsize;
1097 buff->truesize += nlen;
1098 skb->truesize -= nlen;
1100 /* Correct the sequence numbers. */
1101 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1102 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1103 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1105 /* PSH and FIN should only be set in the second packet. */
1106 flags = TCP_SKB_CB(skb)->tcp_flags;
1107 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1108 TCP_SKB_CB(buff)->tcp_flags = flags;
1109 TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1111 if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
1112 /* Copy and checksum data tail into the new buffer. */
1113 buff->csum = csum_partial_copy_nocheck(skb->data + len,
1114 skb_put(buff, nsize),
1119 skb->csum = csum_block_sub(skb->csum, buff->csum, len);
1121 skb->ip_summed = CHECKSUM_PARTIAL;
1122 skb_split(skb, buff, len);
1125 buff->ip_summed = skb->ip_summed;
1127 /* Looks stupid, but our code really uses when of
1128 * skbs, which it never sent before. --ANK
1130 TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
1131 buff->tstamp = skb->tstamp;
1133 old_factor = tcp_skb_pcount(skb);
1135 /* Fix up tso_factor for both original and new SKB. */
1136 tcp_set_skb_tso_segs(sk, skb, mss_now);
1137 tcp_set_skb_tso_segs(sk, buff, mss_now);
1139 /* If this packet has been sent out already, we must
1140 * adjust the various packet counters.
1142 if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1143 int diff = old_factor - tcp_skb_pcount(skb) -
1144 tcp_skb_pcount(buff);
1147 tcp_adjust_pcount(sk, skb, diff);
1150 /* Link BUFF into the send queue. */
1151 skb_header_release(buff);
1152 tcp_insert_write_queue_after(skb, buff, sk);
1157 /* This is similar to __pskb_pull_head() (it will go to core/skbuff.c
1158 * eventually). The difference is that pulled data not copied, but
1159 * immediately discarded.
1161 static void __pskb_trim_head(struct sk_buff *skb, int len)
1163 struct skb_shared_info *shinfo;
1166 eat = min_t(int, len, skb_headlen(skb));
1168 __skb_pull(skb, eat);
1175 shinfo = skb_shinfo(skb);
1176 for (i = 0; i < shinfo->nr_frags; i++) {
1177 int size = skb_frag_size(&shinfo->frags[i]);
1180 skb_frag_unref(skb, i);
1183 shinfo->frags[k] = shinfo->frags[i];
1185 shinfo->frags[k].page_offset += eat;
1186 skb_frag_size_sub(&shinfo->frags[k], eat);
1192 shinfo->nr_frags = k;
1194 skb_reset_tail_pointer(skb);
1195 skb->data_len -= len;
1196 skb->len = skb->data_len;
1199 /* Remove acked data from a packet in the transmit queue. */
1200 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1202 if (skb_unclone(skb, GFP_ATOMIC))
1205 __pskb_trim_head(skb, len);
1207 TCP_SKB_CB(skb)->seq += len;
1208 skb->ip_summed = CHECKSUM_PARTIAL;
1210 skb->truesize -= len;
1211 sk->sk_wmem_queued -= len;
1212 sk_mem_uncharge(sk, len);
1213 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1215 /* Any change of skb->len requires recalculation of tso factor. */
1216 if (tcp_skb_pcount(skb) > 1)
1217 tcp_set_skb_tso_segs(sk, skb, tcp_skb_mss(skb));
1222 /* Calculate MSS not accounting any TCP options. */
1223 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1225 const struct tcp_sock *tp = tcp_sk(sk);
1226 const struct inet_connection_sock *icsk = inet_csk(sk);
1229 /* Calculate base mss without TCP options:
1230 It is MMS_S - sizeof(tcphdr) of rfc1122
1232 mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1234 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1235 if (icsk->icsk_af_ops->net_frag_header_len) {
1236 const struct dst_entry *dst = __sk_dst_get(sk);
1238 if (dst && dst_allfrag(dst))
1239 mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1242 /* Clamp it (mss_clamp does not include tcp options) */
1243 if (mss_now > tp->rx_opt.mss_clamp)
1244 mss_now = tp->rx_opt.mss_clamp;
1246 /* Now subtract optional transport overhead */
1247 mss_now -= icsk->icsk_ext_hdr_len;
1249 /* Then reserve room for full set of TCP options and 8 bytes of data */
1255 /* Calculate MSS. Not accounting for SACKs here. */
1256 int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1258 /* Subtract TCP options size, not including SACKs */
1259 return __tcp_mtu_to_mss(sk, pmtu) -
1260 (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1263 /* Inverse of above */
1264 int tcp_mss_to_mtu(struct sock *sk, int mss)
1266 const struct tcp_sock *tp = tcp_sk(sk);
1267 const struct inet_connection_sock *icsk = inet_csk(sk);
1271 tp->tcp_header_len +
1272 icsk->icsk_ext_hdr_len +
1273 icsk->icsk_af_ops->net_header_len;
1275 /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1276 if (icsk->icsk_af_ops->net_frag_header_len) {
1277 const struct dst_entry *dst = __sk_dst_get(sk);
1279 if (dst && dst_allfrag(dst))
1280 mtu += icsk->icsk_af_ops->net_frag_header_len;
1285 /* MTU probing init per socket */
1286 void tcp_mtup_init(struct sock *sk)
1288 struct tcp_sock *tp = tcp_sk(sk);
1289 struct inet_connection_sock *icsk = inet_csk(sk);
1291 icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1;
1292 icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1293 icsk->icsk_af_ops->net_header_len;
1294 icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss);
1295 icsk->icsk_mtup.probe_size = 0;
1297 EXPORT_SYMBOL(tcp_mtup_init);
1299 /* This function synchronize snd mss to current pmtu/exthdr set.
1301 tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1302 for TCP options, but includes only bare TCP header.
1304 tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1305 It is minimum of user_mss and mss received with SYN.
1306 It also does not include TCP options.
1308 inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1310 tp->mss_cache is current effective sending mss, including
1311 all tcp options except for SACKs. It is evaluated,
1312 taking into account current pmtu, but never exceeds
1313 tp->rx_opt.mss_clamp.
1315 NOTE1. rfc1122 clearly states that advertised MSS
1316 DOES NOT include either tcp or ip options.
1318 NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1319 are READ ONLY outside this function. --ANK (980731)
1321 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1323 struct tcp_sock *tp = tcp_sk(sk);
1324 struct inet_connection_sock *icsk = inet_csk(sk);
1327 if (icsk->icsk_mtup.search_high > pmtu)
1328 icsk->icsk_mtup.search_high = pmtu;
1330 mss_now = tcp_mtu_to_mss(sk, pmtu);
1331 mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1333 /* And store cached results */
1334 icsk->icsk_pmtu_cookie = pmtu;
1335 if (icsk->icsk_mtup.enabled)
1336 mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1337 tp->mss_cache = mss_now;
1341 EXPORT_SYMBOL(tcp_sync_mss);
1343 /* Compute the current effective MSS, taking SACKs and IP options,
1344 * and even PMTU discovery events into account.
1346 unsigned int tcp_current_mss(struct sock *sk)
1348 const struct tcp_sock *tp = tcp_sk(sk);
1349 const struct dst_entry *dst = __sk_dst_get(sk);
1351 unsigned int header_len;
1352 struct tcp_out_options opts;
1353 struct tcp_md5sig_key *md5;
1355 mss_now = tp->mss_cache;
1358 u32 mtu = dst_mtu(dst);
1359 if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1360 mss_now = tcp_sync_mss(sk, mtu);
1363 header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1364 sizeof(struct tcphdr);
1365 /* The mss_cache is sized based on tp->tcp_header_len, which assumes
1366 * some common options. If this is an odd packet (because we have SACK
1367 * blocks etc) then our calculated header_len will be different, and
1368 * we have to adjust mss_now correspondingly */
1369 if (header_len != tp->tcp_header_len) {
1370 int delta = (int) header_len - tp->tcp_header_len;
1377 /* Congestion window validation. (RFC2861) */
1378 static void tcp_cwnd_validate(struct sock *sk)
1380 struct tcp_sock *tp = tcp_sk(sk);
1382 if (tp->packets_out >= tp->snd_cwnd) {
1383 /* Network is feed fully. */
1384 tp->snd_cwnd_used = 0;
1385 tp->snd_cwnd_stamp = tcp_time_stamp;
1387 /* Network starves. */
1388 if (tp->packets_out > tp->snd_cwnd_used)
1389 tp->snd_cwnd_used = tp->packets_out;
1391 if (sysctl_tcp_slow_start_after_idle &&
1392 (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto)
1393 tcp_cwnd_application_limited(sk);
1397 /* Minshall's variant of the Nagle send check. */
1398 static bool tcp_minshall_check(const struct tcp_sock *tp)
1400 return after(tp->snd_sml, tp->snd_una) &&
1401 !after(tp->snd_sml, tp->snd_nxt);
1404 /* Update snd_sml if this skb is under mss
1405 * Note that a TSO packet might end with a sub-mss segment
1406 * The test is really :
1407 * if ((skb->len % mss) != 0)
1408 * tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1409 * But we can avoid doing the divide again given we already have
1410 * skb_pcount = skb->len / mss_now
1412 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1413 const struct sk_buff *skb)
1415 if (skb->len < tcp_skb_pcount(skb) * mss_now)
1416 tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1419 /* Return false, if packet can be sent now without violation Nagle's rules:
1420 * 1. It is full sized. (provided by caller in %partial bool)
1421 * 2. Or it contains FIN. (already checked by caller)
1422 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1423 * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1424 * With Minshall's modification: all sent small packets are ACKed.
1426 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1427 unsigned int mss_now, int nonagle)
1430 ((nonagle & TCP_NAGLE_CORK) ||
1431 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1433 /* Returns the portion of skb which can be sent right away */
1434 static unsigned int tcp_mss_split_point(const struct sock *sk,
1435 const struct sk_buff *skb,
1436 unsigned int mss_now,
1437 unsigned int max_segs,
1440 const struct tcp_sock *tp = tcp_sk(sk);
1441 u32 partial, needed, window, max_len;
1443 window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1444 max_len = mss_now * max_segs;
1446 if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
1449 needed = min(skb->len, window);
1451 if (max_len <= needed)
1454 partial = needed % mss_now;
1455 /* If last segment is not a full MSS, check if Nagle rules allow us
1456 * to include this last segment in this skb.
1457 * Otherwise, we'll split the skb at last MSS boundary
1459 if (tcp_nagle_check(partial != 0, tp, mss_now, nonagle))
1460 return needed - partial;
1465 /* Can at least one segment of SKB be sent right now, according to the
1466 * congestion window rules? If so, return how many segments are allowed.
1468 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
1469 const struct sk_buff *skb)
1471 u32 in_flight, cwnd;
1473 /* Don't be strict about the congestion window for the final FIN. */
1474 if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
1475 tcp_skb_pcount(skb) == 1)
1478 in_flight = tcp_packets_in_flight(tp);
1479 cwnd = tp->snd_cwnd;
1480 if (in_flight < cwnd)
1481 return (cwnd - in_flight);
1486 /* Initialize TSO state of a skb.
1487 * This must be invoked the first time we consider transmitting
1488 * SKB onto the wire.
1490 static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb,
1491 unsigned int mss_now)
1493 int tso_segs = tcp_skb_pcount(skb);
1495 if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
1496 tcp_set_skb_tso_segs(sk, skb, mss_now);
1497 tso_segs = tcp_skb_pcount(skb);
1503 /* Return true if the Nagle test allows this packet to be
1506 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
1507 unsigned int cur_mss, int nonagle)
1509 /* Nagle rule does not apply to frames, which sit in the middle of the
1510 * write_queue (they have no chances to get new data).
1512 * This is implemented in the callers, where they modify the 'nonagle'
1513 * argument based upon the location of SKB in the send queue.
1515 if (nonagle & TCP_NAGLE_PUSH)
1518 /* Don't use the nagle rule for urgent data (or for the final FIN). */
1519 if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1522 if (!tcp_nagle_check(skb->len < cur_mss, tp, cur_mss, nonagle))
1528 /* Does at least the first segment of SKB fit into the send window? */
1529 static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
1530 const struct sk_buff *skb,
1531 unsigned int cur_mss)
1533 u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1535 if (skb->len > cur_mss)
1536 end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1538 return !after(end_seq, tcp_wnd_end(tp));
1541 /* This checks if the data bearing packet SKB (usually tcp_send_head(sk))
1542 * should be put on the wire right now. If so, it returns the number of
1543 * packets allowed by the congestion window.
1545 static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb,
1546 unsigned int cur_mss, int nonagle)
1548 const struct tcp_sock *tp = tcp_sk(sk);
1549 unsigned int cwnd_quota;
1551 tcp_init_tso_segs(sk, skb, cur_mss);
1553 if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
1556 cwnd_quota = tcp_cwnd_test(tp, skb);
1557 if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss))
1563 /* Test if sending is allowed right now. */
1564 bool tcp_may_send_now(struct sock *sk)
1566 const struct tcp_sock *tp = tcp_sk(sk);
1567 struct sk_buff *skb = tcp_send_head(sk);
1570 tcp_snd_test(sk, skb, tcp_current_mss(sk),
1571 (tcp_skb_is_last(sk, skb) ?
1572 tp->nonagle : TCP_NAGLE_PUSH));
1575 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
1576 * which is put after SKB on the list. It is very much like
1577 * tcp_fragment() except that it may make several kinds of assumptions
1578 * in order to speed up the splitting operation. In particular, we
1579 * know that all the data is in scatter-gather pages, and that the
1580 * packet has never been sent out before (and thus is not cloned).
1582 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
1583 unsigned int mss_now, gfp_t gfp)
1585 struct sk_buff *buff;
1586 int nlen = skb->len - len;
1589 /* All of a TSO frame must be composed of paged data. */
1590 if (skb->len != skb->data_len)
1591 return tcp_fragment(sk, skb, len, mss_now);
1593 buff = sk_stream_alloc_skb(sk, 0, gfp);
1594 if (unlikely(buff == NULL))
1597 sk->sk_wmem_queued += buff->truesize;
1598 sk_mem_charge(sk, buff->truesize);
1599 buff->truesize += nlen;
1600 skb->truesize -= nlen;
1602 /* Correct the sequence numbers. */
1603 TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1604 TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1605 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1607 /* PSH and FIN should only be set in the second packet. */
1608 flags = TCP_SKB_CB(skb)->tcp_flags;
1609 TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1610 TCP_SKB_CB(buff)->tcp_flags = flags;
1612 /* This packet was never sent out yet, so no SACK bits. */
1613 TCP_SKB_CB(buff)->sacked = 0;
1615 buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
1616 skb_split(skb, buff, len);
1618 /* Fix up tso_factor for both original and new SKB. */
1619 tcp_set_skb_tso_segs(sk, skb, mss_now);
1620 tcp_set_skb_tso_segs(sk, buff, mss_now);
1622 /* Link BUFF into the send queue. */
1623 skb_header_release(buff);
1624 tcp_insert_write_queue_after(skb, buff, sk);
1629 /* Try to defer sending, if possible, in order to minimize the amount
1630 * of TSO splitting we do. View it as a kind of TSO Nagle test.
1632 * This algorithm is from John Heffner.
1634 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb)
1636 struct tcp_sock *tp = tcp_sk(sk);
1637 const struct inet_connection_sock *icsk = inet_csk(sk);
1638 u32 send_win, cong_win, limit, in_flight;
1641 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1644 if (icsk->icsk_ca_state != TCP_CA_Open)
1647 /* Defer for less than two clock ticks. */
1648 if (tp->tso_deferred &&
1649 (((u32)jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1)
1652 in_flight = tcp_packets_in_flight(tp);
1654 BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
1656 send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1658 /* From in_flight test above, we know that cwnd > in_flight. */
1659 cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1661 limit = min(send_win, cong_win);
1663 /* If a full-sized TSO skb can be sent, do it. */
1664 if (limit >= min_t(unsigned int, sk->sk_gso_max_size,
1665 tp->xmit_size_goal_segs * tp->mss_cache))
1668 /* Middle in queue won't get any more data, full sendable already? */
1669 if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1672 win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);
1674 u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1676 /* If at least some fraction of a window is available,
1679 chunk /= win_divisor;
1683 /* Different approach, try not to defer past a single
1684 * ACK. Receiver should ACK every other full sized
1685 * frame, so if we have space for more than 3 frames
1688 if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
1692 /* Ok, it looks like it is advisable to defer.
1693 * Do not rearm the timer if already set to not break TCP ACK clocking.
1695 if (!tp->tso_deferred)
1696 tp->tso_deferred = 1 | (jiffies << 1);
1701 tp->tso_deferred = 0;
1705 /* Create a new MTU probe if we are ready.
1706 * MTU probe is regularly attempting to increase the path MTU by
1707 * deliberately sending larger packets. This discovers routing
1708 * changes resulting in larger path MTUs.
1710 * Returns 0 if we should wait to probe (no cwnd available),
1711 * 1 if a probe was sent,
1714 static int tcp_mtu_probe(struct sock *sk)
1716 struct tcp_sock *tp = tcp_sk(sk);
1717 struct inet_connection_sock *icsk = inet_csk(sk);
1718 struct sk_buff *skb, *nskb, *next;
1725 /* Not currently probing/verifying,
1727 * have enough cwnd, and
1728 * not SACKing (the variable headers throw things off) */
1729 if (!icsk->icsk_mtup.enabled ||
1730 icsk->icsk_mtup.probe_size ||
1731 inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
1732 tp->snd_cwnd < 11 ||
1733 tp->rx_opt.num_sacks || tp->rx_opt.dsack)
1736 /* Very simple search strategy: just double the MSS. */
1737 mss_now = tcp_current_mss(sk);
1738 probe_size = 2 * tp->mss_cache;
1739 size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
1740 if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) {
1741 /* TODO: set timer for probe_converge_event */
1745 /* Have enough data in the send queue to probe? */
1746 if (tp->write_seq - tp->snd_nxt < size_needed)
1749 if (tp->snd_wnd < size_needed)
1751 if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
1754 /* Do we need to wait to drain cwnd? With none in flight, don't stall */
1755 if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
1756 if (!tcp_packets_in_flight(tp))
1762 /* We're allowed to probe. Build it now. */
1763 if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL)
1765 sk->sk_wmem_queued += nskb->truesize;
1766 sk_mem_charge(sk, nskb->truesize);
1768 skb = tcp_send_head(sk);
1770 TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
1771 TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
1772 TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
1773 TCP_SKB_CB(nskb)->sacked = 0;
1775 nskb->ip_summed = skb->ip_summed;
1777 tcp_insert_write_queue_before(nskb, skb, sk);
1780 tcp_for_write_queue_from_safe(skb, next, sk) {
1781 copy = min_t(int, skb->len, probe_size - len);
1782 if (nskb->ip_summed)
1783 skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
1785 nskb->csum = skb_copy_and_csum_bits(skb, 0,
1786 skb_put(nskb, copy),
1789 if (skb->len <= copy) {
1790 /* We've eaten all the data from this skb.
1792 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1793 tcp_unlink_write_queue(skb, sk);
1794 sk_wmem_free_skb(sk, skb);
1796 TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
1797 ~(TCPHDR_FIN|TCPHDR_PSH);
1798 if (!skb_shinfo(skb)->nr_frags) {
1799 skb_pull(skb, copy);
1800 if (skb->ip_summed != CHECKSUM_PARTIAL)
1801 skb->csum = csum_partial(skb->data,
1804 __pskb_trim_head(skb, copy);
1805 tcp_set_skb_tso_segs(sk, skb, mss_now);
1807 TCP_SKB_CB(skb)->seq += copy;
1812 if (len >= probe_size)
1815 tcp_init_tso_segs(sk, nskb, nskb->len);
1817 /* We're ready to send. If this fails, the probe will
1818 * be resegmented into mss-sized pieces by tcp_write_xmit(). */
1819 TCP_SKB_CB(nskb)->when = tcp_time_stamp;
1820 if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
1821 /* Decrement cwnd here because we are sending
1822 * effectively two packets. */
1824 tcp_event_new_data_sent(sk, nskb);
1826 icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
1827 tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
1828 tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
1836 /* This routine writes packets to the network. It advances the
1837 * send_head. This happens as incoming acks open up the remote
1840 * LARGESEND note: !tcp_urg_mode is overkill, only frames between
1841 * snd_up-64k-mss .. snd_up cannot be large. However, taking into
1842 * account rare use of URG, this is not a big flaw.
1844 * Send at most one packet when push_one > 0. Temporarily ignore
1845 * cwnd limit to force at most one packet out when push_one == 2.
1847 * Returns true, if no segments are in flight and we have queued segments,
1848 * but cannot send anything now because of SWS or another problem.
1850 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
1851 int push_one, gfp_t gfp)
1853 struct tcp_sock *tp = tcp_sk(sk);
1854 struct sk_buff *skb;
1855 unsigned int tso_segs, sent_pkts;
1862 /* Do MTU probing. */
1863 result = tcp_mtu_probe(sk);
1866 } else if (result > 0) {
1871 while ((skb = tcp_send_head(sk))) {
1874 tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
1877 if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE)
1878 goto repair; /* Skip network transmission */
1880 cwnd_quota = tcp_cwnd_test(tp, skb);
1883 /* Force out a loss probe pkt. */
1889 if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
1892 if (tso_segs == 1) {
1893 if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
1894 (tcp_skb_is_last(sk, skb) ?
1895 nonagle : TCP_NAGLE_PUSH))))
1898 if (!push_one && tcp_tso_should_defer(sk, skb))
1902 /* TCP Small Queues :
1903 * Control number of packets in qdisc/devices to two packets / or ~1 ms.
1905 * - better RTT estimation and ACK scheduling
1908 * Alas, some drivers / subsystems require a fair amount
1909 * of queued bytes to ensure line rate.
1910 * One example is wifi aggregation (802.11 AMPDU)
1912 limit = max_t(unsigned int, sysctl_tcp_limit_output_bytes,
1913 sk->sk_pacing_rate >> 10);
1915 if (atomic_read(&sk->sk_wmem_alloc) > limit) {
1916 set_bit(TSQ_THROTTLED, &tp->tsq_flags);
1917 /* It is possible TX completion already happened
1918 * before we set TSQ_THROTTLED, so we must
1919 * test again the condition.
1920 * We abuse smp_mb__after_clear_bit() because
1921 * there is no smp_mb__after_set_bit() yet
1923 smp_mb__after_clear_bit();
1924 if (atomic_read(&sk->sk_wmem_alloc) > limit)
1929 if (tso_segs > 1 && !tcp_urg_mode(tp))
1930 limit = tcp_mss_split_point(sk, skb, mss_now,
1933 sk->sk_gso_max_segs),
1936 if (skb->len > limit &&
1937 unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
1940 TCP_SKB_CB(skb)->when = tcp_time_stamp;
1942 if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
1946 /* Advance the send_head. This one is sent out.
1947 * This call will increment packets_out.
1949 tcp_event_new_data_sent(sk, skb);
1951 tcp_minshall_update(tp, mss_now, skb);
1952 sent_pkts += tcp_skb_pcount(skb);
1958 if (likely(sent_pkts)) {
1959 if (tcp_in_cwnd_reduction(sk))
1960 tp->prr_out += sent_pkts;
1962 /* Send one loss probe per tail loss episode. */
1964 tcp_schedule_loss_probe(sk);
1965 tcp_cwnd_validate(sk);
1968 return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk));
1971 bool tcp_schedule_loss_probe(struct sock *sk)
1973 struct inet_connection_sock *icsk = inet_csk(sk);
1974 struct tcp_sock *tp = tcp_sk(sk);
1975 u32 timeout, tlp_time_stamp, rto_time_stamp;
1976 u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3);
1978 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS))
1980 /* No consecutive loss probes. */
1981 if (WARN_ON(icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)) {
1985 /* Don't do any loss probe on a Fast Open connection before 3WHS
1988 if (sk->sk_state == TCP_SYN_RECV)
1991 /* TLP is only scheduled when next timer event is RTO. */
1992 if (icsk->icsk_pending != ICSK_TIME_RETRANS)
1995 /* Schedule a loss probe in 2*RTT for SACK capable connections
1996 * in Open state, that are either limited by cwnd or application.
1998 if (sysctl_tcp_early_retrans < 3 || !tp->srtt_us || !tp->packets_out ||
1999 !tcp_is_sack(tp) || inet_csk(sk)->icsk_ca_state != TCP_CA_Open)
2002 if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) &&
2006 /* Probe timeout is at least 1.5*rtt + TCP_DELACK_MAX to account
2007 * for delayed ack when there's one outstanding packet.
2010 if (tp->packets_out == 1)
2011 timeout = max_t(u32, timeout,
2012 (rtt + (rtt >> 1) + TCP_DELACK_MAX));
2013 timeout = max_t(u32, timeout, msecs_to_jiffies(10));
2015 /* If RTO is shorter, just schedule TLP in its place. */
2016 tlp_time_stamp = tcp_time_stamp + timeout;
2017 rto_time_stamp = (u32)inet_csk(sk)->icsk_timeout;
2018 if ((s32)(tlp_time_stamp - rto_time_stamp) > 0) {
2019 s32 delta = rto_time_stamp - tcp_time_stamp;
2024 inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout,
2029 /* When probe timeout (PTO) fires, send a new segment if one exists, else
2030 * retransmit the last segment.
2032 void tcp_send_loss_probe(struct sock *sk)
2034 struct tcp_sock *tp = tcp_sk(sk);
2035 struct sk_buff *skb;
2037 int mss = tcp_current_mss(sk);
2040 if (tcp_send_head(sk) != NULL) {
2041 err = tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2045 /* At most one outstanding TLP retransmission. */
2046 if (tp->tlp_high_seq)
2049 /* Retransmit last segment. */
2050 skb = tcp_write_queue_tail(sk);
2054 pcount = tcp_skb_pcount(skb);
2055 if (WARN_ON(!pcount))
2058 if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2059 if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss)))
2061 skb = tcp_write_queue_tail(sk);
2064 if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2067 /* Probe with zero data doesn't trigger fast recovery. */
2069 err = __tcp_retransmit_skb(sk, skb);
2071 /* Record snd_nxt for loss detection. */
2073 tp->tlp_high_seq = tp->snd_nxt;
2076 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2077 inet_csk(sk)->icsk_rto,
2081 NET_INC_STATS_BH(sock_net(sk),
2082 LINUX_MIB_TCPLOSSPROBES);
2085 /* Push out any pending frames which were held back due to
2086 * TCP_CORK or attempt at coalescing tiny packets.
2087 * The socket must be locked by the caller.
2089 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2092 /* If we are closed, the bytes will have to remain here.
2093 * In time closedown will finish, we empty the write queue and
2094 * all will be happy.
2096 if (unlikely(sk->sk_state == TCP_CLOSE))
2099 if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2100 sk_gfp_atomic(sk, GFP_ATOMIC)))
2101 tcp_check_probe_timer(sk);
2104 /* Send _single_ skb sitting at the send head. This function requires
2105 * true push pending frames to setup probe timer etc.
2107 void tcp_push_one(struct sock *sk, unsigned int mss_now)
2109 struct sk_buff *skb = tcp_send_head(sk);
2111 BUG_ON(!skb || skb->len < mss_now);
2113 tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2116 /* This function returns the amount that we can raise the
2117 * usable window based on the following constraints
2119 * 1. The window can never be shrunk once it is offered (RFC 793)
2120 * 2. We limit memory per socket
2123 * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2124 * RECV.NEXT + RCV.WIN fixed until:
2125 * RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2127 * i.e. don't raise the right edge of the window until you can raise
2128 * it at least MSS bytes.
2130 * Unfortunately, the recommended algorithm breaks header prediction,
2131 * since header prediction assumes th->window stays fixed.
2133 * Strictly speaking, keeping th->window fixed violates the receiver
2134 * side SWS prevention criteria. The problem is that under this rule
2135 * a stream of single byte packets will cause the right side of the
2136 * window to always advance by a single byte.
2138 * Of course, if the sender implements sender side SWS prevention
2139 * then this will not be a problem.
2141 * BSD seems to make the following compromise:
2143 * If the free space is less than the 1/4 of the maximum
2144 * space available and the free space is less than 1/2 mss,
2145 * then set the window to 0.
2146 * [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2147 * Otherwise, just prevent the window from shrinking
2148 * and from being larger than the largest representable value.
2150 * This prevents incremental opening of the window in the regime
2151 * where TCP is limited by the speed of the reader side taking
2152 * data out of the TCP receive queue. It does nothing about
2153 * those cases where the window is constrained on the sender side
2154 * because the pipeline is full.
2156 * BSD also seems to "accidentally" limit itself to windows that are a
2157 * multiple of MSS, at least until the free space gets quite small.
2158 * This would appear to be a side effect of the mbuf implementation.
2159 * Combining these two algorithms results in the observed behavior
2160 * of having a fixed window size at almost all times.
2162 * Below we obtain similar behavior by forcing the offered window to
2163 * a multiple of the mss when it is feasible to do so.
2165 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
2166 * Regular options like TIMESTAMP are taken into account.
2168 u32 __tcp_select_window(struct sock *sk)
2170 struct inet_connection_sock *icsk = inet_csk(sk);
2171 struct tcp_sock *tp = tcp_sk(sk);
2172 /* MSS for the peer's data. Previous versions used mss_clamp
2173 * here. I don't know if the value based on our guesses
2174 * of peer's MSS is better for the performance. It's more correct
2175 * but may be worse for the performance because of rcv_mss
2176 * fluctuations. --SAW 1998/11/1
2178 int mss = icsk->icsk_ack.rcv_mss;
2179 int free_space = tcp_space(sk);
2180 int allowed_space = tcp_full_space(sk);
2181 int full_space = min_t(int, tp->window_clamp, allowed_space);
2184 if (mss > full_space)
2187 if (free_space < (full_space >> 1)) {
2188 icsk->icsk_ack.quick = 0;
2190 if (sk_under_memory_pressure(sk))
2191 tp->rcv_ssthresh = min(tp->rcv_ssthresh,
2194 /* free_space might become our new window, make sure we don't
2195 * increase it due to wscale.
2197 free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
2199 /* if free space is less than mss estimate, or is below 1/16th
2200 * of the maximum allowed, try to move to zero-window, else
2201 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
2202 * new incoming data is dropped due to memory limits.
2203 * With large window, mss test triggers way too late in order
2204 * to announce zero window in time before rmem limit kicks in.
2206 if (free_space < (allowed_space >> 4) || free_space < mss)
2210 if (free_space > tp->rcv_ssthresh)
2211 free_space = tp->rcv_ssthresh;
2213 /* Don't do rounding if we are using window scaling, since the
2214 * scaled window will not line up with the MSS boundary anyway.
2216 window = tp->rcv_wnd;
2217 if (tp->rx_opt.rcv_wscale) {
2218 window = free_space;
2220 /* Advertise enough space so that it won't get scaled away.
2221 * Import case: prevent zero window announcement if
2222 * 1<<rcv_wscale > mss.
2224 if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window)
2225 window = (((window >> tp->rx_opt.rcv_wscale) + 1)
2226 << tp->rx_opt.rcv_wscale);
2228 /* Get the largest window that is a nice multiple of mss.
2229 * Window clamp already applied above.
2230 * If our current window offering is within 1 mss of the
2231 * free space we just keep it. This prevents the divide
2232 * and multiply from happening most of the time.
2233 * We also don't do any window rounding when the free space
2236 if (window <= free_space - mss || window > free_space)
2237 window = (free_space / mss) * mss;
2238 else if (mss == full_space &&
2239 free_space > window + (full_space >> 1))
2240 window = free_space;
2246 /* Collapses two adjacent SKB's during retransmission. */
2247 static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
2249 struct tcp_sock *tp = tcp_sk(sk);
2250 struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
2251 int skb_size, next_skb_size;
2253 skb_size = skb->len;
2254 next_skb_size = next_skb->len;
2256 BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
2258 tcp_highest_sack_combine(sk, next_skb, skb);
2260 tcp_unlink_write_queue(next_skb, sk);
2262 skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size),
2265 if (next_skb->ip_summed == CHECKSUM_PARTIAL)
2266 skb->ip_summed = CHECKSUM_PARTIAL;
2268 if (skb->ip_summed != CHECKSUM_PARTIAL)
2269 skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
2271 /* Update sequence range on original skb. */
2272 TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2274 /* Merge over control information. This moves PSH/FIN etc. over */
2275 TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
2277 /* All done, get rid of second SKB and account for it so
2278 * packet counting does not break.
2280 TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
2282 /* changed transmit queue under us so clear hints */
2283 tcp_clear_retrans_hints_partial(tp);
2284 if (next_skb == tp->retransmit_skb_hint)
2285 tp->retransmit_skb_hint = skb;
2287 tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
2289 sk_wmem_free_skb(sk, next_skb);
2292 /* Check if coalescing SKBs is legal. */
2293 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
2295 if (tcp_skb_pcount(skb) > 1)
2297 /* TODO: SACK collapsing could be used to remove this condition */
2298 if (skb_shinfo(skb)->nr_frags != 0)
2300 if (skb_cloned(skb))
2302 if (skb == tcp_send_head(sk))
2304 /* Some heurestics for collapsing over SACK'd could be invented */
2305 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2311 /* Collapse packets in the retransmit queue to make to create
2312 * less packets on the wire. This is only done on retransmission.
2314 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
2317 struct tcp_sock *tp = tcp_sk(sk);
2318 struct sk_buff *skb = to, *tmp;
2321 if (!sysctl_tcp_retrans_collapse)
2323 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2326 tcp_for_write_queue_from_safe(skb, tmp, sk) {
2327 if (!tcp_can_collapse(sk, skb))
2339 /* Punt if not enough space exists in the first SKB for
2340 * the data in the second
2342 if (skb->len > skb_availroom(to))
2345 if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
2348 tcp_collapse_retrans(sk, to);
2352 /* This retransmits one SKB. Policy decisions and retransmit queue
2353 * state updates are done by the caller. Returns non-zero if an
2354 * error occurred which prevented the send.
2356 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
2358 struct tcp_sock *tp = tcp_sk(sk);
2359 struct inet_connection_sock *icsk = inet_csk(sk);
2360 unsigned int cur_mss;
2362 /* Inconslusive MTU probe */
2363 if (icsk->icsk_mtup.probe_size) {
2364 icsk->icsk_mtup.probe_size = 0;
2367 /* Do not sent more than we queued. 1/4 is reserved for possible
2368 * copying overhead: fragmentation, tunneling, mangling etc.
2370 if (atomic_read(&sk->sk_wmem_alloc) >
2371 min(sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2), sk->sk_sndbuf))
2374 if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
2375 if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2377 if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
2381 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
2382 return -EHOSTUNREACH; /* Routing failure or similar. */
2384 cur_mss = tcp_current_mss(sk);
2386 /* If receiver has shrunk his window, and skb is out of
2387 * new window, do not retransmit it. The exception is the
2388 * case, when window is shrunk to zero. In this case
2389 * our retransmit serves as a zero window probe.
2391 if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
2392 TCP_SKB_CB(skb)->seq != tp->snd_una)
2395 if (skb->len > cur_mss) {
2396 if (tcp_fragment(sk, skb, cur_mss, cur_mss))
2397 return -ENOMEM; /* We'll try again later. */
2399 int oldpcount = tcp_skb_pcount(skb);
2401 if (unlikely(oldpcount > 1)) {
2402 if (skb_unclone(skb, GFP_ATOMIC))
2404 tcp_init_tso_segs(sk, skb, cur_mss);
2405 tcp_adjust_pcount(sk, skb, oldpcount - tcp_skb_pcount(skb));
2409 tcp_retrans_try_collapse(sk, skb, cur_mss);
2411 /* Make a copy, if the first transmission SKB clone we made
2412 * is still in somebody's hands, else make a clone.
2414 TCP_SKB_CB(skb)->when = tcp_time_stamp;
2416 /* make sure skb->data is aligned on arches that require it
2417 * and check if ack-trimming & collapsing extended the headroom
2418 * beyond what csum_start can cover.
2420 if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
2421 skb_headroom(skb) >= 0xFFFF)) {
2422 struct sk_buff *nskb = __pskb_copy(skb, MAX_TCP_HEADER,
2424 return nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
2427 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2431 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
2433 struct tcp_sock *tp = tcp_sk(sk);
2434 int err = __tcp_retransmit_skb(sk, skb);
2437 /* Update global TCP statistics. */
2438 TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
2439 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2440 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
2441 tp->total_retrans++;
2443 #if FASTRETRANS_DEBUG > 0
2444 if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
2445 net_dbg_ratelimited("retrans_out leaked\n");
2448 if (!tp->retrans_out)
2449 tp->lost_retrans_low = tp->snd_nxt;
2450 TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
2451 tp->retrans_out += tcp_skb_pcount(skb);
2453 /* Save stamp of the first retransmit. */
2454 if (!tp->retrans_stamp)
2455 tp->retrans_stamp = TCP_SKB_CB(skb)->when;
2457 tp->undo_retrans += tcp_skb_pcount(skb);
2459 /* snd_nxt is stored to detect loss of retransmitted segment,
2460 * see tcp_input.c tcp_sacktag_write_queue().
2462 TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
2464 NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
2469 /* Check if we forward retransmits are possible in the current
2470 * window/congestion state.
2472 static bool tcp_can_forward_retransmit(struct sock *sk)
2474 const struct inet_connection_sock *icsk = inet_csk(sk);
2475 const struct tcp_sock *tp = tcp_sk(sk);
2477 /* Forward retransmissions are possible only during Recovery. */
2478 if (icsk->icsk_ca_state != TCP_CA_Recovery)
2481 /* No forward retransmissions in Reno are possible. */
2482 if (tcp_is_reno(tp))
2485 /* Yeah, we have to make difficult choice between forward transmission
2486 * and retransmission... Both ways have their merits...
2488 * For now we do not retransmit anything, while we have some new
2489 * segments to send. In the other cases, follow rule 3 for
2490 * NextSeg() specified in RFC3517.
2493 if (tcp_may_send_now(sk))
2499 /* This gets called after a retransmit timeout, and the initially
2500 * retransmitted data is acknowledged. It tries to continue
2501 * resending the rest of the retransmit queue, until either
2502 * we've sent it all or the congestion window limit is reached.
2503 * If doing SACK, the first ACK which comes back for a timeout
2504 * based retransmit packet might feed us FACK information again.
2505 * If so, we use it to avoid unnecessarily retransmissions.
2507 void tcp_xmit_retransmit_queue(struct sock *sk)
2509 const struct inet_connection_sock *icsk = inet_csk(sk);
2510 struct tcp_sock *tp = tcp_sk(sk);
2511 struct sk_buff *skb;
2512 struct sk_buff *hole = NULL;
2515 int fwd_rexmitting = 0;
2517 if (!tp->packets_out)
2521 tp->retransmit_high = tp->snd_una;
2523 if (tp->retransmit_skb_hint) {
2524 skb = tp->retransmit_skb_hint;
2525 last_lost = TCP_SKB_CB(skb)->end_seq;
2526 if (after(last_lost, tp->retransmit_high))
2527 last_lost = tp->retransmit_high;
2529 skb = tcp_write_queue_head(sk);
2530 last_lost = tp->snd_una;
2533 tcp_for_write_queue_from(skb, sk) {
2534 __u8 sacked = TCP_SKB_CB(skb)->sacked;
2536 if (skb == tcp_send_head(sk))
2538 /* we could do better than to assign each time */
2540 tp->retransmit_skb_hint = skb;
2542 /* Assume this retransmit will generate
2543 * only one packet for congestion window
2544 * calculation purposes. This works because
2545 * tcp_retransmit_skb() will chop up the
2546 * packet to be MSS sized and all the
2547 * packet counting works out.
2549 if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
2552 if (fwd_rexmitting) {
2554 if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp)))
2556 mib_idx = LINUX_MIB_TCPFORWARDRETRANS;
2558 } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) {
2559 tp->retransmit_high = last_lost;
2560 if (!tcp_can_forward_retransmit(sk))
2562 /* Backtrack if necessary to non-L'ed skb */
2570 } else if (!(sacked & TCPCB_LOST)) {
2571 if (hole == NULL && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
2576 last_lost = TCP_SKB_CB(skb)->end_seq;
2577 if (icsk->icsk_ca_state != TCP_CA_Loss)
2578 mib_idx = LINUX_MIB_TCPFASTRETRANS;
2580 mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
2583 if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
2586 if (tcp_retransmit_skb(sk, skb))
2589 NET_INC_STATS_BH(sock_net(sk), mib_idx);
2591 if (tcp_in_cwnd_reduction(sk))
2592 tp->prr_out += tcp_skb_pcount(skb);
2594 if (skb == tcp_write_queue_head(sk))
2595 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2596 inet_csk(sk)->icsk_rto,
2601 /* Send a fin. The caller locks the socket for us. This cannot be
2602 * allowed to fail queueing a FIN frame under any circumstances.
2604 void tcp_send_fin(struct sock *sk)
2606 struct tcp_sock *tp = tcp_sk(sk);
2607 struct sk_buff *skb = tcp_write_queue_tail(sk);
2610 /* Optimization, tack on the FIN if we have a queue of
2611 * unsent frames. But be careful about outgoing SACKS
2614 mss_now = tcp_current_mss(sk);
2616 if (tcp_send_head(sk) != NULL) {
2617 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_FIN;
2618 TCP_SKB_CB(skb)->end_seq++;
2621 /* Socket is locked, keep trying until memory is available. */
2623 skb = alloc_skb_fclone(MAX_TCP_HEADER,
2630 /* Reserve space for headers and prepare control bits. */
2631 skb_reserve(skb, MAX_TCP_HEADER);
2632 /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
2633 tcp_init_nondata_skb(skb, tp->write_seq,
2634 TCPHDR_ACK | TCPHDR_FIN);
2635 tcp_queue_skb(sk, skb);
2637 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_OFF);
2640 /* We get here when a process closes a file descriptor (either due to
2641 * an explicit close() or as a byproduct of exit()'ing) and there
2642 * was unread data in the receive queue. This behavior is recommended
2643 * by RFC 2525, section 2.17. -DaveM
2645 void tcp_send_active_reset(struct sock *sk, gfp_t priority)
2647 struct sk_buff *skb;
2649 /* NOTE: No TCP options attached and we never retransmit this. */
2650 skb = alloc_skb(MAX_TCP_HEADER, priority);
2652 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
2656 /* Reserve space for headers and prepare control bits. */
2657 skb_reserve(skb, MAX_TCP_HEADER);
2658 tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
2659 TCPHDR_ACK | TCPHDR_RST);
2661 TCP_SKB_CB(skb)->when = tcp_time_stamp;
2662 if (tcp_transmit_skb(sk, skb, 0, priority))
2663 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
2665 TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
2668 /* Send a crossed SYN-ACK during socket establishment.
2669 * WARNING: This routine must only be called when we have already sent
2670 * a SYN packet that crossed the incoming SYN that caused this routine
2671 * to get called. If this assumption fails then the initial rcv_wnd
2672 * and rcv_wscale values will not be correct.
2674 int tcp_send_synack(struct sock *sk)
2676 struct sk_buff *skb;
2678 skb = tcp_write_queue_head(sk);
2679 if (skb == NULL || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2680 pr_debug("%s: wrong queue state\n", __func__);
2683 if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
2684 if (skb_cloned(skb)) {
2685 struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
2688 tcp_unlink_write_queue(skb, sk);
2689 skb_header_release(nskb);
2690 __tcp_add_write_queue_head(sk, nskb);
2691 sk_wmem_free_skb(sk, skb);
2692 sk->sk_wmem_queued += nskb->truesize;
2693 sk_mem_charge(sk, nskb->truesize);
2697 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
2698 TCP_ECN_send_synack(tcp_sk(sk), skb);
2700 TCP_SKB_CB(skb)->when = tcp_time_stamp;
2701 return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2705 * tcp_make_synack - Prepare a SYN-ACK.
2706 * sk: listener socket
2707 * dst: dst entry attached to the SYNACK
2708 * req: request_sock pointer
2710 * Allocate one skb and build a SYNACK packet.
2711 * @dst is consumed : Caller should not use it again.
2713 struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
2714 struct request_sock *req,
2715 struct tcp_fastopen_cookie *foc)
2717 struct tcp_out_options opts;
2718 struct inet_request_sock *ireq = inet_rsk(req);
2719 struct tcp_sock *tp = tcp_sk(sk);
2721 struct sk_buff *skb;
2722 struct tcp_md5sig_key *md5;
2723 int tcp_header_size;
2726 skb = sock_wmalloc(sk, MAX_TCP_HEADER + 15, 1, GFP_ATOMIC);
2727 if (unlikely(!skb)) {
2731 /* Reserve space for headers. */
2732 skb_reserve(skb, MAX_TCP_HEADER);
2734 skb_dst_set(skb, dst);
2735 security_skb_owned_by(skb, sk);
2737 mss = dst_metric_advmss(dst);
2738 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss)
2739 mss = tp->rx_opt.user_mss;
2741 if (req->rcv_wnd == 0) { /* ignored for retransmitted syns */
2743 /* Set this up on the first call only */
2744 req->window_clamp = tp->window_clamp ? : dst_metric(dst, RTAX_WINDOW);
2746 /* limit the window selection if the user enforce a smaller rx buffer */
2747 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
2748 (req->window_clamp > tcp_full_space(sk) || req->window_clamp == 0))
2749 req->window_clamp = tcp_full_space(sk);
2751 /* tcp_full_space because it is guaranteed to be the first packet */
2752 tcp_select_initial_window(tcp_full_space(sk),
2753 mss - (ireq->tstamp_ok ? TCPOLEN_TSTAMP_ALIGNED : 0),
2758 dst_metric(dst, RTAX_INITRWND));
2759 ireq->rcv_wscale = rcv_wscale;
2762 memset(&opts, 0, sizeof(opts));
2763 #ifdef CONFIG_SYN_COOKIES
2764 if (unlikely(req->cookie_ts))
2765 TCP_SKB_CB(skb)->when = cookie_init_timestamp(req);
2768 TCP_SKB_CB(skb)->when = tcp_time_stamp;
2769 tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, &md5,
2772 skb_push(skb, tcp_header_size);
2773 skb_reset_transport_header(skb);
2776 memset(th, 0, sizeof(struct tcphdr));
2779 TCP_ECN_make_synack(req, th);
2780 th->source = htons(ireq->ir_num);
2781 th->dest = ireq->ir_rmt_port;
2782 /* Setting of flags are superfluous here for callers (and ECE is
2783 * not even correctly set)
2785 tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
2786 TCPHDR_SYN | TCPHDR_ACK);
2788 th->seq = htonl(TCP_SKB_CB(skb)->seq);
2789 /* XXX data is queued and acked as is. No buffer/window check */
2790 th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
2792 /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
2793 th->window = htons(min(req->rcv_wnd, 65535U));
2794 tcp_options_write((__be32 *)(th + 1), tp, &opts);
2795 th->doff = (tcp_header_size >> 2);
2796 TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS, tcp_skb_pcount(skb));
2798 #ifdef CONFIG_TCP_MD5SIG
2799 /* Okay, we have all we need - do the md5 hash if needed */
2801 tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
2802 md5, NULL, req, skb);
2808 EXPORT_SYMBOL(tcp_make_synack);
2810 /* Do all connect socket setups that can be done AF independent. */
2811 static void tcp_connect_init(struct sock *sk)
2813 const struct dst_entry *dst = __sk_dst_get(sk);
2814 struct tcp_sock *tp = tcp_sk(sk);
2817 /* We'll fix this up when we get a response from the other end.
2818 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
2820 tp->tcp_header_len = sizeof(struct tcphdr) +
2821 (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);
2823 #ifdef CONFIG_TCP_MD5SIG
2824 if (tp->af_specific->md5_lookup(sk, sk) != NULL)
2825 tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
2828 /* If user gave his TCP_MAXSEG, record it to clamp */
2829 if (tp->rx_opt.user_mss)
2830 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
2833 tcp_sync_mss(sk, dst_mtu(dst));
2835 if (!tp->window_clamp)
2836 tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
2837 tp->advmss = dst_metric_advmss(dst);
2838 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss)
2839 tp->advmss = tp->rx_opt.user_mss;
2841 tcp_initialize_rcv_mss(sk);
2843 /* limit the window selection if the user enforce a smaller rx buffer */
2844 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
2845 (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
2846 tp->window_clamp = tcp_full_space(sk);
2848 tcp_select_initial_window(tcp_full_space(sk),
2849 tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
2852 sysctl_tcp_window_scaling,
2854 dst_metric(dst, RTAX_INITRWND));
2856 tp->rx_opt.rcv_wscale = rcv_wscale;
2857 tp->rcv_ssthresh = tp->rcv_wnd;
2860 sock_reset_flag(sk, SOCK_DONE);
2863 tp->snd_una = tp->write_seq;
2864 tp->snd_sml = tp->write_seq;
2865 tp->snd_up = tp->write_seq;
2866 tp->snd_nxt = tp->write_seq;
2868 if (likely(!tp->repair))
2871 tp->rcv_tstamp = tcp_time_stamp;
2872 tp->rcv_wup = tp->rcv_nxt;
2873 tp->copied_seq = tp->rcv_nxt;
2875 inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
2876 inet_csk(sk)->icsk_retransmits = 0;
2877 tcp_clear_retrans(tp);
2880 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
2882 struct tcp_sock *tp = tcp_sk(sk);
2883 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
2885 tcb->end_seq += skb->len;
2886 skb_header_release(skb);
2887 __tcp_add_write_queue_tail(sk, skb);
2888 sk->sk_wmem_queued += skb->truesize;
2889 sk_mem_charge(sk, skb->truesize);
2890 tp->write_seq = tcb->end_seq;
2891 tp->packets_out += tcp_skb_pcount(skb);
2894 /* Build and send a SYN with data and (cached) Fast Open cookie. However,
2895 * queue a data-only packet after the regular SYN, such that regular SYNs
2896 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
2897 * only the SYN sequence, the data are retransmitted in the first ACK.
2898 * If cookie is not cached or other error occurs, falls back to send a
2899 * regular SYN with Fast Open cookie request option.
2901 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
2903 struct tcp_sock *tp = tcp_sk(sk);
2904 struct tcp_fastopen_request *fo = tp->fastopen_req;
2905 int syn_loss = 0, space, i, err = 0, iovlen = fo->data->msg_iovlen;
2906 struct sk_buff *syn_data = NULL, *data;
2907 unsigned long last_syn_loss = 0;
2909 tp->rx_opt.mss_clamp = tp->advmss; /* If MSS is not cached */
2910 tcp_fastopen_cache_get(sk, &tp->rx_opt.mss_clamp, &fo->cookie,
2911 &syn_loss, &last_syn_loss);
2912 /* Recurring FO SYN losses: revert to regular handshake temporarily */
2914 time_before(jiffies, last_syn_loss + (60*HZ << syn_loss))) {
2915 fo->cookie.len = -1;
2919 if (sysctl_tcp_fastopen & TFO_CLIENT_NO_COOKIE)
2920 fo->cookie.len = -1;
2921 else if (fo->cookie.len <= 0)
2924 /* MSS for SYN-data is based on cached MSS and bounded by PMTU and
2925 * user-MSS. Reserve maximum option space for middleboxes that add
2926 * private TCP options. The cost is reduced data space in SYN :(
2928 if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->rx_opt.mss_clamp)
2929 tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
2930 space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
2931 MAX_TCP_OPTION_SPACE;
2933 syn_data = skb_copy_expand(syn, skb_headroom(syn), space,
2935 if (syn_data == NULL)
2938 for (i = 0; i < iovlen && syn_data->len < space; ++i) {
2939 struct iovec *iov = &fo->data->msg_iov[i];
2940 unsigned char __user *from = iov->iov_base;
2941 int len = iov->iov_len;
2943 if (syn_data->len + len > space)
2944 len = space - syn_data->len;
2945 else if (i + 1 == iovlen)
2946 /* No more data pending in inet_wait_for_connect() */
2949 if (skb_add_data(syn_data, from, len))
2953 /* Queue a data-only packet after the regular SYN for retransmission */
2954 data = pskb_copy(syn_data, sk->sk_allocation);
2957 TCP_SKB_CB(data)->seq++;
2958 TCP_SKB_CB(data)->tcp_flags &= ~TCPHDR_SYN;
2959 TCP_SKB_CB(data)->tcp_flags = (TCPHDR_ACK|TCPHDR_PSH);
2960 tcp_connect_queue_skb(sk, data);
2961 fo->copied = data->len;
2963 if (tcp_transmit_skb(sk, syn_data, 0, sk->sk_allocation) == 0) {
2964 tp->syn_data = (fo->copied > 0);
2965 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
2971 /* Send a regular SYN with Fast Open cookie request option */
2972 if (fo->cookie.len > 0)
2974 err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
2976 tp->syn_fastopen = 0;
2977 kfree_skb(syn_data);
2979 fo->cookie.len = -1; /* Exclude Fast Open option for SYN retries */
2983 /* Build a SYN and send it off. */
2984 int tcp_connect(struct sock *sk)
2986 struct tcp_sock *tp = tcp_sk(sk);
2987 struct sk_buff *buff;
2990 tcp_connect_init(sk);
2992 if (unlikely(tp->repair)) {
2993 tcp_finish_connect(sk, NULL);
2997 buff = alloc_skb_fclone(MAX_TCP_HEADER + 15, sk->sk_allocation);
2998 if (unlikely(buff == NULL))
3001 /* Reserve space for headers. */
3002 skb_reserve(buff, MAX_TCP_HEADER);
3004 tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3005 tp->retrans_stamp = TCP_SKB_CB(buff)->when = tcp_time_stamp;
3006 tcp_connect_queue_skb(sk, buff);
3007 TCP_ECN_send_syn(sk, buff);
3009 /* Send off SYN; include data in Fast Open. */
3010 err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3011 tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3012 if (err == -ECONNREFUSED)
3015 /* We change tp->snd_nxt after the tcp_transmit_skb() call
3016 * in order to make this packet get counted in tcpOutSegs.
3018 tp->snd_nxt = tp->write_seq;
3019 tp->pushed_seq = tp->write_seq;
3020 TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3022 /* Timer for repeating the SYN until an answer. */
3023 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3024 inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
3027 EXPORT_SYMBOL(tcp_connect);
3029 /* Send out a delayed ack, the caller does the policy checking
3030 * to see if we should even be here. See tcp_input.c:tcp_ack_snd_check()
3033 void tcp_send_delayed_ack(struct sock *sk)
3035 struct inet_connection_sock *icsk = inet_csk(sk);
3036 int ato = icsk->icsk_ack.ato;
3037 unsigned long timeout;
3039 if (ato > TCP_DELACK_MIN) {
3040 const struct tcp_sock *tp = tcp_sk(sk);
3041 int max_ato = HZ / 2;
3043 if (icsk->icsk_ack.pingpong ||
3044 (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
3045 max_ato = TCP_DELACK_MAX;
3047 /* Slow path, intersegment interval is "high". */
3049 /* If some rtt estimate is known, use it to bound delayed ack.
3050 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
3054 int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
3061 ato = min(ato, max_ato);
3064 /* Stay within the limit we were given */
3065 timeout = jiffies + ato;
3067 /* Use new timeout only if there wasn't a older one earlier. */
3068 if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
3069 /* If delack timer was blocked or is about to expire,
3072 if (icsk->icsk_ack.blocked ||
3073 time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
3078 if (!time_before(timeout, icsk->icsk_ack.timeout))
3079 timeout = icsk->icsk_ack.timeout;
3081 icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3082 icsk->icsk_ack.timeout = timeout;
3083 sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
3086 /* This routine sends an ack and also updates the window. */
3087 void tcp_send_ack(struct sock *sk)
3089 struct sk_buff *buff;
3091 /* If we have been reset, we may not send again. */
3092 if (sk->sk_state == TCP_CLOSE)
3095 /* We are not putting this on the write queue, so
3096 * tcp_transmit_skb() will set the ownership to this
3099 buff = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
3101 inet_csk_schedule_ack(sk);
3102 inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
3103 inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
3104 TCP_DELACK_MAX, TCP_RTO_MAX);
3108 /* Reserve space for headers and prepare control bits. */
3109 skb_reserve(buff, MAX_TCP_HEADER);
3110 tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
3112 /* Send it off, this clears delayed acks for us. */
3113 TCP_SKB_CB(buff)->when = tcp_time_stamp;
3114 tcp_transmit_skb(sk, buff, 0, sk_gfp_atomic(sk, GFP_ATOMIC));
3117 /* This routine sends a packet with an out of date sequence
3118 * number. It assumes the other end will try to ack it.
3120 * Question: what should we make while urgent mode?
3121 * 4.4BSD forces sending single byte of data. We cannot send
3122 * out of window data, because we have SND.NXT==SND.MAX...
3124 * Current solution: to send TWO zero-length segments in urgent mode:
3125 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
3126 * out-of-date with SND.UNA-1 to probe window.
3128 static int tcp_xmit_probe_skb(struct sock *sk, int urgent)
3130 struct tcp_sock *tp = tcp_sk(sk);
3131 struct sk_buff *skb;
3133 /* We don't queue it, tcp_transmit_skb() sets ownership. */
3134 skb = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
3138 /* Reserve space for headers and set control bits. */
3139 skb_reserve(skb, MAX_TCP_HEADER);
3140 /* Use a previous sequence. This should cause the other
3141 * end to send an ack. Don't queue or clone SKB, just
3144 tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
3145 TCP_SKB_CB(skb)->when = tcp_time_stamp;
3146 return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC);
3149 void tcp_send_window_probe(struct sock *sk)
3151 if (sk->sk_state == TCP_ESTABLISHED) {
3152 tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
3153 tcp_xmit_probe_skb(sk, 0);
3157 /* Initiate keepalive or window probe from timer. */
3158 int tcp_write_wakeup(struct sock *sk)
3160 struct tcp_sock *tp = tcp_sk(sk);
3161 struct sk_buff *skb;
3163 if (sk->sk_state == TCP_CLOSE)
3166 if ((skb = tcp_send_head(sk)) != NULL &&
3167 before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
3169 unsigned int mss = tcp_current_mss(sk);
3170 unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3172 if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
3173 tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
3175 /* We are probing the opening of a window
3176 * but the window size is != 0
3177 * must have been a result SWS avoidance ( sender )
3179 if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
3181 seg_size = min(seg_size, mss);
3182 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3183 if (tcp_fragment(sk, skb, seg_size, mss))
3185 } else if (!tcp_skb_pcount(skb))
3186 tcp_set_skb_tso_segs(sk, skb, mss);
3188 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3189 TCP_SKB_CB(skb)->when = tcp_time_stamp;
3190 err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3192 tcp_event_new_data_sent(sk, skb);
3195 if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
3196 tcp_xmit_probe_skb(sk, 1);
3197 return tcp_xmit_probe_skb(sk, 0);
3201 /* A window probe timeout has occurred. If window is not closed send
3202 * a partial packet else a zero probe.
3204 void tcp_send_probe0(struct sock *sk)
3206 struct inet_connection_sock *icsk = inet_csk(sk);
3207 struct tcp_sock *tp = tcp_sk(sk);
3210 err = tcp_write_wakeup(sk);
3212 if (tp->packets_out || !tcp_send_head(sk)) {
3213 /* Cancel probe timer, if it is not required. */
3214 icsk->icsk_probes_out = 0;
3215 icsk->icsk_backoff = 0;
3220 if (icsk->icsk_backoff < sysctl_tcp_retries2)
3221 icsk->icsk_backoff++;
3222 icsk->icsk_probes_out++;
3223 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3224 min(icsk->icsk_rto << icsk->icsk_backoff, TCP_RTO_MAX),
3227 /* If packet was not sent due to local congestion,
3228 * do not backoff and do not remember icsk_probes_out.
3229 * Let local senders to fight for local resources.
3231 * Use accumulated backoff yet.
3233 if (!icsk->icsk_probes_out)
3234 icsk->icsk_probes_out = 1;
3235 inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3236 min(icsk->icsk_rto << icsk->icsk_backoff,
3237 TCP_RESOURCE_PROBE_INTERVAL),