Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
8663e02a VY |
2 | /* |
3 | * IPV6 GSO/GRO offload support | |
4 | * Linux INET6 implementation | |
5 | * | |
8663e02a VY |
6 | * TCPv6 GSO/GRO support |
7 | */ | |
028e0a47 | 8 | #include <linux/indirect_call_wrapper.h> |
8663e02a | 9 | #include <linux/skbuff.h> |
c9d1d23e | 10 | #include <net/inet6_hashtables.h> |
4721031c | 11 | #include <net/gro.h> |
8663e02a VY |
12 | #include <net/protocol.h> |
13 | #include <net/tcp.h> | |
14 | #include <net/ip6_checksum.h> | |
15 | #include "ip6_offload.h" | |
16 | ||
c9d1d23e FF |
17 | static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, |
18 | struct tcphdr *th) | |
19 | { | |
20 | #if IS_ENABLED(CONFIG_IPV6) | |
21 | const struct ipv6hdr *hdr; | |
22 | struct sk_buff *p; | |
23 | struct sock *sk; | |
24 | struct net *net; | |
25 | int iif, sdif; | |
26 | ||
27 | if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST))) | |
28 | return; | |
29 | ||
30 | p = tcp_gro_lookup(head, th); | |
31 | if (p) { | |
32 | NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; | |
33 | return; | |
34 | } | |
35 | ||
36 | inet6_get_iif_sdif(skb, &iif, &sdif); | |
37 | hdr = skb_gro_network_header(skb); | |
38 | net = dev_net(skb->dev); | |
39 | sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, | |
40 | &hdr->saddr, th->source, | |
41 | &hdr->daddr, ntohs(th->dest), | |
42 | iif, sdif); | |
43 | NAPI_GRO_CB(skb)->is_flist = !sk; | |
44 | if (sk) | |
45 | sock_put(sk); | |
46 | #endif /* IS_ENABLED(CONFIG_IPV6) */ | |
47 | } | |
48 | ||
028e0a47 PA |
49 | INDIRECT_CALLABLE_SCOPE |
50 | struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb) | |
8663e02a | 51 | { |
7516b27c FF |
52 | struct tcphdr *th; |
53 | ||
cc5c00bb | 54 | /* Don't bother verifying checksum if we're going to flush anyway. */ |
149d0774 TH |
55 | if (!NAPI_GRO_CB(skb)->flush && |
56 | skb_gro_checksum_validate(skb, IPPROTO_TCP, | |
7516b27c FF |
57 | ip6_gro_compute_pseudo)) |
58 | goto flush; | |
59 | ||
60 | th = tcp_gro_pull_header(skb); | |
61 | if (!th) | |
62 | goto flush; | |
63 | ||
c9d1d23e FF |
64 | tcp6_check_fraglist_gro(head, skb, th); |
65 | ||
7516b27c | 66 | return tcp_gro_receive(head, skb, th); |
8663e02a | 67 | |
7516b27c FF |
68 | flush: |
69 | NAPI_GRO_CB(skb)->flush = 1; | |
70 | return NULL; | |
8663e02a VY |
71 | } |
72 | ||
028e0a47 | 73 | INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff) |
8663e02a VY |
74 | { |
75 | const struct ipv6hdr *iph = ipv6_hdr(skb); | |
76 | struct tcphdr *th = tcp_hdr(skb); | |
77 | ||
8d95dc47 FF |
78 | if (unlikely(NAPI_GRO_CB(skb)->is_flist)) { |
79 | skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6; | |
80 | skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; | |
81 | ||
82 | __skb_incr_checksum_unnecessary(skb); | |
83 | ||
84 | return 0; | |
85 | } | |
86 | ||
299603e8 JC |
87 | th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr, |
88 | &iph->daddr, 0); | |
c3caf119 | 89 | skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6; |
8663e02a | 90 | |
b1f2abcf PP |
91 | tcp_gro_complete(skb); |
92 | return 0; | |
8663e02a VY |
93 | } |
94 | ||
bee88cd5 FF |
95 | static void __tcpv6_gso_segment_csum(struct sk_buff *seg, |
96 | __be16 *oldport, __be16 newport) | |
97 | { | |
98 | struct tcphdr *th; | |
99 | ||
100 | if (*oldport == newport) | |
101 | return; | |
102 | ||
103 | th = tcp_hdr(seg); | |
104 | inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false); | |
105 | *oldport = newport; | |
106 | } | |
107 | ||
108 | static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs) | |
109 | { | |
110 | const struct tcphdr *th; | |
111 | const struct ipv6hdr *iph; | |
112 | struct sk_buff *seg; | |
113 | struct tcphdr *th2; | |
114 | struct ipv6hdr *iph2; | |
115 | ||
116 | seg = segs; | |
117 | th = tcp_hdr(seg); | |
118 | iph = ipv6_hdr(seg); | |
119 | th2 = tcp_hdr(seg->next); | |
120 | iph2 = ipv6_hdr(seg->next); | |
121 | ||
122 | if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) && | |
123 | ipv6_addr_equal(&iph->saddr, &iph2->saddr) && | |
124 | ipv6_addr_equal(&iph->daddr, &iph2->daddr)) | |
125 | return segs; | |
126 | ||
127 | while ((seg = seg->next)) { | |
128 | th2 = tcp_hdr(seg); | |
129 | iph2 = ipv6_hdr(seg); | |
130 | ||
131 | iph2->saddr = iph->saddr; | |
132 | iph2->daddr = iph->daddr; | |
133 | __tcpv6_gso_segment_csum(seg, &th2->source, th->source); | |
134 | __tcpv6_gso_segment_csum(seg, &th2->dest, th->dest); | |
135 | } | |
136 | ||
137 | return segs; | |
138 | } | |
139 | ||
140 | static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb, | |
141 | netdev_features_t features) | |
142 | { | |
143 | skb = skb_segment_list(skb, features, skb_mac_header_len(skb)); | |
144 | if (IS_ERR(skb)) | |
145 | return skb; | |
146 | ||
147 | return __tcpv6_gso_segment_list_csum(skb); | |
148 | } | |
149 | ||
74abc20c ED |
150 | static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb, |
151 | netdev_features_t features) | |
d020f8f7 TH |
152 | { |
153 | struct tcphdr *th; | |
154 | ||
121d57af WB |
155 | if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)) |
156 | return ERR_PTR(-EINVAL); | |
157 | ||
d020f8f7 TH |
158 | if (!pskb_may_pull(skb, sizeof(*th))) |
159 | return ERR_PTR(-EINVAL); | |
160 | ||
bee88cd5 FF |
161 | if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) |
162 | return __tcp6_gso_segment_list(skb, features); | |
163 | ||
d020f8f7 TH |
164 | if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { |
165 | const struct ipv6hdr *ipv6h = ipv6_hdr(skb); | |
166 | struct tcphdr *th = tcp_hdr(skb); | |
167 | ||
168 | /* Set up pseudo header, usually expect stack to have done | |
169 | * this. | |
170 | */ | |
171 | ||
172 | th->check = 0; | |
173 | skb->ip_summed = CHECKSUM_PARTIAL; | |
174 | __tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr); | |
175 | } | |
176 | ||
177 | return tcp_gso_segment(skb, features); | |
178 | } | |
8663e02a VY |
179 | |
180 | int __init tcpv6_offload_init(void) | |
181 | { | |
0139806e ED |
182 | net_hotdata.tcpv6_offload = (struct net_offload) { |
183 | .callbacks = { | |
184 | .gso_segment = tcp6_gso_segment, | |
185 | .gro_receive = tcp6_gro_receive, | |
186 | .gro_complete = tcp6_gro_complete, | |
187 | }, | |
188 | }; | |
189 | return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP); | |
8663e02a | 190 | } |