Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
406ef77c HX |
2 | /* |
3 | * xfrm_output.c - Common IPsec encapsulation code. | |
4 | * | |
5 | * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> | |
406ef77c HX |
6 | */ |
7 | ||
8 | #include <linux/errno.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/netdevice.h> | |
862b82c6 | 11 | #include <linux/netfilter.h> |
406ef77c | 12 | #include <linux/skbuff.h> |
5a0e3ad6 | 13 | #include <linux/slab.h> |
406ef77c | 14 | #include <linux/spinlock.h> |
406ef77c | 15 | #include <net/dst.h> |
d457a0e3 | 16 | #include <net/gso.h> |
6d64be3d | 17 | #include <net/icmp.h> |
1de70830 | 18 | #include <net/inet_ecn.h> |
406ef77c HX |
19 | #include <net/xfrm.h> |
20 | ||
f3075f48 FW |
21 | #if IS_ENABLED(CONFIG_IPV6) |
22 | #include <net/ip6_route.h> | |
23 | #include <net/ipv6_stubs.h> | |
24 | #endif | |
25 | ||
1de70830 FW |
26 | #include "xfrm_inout.h" |
27 | ||
0c4b51f0 | 28 | static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb); |
0c620e97 | 29 | static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb); |
c6581a45 | 30 | |
26b2072e | 31 | static int xfrm_skb_check_space(struct sk_buff *skb) |
83815dea | 32 | { |
adf30907 | 33 | struct dst_entry *dst = skb_dst(skb); |
550ade84 | 34 | int nhead = dst->header_len + LL_RESERVED_SPACE(dst->dev) |
83815dea | 35 | - skb_headroom(skb); |
f5184d26 | 36 | int ntail = dst->dev->needed_tailroom - skb_tailroom(skb); |
83815dea | 37 | |
d01dbeb6 HX |
38 | if (nhead <= 0) { |
39 | if (ntail <= 0) | |
40 | return 0; | |
41 | nhead = 0; | |
42 | } else if (ntail < 0) | |
43 | ntail = 0; | |
44 | ||
45 | return pskb_expand_head(skb, nhead, ntail, GFP_ATOMIC); | |
83815dea HX |
46 | } |
47 | ||
9449c3cd YX |
48 | /* Children define the path of the packet through the |
49 | * Linux networking. Thus, destinations are stackable. | |
50 | */ | |
51 | ||
52 | static struct dst_entry *skb_dst_pop(struct sk_buff *skb) | |
53 | { | |
b92cf4aa | 54 | struct dst_entry *child = dst_clone(xfrm_dst_child(skb_dst(skb))); |
9449c3cd YX |
55 | |
56 | skb_dst_drop(skb); | |
57 | return child; | |
58 | } | |
59 | ||
0c620e97 FW |
60 | /* Add encapsulation header. |
61 | * | |
62 | * The IP header will be moved forward to make space for the encapsulation | |
63 | * header. | |
64 | */ | |
65 | static int xfrm4_transport_output(struct xfrm_state *x, struct sk_buff *skb) | |
66 | { | |
0c620e97 FW |
67 | struct iphdr *iph = ip_hdr(skb); |
68 | int ihl = iph->ihl * 4; | |
69 | ||
70 | skb_set_inner_transport_header(skb, skb_transport_offset(skb)); | |
71 | ||
72 | skb_set_network_header(skb, -x->props.header_len); | |
73 | skb->mac_header = skb->network_header + | |
74 | offsetof(struct iphdr, protocol); | |
75 | skb->transport_header = skb->network_header + ihl; | |
76 | __skb_pull(skb, ihl); | |
77 | memmove(skb_network_header(skb), iph, ihl); | |
78 | return 0; | |
0c620e97 FW |
79 | } |
80 | ||
37b9e7eb | 81 | #if IS_ENABLED(CONFIG_IPV6_MIP6) |
3ca5ca83 | 82 | static int mip6_rthdr_offset(struct sk_buff *skb, u8 **nexthdr, int type) |
37b9e7eb | 83 | { |
37b9e7eb | 84 | const unsigned char *nh = skb_network_header(skb); |
3ca5ca83 FW |
85 | unsigned int offset = sizeof(struct ipv6hdr); |
86 | unsigned int packet_len; | |
37b9e7eb FW |
87 | int found_rhdr = 0; |
88 | ||
3ca5ca83 | 89 | packet_len = skb_tail_pointer(skb) - nh; |
37b9e7eb FW |
90 | *nexthdr = &ipv6_hdr(skb)->nexthdr; |
91 | ||
3ca5ca83 FW |
92 | while (offset <= packet_len) { |
93 | struct ipv6_opt_hdr *exthdr; | |
94 | ||
37b9e7eb FW |
95 | switch (**nexthdr) { |
96 | case NEXTHDR_HOP: | |
97 | break; | |
98 | case NEXTHDR_ROUTING: | |
3ca5ca83 FW |
99 | if (type == IPPROTO_ROUTING && offset + 3 <= packet_len) { |
100 | struct ipv6_rt_hdr *rt; | |
101 | ||
102 | rt = (struct ipv6_rt_hdr *)(nh + offset); | |
103 | if (rt->type != 0) | |
104 | return offset; | |
105 | } | |
37b9e7eb FW |
106 | found_rhdr = 1; |
107 | break; | |
108 | case NEXTHDR_DEST: | |
109 | /* HAO MUST NOT appear more than once. | |
110 | * XXX: It is better to try to find by the end of | |
111 | * XXX: packet if HAO exists. | |
112 | */ | |
113 | if (ipv6_find_tlv(skb, offset, IPV6_TLV_HAO) >= 0) { | |
114 | net_dbg_ratelimited("mip6: hao exists already, override\n"); | |
115 | return offset; | |
116 | } | |
117 | ||
118 | if (found_rhdr) | |
119 | return offset; | |
120 | ||
121 | break; | |
122 | default: | |
123 | return offset; | |
124 | } | |
125 | ||
3ca5ca83 FW |
126 | if (offset + sizeof(struct ipv6_opt_hdr) > packet_len) |
127 | return -EINVAL; | |
848b18fb | 128 | |
3ca5ca83 FW |
129 | exthdr = (struct ipv6_opt_hdr *)(skb_network_header(skb) + |
130 | offset); | |
848b18fb | 131 | offset += ipv6_optlen(exthdr); |
3ca5ca83 FW |
132 | if (offset > IPV6_MAXPLEN) |
133 | return -EINVAL; | |
848b18fb | 134 | *nexthdr = &exthdr->nexthdr; |
848b18fb FW |
135 | } |
136 | ||
3ca5ca83 | 137 | return -EINVAL; |
848b18fb | 138 | } |
37b9e7eb FW |
139 | #endif |
140 | ||
30ad6a84 | 141 | #if IS_ENABLED(CONFIG_IPV6) |
9acf4d3b FW |
142 | static int xfrm6_hdr_offset(struct xfrm_state *x, struct sk_buff *skb, u8 **prevhdr) |
143 | { | |
37b9e7eb FW |
144 | switch (x->type->proto) { |
145 | #if IS_ENABLED(CONFIG_IPV6_MIP6) | |
146 | case IPPROTO_DSTOPTS: | |
848b18fb | 147 | case IPPROTO_ROUTING: |
3ca5ca83 | 148 | return mip6_rthdr_offset(skb, prevhdr, x->type->proto); |
37b9e7eb FW |
149 | #endif |
150 | default: | |
151 | break; | |
152 | } | |
153 | ||
d1002d24 | 154 | return ip6_find_1stfragopt(skb, prevhdr); |
9acf4d3b | 155 | } |
30ad6a84 | 156 | #endif |
9acf4d3b | 157 | |
0c620e97 FW |
158 | /* Add encapsulation header. |
159 | * | |
160 | * The IP header and mutable extension headers will be moved forward to make | |
161 | * space for the encapsulation header. | |
162 | */ | |
163 | static int xfrm6_transport_output(struct xfrm_state *x, struct sk_buff *skb) | |
164 | { | |
4c145dce | 165 | #if IS_ENABLED(CONFIG_IPV6) |
0c620e97 FW |
166 | struct ipv6hdr *iph; |
167 | u8 *prevhdr; | |
168 | int hdr_len; | |
169 | ||
170 | iph = ipv6_hdr(skb); | |
171 | skb_set_inner_transport_header(skb, skb_transport_offset(skb)); | |
172 | ||
9acf4d3b | 173 | hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr); |
0c620e97 FW |
174 | if (hdr_len < 0) |
175 | return hdr_len; | |
176 | skb_set_mac_header(skb, | |
177 | (prevhdr - x->props.header_len) - skb->data); | |
178 | skb_set_network_header(skb, -x->props.header_len); | |
179 | skb->transport_header = skb->network_header + hdr_len; | |
180 | __skb_pull(skb, hdr_len); | |
181 | memmove(ipv6_hdr(skb), iph, hdr_len); | |
182 | return 0; | |
183 | #else | |
184 | WARN_ON_ONCE(1); | |
4c145dce | 185 | return -EAFNOSUPPORT; |
0c620e97 FW |
186 | #endif |
187 | } | |
188 | ||
189 | /* Add route optimization header space. | |
190 | * | |
191 | * The IP header and mutable extension headers will be moved forward to make | |
192 | * space for the route optimization header. | |
193 | */ | |
194 | static int xfrm6_ro_output(struct xfrm_state *x, struct sk_buff *skb) | |
195 | { | |
4c145dce | 196 | #if IS_ENABLED(CONFIG_IPV6) |
0c620e97 FW |
197 | struct ipv6hdr *iph; |
198 | u8 *prevhdr; | |
199 | int hdr_len; | |
200 | ||
201 | iph = ipv6_hdr(skb); | |
202 | ||
9acf4d3b | 203 | hdr_len = xfrm6_hdr_offset(x, skb, &prevhdr); |
0c620e97 FW |
204 | if (hdr_len < 0) |
205 | return hdr_len; | |
206 | skb_set_mac_header(skb, | |
207 | (prevhdr - x->props.header_len) - skb->data); | |
208 | skb_set_network_header(skb, -x->props.header_len); | |
209 | skb->transport_header = skb->network_header + hdr_len; | |
210 | __skb_pull(skb, hdr_len); | |
211 | memmove(ipv6_hdr(skb), iph, hdr_len); | |
212 | ||
0c620e97 FW |
213 | return 0; |
214 | #else | |
215 | WARN_ON_ONCE(1); | |
4c145dce | 216 | return -EAFNOSUPPORT; |
0c620e97 FW |
217 | #endif |
218 | } | |
219 | ||
1de70830 FW |
220 | /* Add encapsulation header. |
221 | * | |
222 | * The top IP header will be constructed per draft-nikander-esp-beet-mode-06.txt. | |
223 | */ | |
224 | static int xfrm4_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb) | |
225 | { | |
226 | struct ip_beet_phdr *ph; | |
227 | struct iphdr *top_iph; | |
228 | int hdrlen, optlen; | |
229 | ||
230 | hdrlen = 0; | |
231 | optlen = XFRM_MODE_SKB_CB(skb)->optlen; | |
232 | if (unlikely(optlen)) | |
233 | hdrlen += IPV4_BEET_PHMAXLEN - (optlen & 4); | |
234 | ||
235 | skb_set_network_header(skb, -x->props.header_len - hdrlen + | |
236 | (XFRM_MODE_SKB_CB(skb)->ihl - sizeof(*top_iph))); | |
237 | if (x->sel.family != AF_INET6) | |
238 | skb->network_header += IPV4_BEET_PHMAXLEN; | |
239 | skb->mac_header = skb->network_header + | |
240 | offsetof(struct iphdr, protocol); | |
241 | skb->transport_header = skb->network_header + sizeof(*top_iph); | |
242 | ||
243 | xfrm4_beet_make_header(skb); | |
244 | ||
245 | ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdrlen); | |
246 | ||
247 | top_iph = ip_hdr(skb); | |
248 | ||
249 | if (unlikely(optlen)) { | |
250 | if (WARN_ON(optlen < 0)) | |
251 | return -EINVAL; | |
252 | ||
253 | ph->padlen = 4 - (optlen & 4); | |
254 | ph->hdrlen = optlen / 8; | |
255 | ph->nexthdr = top_iph->protocol; | |
256 | if (ph->padlen) | |
257 | memset(ph + 1, IPOPT_NOP, ph->padlen); | |
258 | ||
259 | top_iph->protocol = IPPROTO_BEETPH; | |
260 | top_iph->ihl = sizeof(struct iphdr) / 4; | |
261 | } | |
262 | ||
263 | top_iph->saddr = x->props.saddr.a4; | |
264 | top_iph->daddr = x->id.daddr.a4; | |
265 | ||
266 | return 0; | |
267 | } | |
268 | ||
269 | /* Add encapsulation header. | |
270 | * | |
271 | * The top IP header will be constructed per RFC 2401. | |
272 | */ | |
273 | static int xfrm4_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb) | |
274 | { | |
6821ad87 | 275 | bool small_ipv6 = (skb->protocol == htons(ETH_P_IPV6)) && (skb->len <= IPV6_MIN_MTU); |
1de70830 FW |
276 | struct dst_entry *dst = skb_dst(skb); |
277 | struct iphdr *top_iph; | |
278 | int flags; | |
279 | ||
280 | skb_set_inner_network_header(skb, skb_network_offset(skb)); | |
281 | skb_set_inner_transport_header(skb, skb_transport_offset(skb)); | |
282 | ||
283 | skb_set_network_header(skb, -x->props.header_len); | |
284 | skb->mac_header = skb->network_header + | |
285 | offsetof(struct iphdr, protocol); | |
286 | skb->transport_header = skb->network_header + sizeof(*top_iph); | |
287 | top_iph = ip_hdr(skb); | |
288 | ||
289 | top_iph->ihl = 5; | |
290 | top_iph->version = 4; | |
291 | ||
292 | top_iph->protocol = xfrm_af2proto(skb_dst(skb)->ops->family); | |
293 | ||
294 | /* DS disclosing depends on XFRM_SA_XFLAG_DONT_ENCAP_DSCP */ | |
295 | if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP) | |
296 | top_iph->tos = 0; | |
297 | else | |
298 | top_iph->tos = XFRM_MODE_SKB_CB(skb)->tos; | |
299 | top_iph->tos = INET_ECN_encapsulate(top_iph->tos, | |
300 | XFRM_MODE_SKB_CB(skb)->tos); | |
301 | ||
302 | flags = x->props.flags; | |
303 | if (flags & XFRM_STATE_NOECN) | |
304 | IP_ECN_clear(top_iph); | |
305 | ||
6821ad87 | 306 | top_iph->frag_off = (flags & XFRM_STATE_NOPMTUDISC) || small_ipv6 ? |
1de70830 FW |
307 | 0 : (XFRM_MODE_SKB_CB(skb)->frag_off & htons(IP_DF)); |
308 | ||
309 | top_iph->ttl = ip4_dst_hoplimit(xfrm_dst_child(dst)); | |
310 | ||
311 | top_iph->saddr = x->props.saddr.a4; | |
312 | top_iph->daddr = x->id.daddr.a4; | |
313 | ip_select_ident(dev_net(dst->dev), skb, NULL); | |
314 | ||
315 | return 0; | |
316 | } | |
317 | ||
318 | #if IS_ENABLED(CONFIG_IPV6) | |
319 | static int xfrm6_tunnel_encap_add(struct xfrm_state *x, struct sk_buff *skb) | |
320 | { | |
321 | struct dst_entry *dst = skb_dst(skb); | |
322 | struct ipv6hdr *top_iph; | |
323 | int dsfield; | |
324 | ||
325 | skb_set_inner_network_header(skb, skb_network_offset(skb)); | |
326 | skb_set_inner_transport_header(skb, skb_transport_offset(skb)); | |
327 | ||
328 | skb_set_network_header(skb, -x->props.header_len); | |
329 | skb->mac_header = skb->network_header + | |
330 | offsetof(struct ipv6hdr, nexthdr); | |
331 | skb->transport_header = skb->network_header + sizeof(*top_iph); | |
332 | top_iph = ipv6_hdr(skb); | |
333 | ||
334 | top_iph->version = 6; | |
335 | ||
336 | memcpy(top_iph->flow_lbl, XFRM_MODE_SKB_CB(skb)->flow_lbl, | |
337 | sizeof(top_iph->flow_lbl)); | |
338 | top_iph->nexthdr = xfrm_af2proto(skb_dst(skb)->ops->family); | |
339 | ||
340 | if (x->props.extra_flags & XFRM_SA_XFLAG_DONT_ENCAP_DSCP) | |
341 | dsfield = 0; | |
342 | else | |
343 | dsfield = XFRM_MODE_SKB_CB(skb)->tos; | |
344 | dsfield = INET_ECN_encapsulate(dsfield, XFRM_MODE_SKB_CB(skb)->tos); | |
345 | if (x->props.flags & XFRM_STATE_NOECN) | |
346 | dsfield &= ~INET_ECN_MASK; | |
347 | ipv6_change_dsfield(top_iph, 0, dsfield); | |
348 | top_iph->hop_limit = ip6_dst_hoplimit(xfrm_dst_child(dst)); | |
349 | top_iph->saddr = *(struct in6_addr *)&x->props.saddr; | |
350 | top_iph->daddr = *(struct in6_addr *)&x->id.daddr; | |
351 | return 0; | |
352 | } | |
353 | ||
354 | static int xfrm6_beet_encap_add(struct xfrm_state *x, struct sk_buff *skb) | |
355 | { | |
356 | struct ipv6hdr *top_iph; | |
357 | struct ip_beet_phdr *ph; | |
358 | int optlen, hdr_len; | |
359 | ||
360 | hdr_len = 0; | |
361 | optlen = XFRM_MODE_SKB_CB(skb)->optlen; | |
362 | if (unlikely(optlen)) | |
363 | hdr_len += IPV4_BEET_PHMAXLEN - (optlen & 4); | |
364 | ||
365 | skb_set_network_header(skb, -x->props.header_len - hdr_len); | |
366 | if (x->sel.family != AF_INET6) | |
367 | skb->network_header += IPV4_BEET_PHMAXLEN; | |
368 | skb->mac_header = skb->network_header + | |
369 | offsetof(struct ipv6hdr, nexthdr); | |
370 | skb->transport_header = skb->network_header + sizeof(*top_iph); | |
371 | ph = __skb_pull(skb, XFRM_MODE_SKB_CB(skb)->ihl - hdr_len); | |
372 | ||
373 | xfrm6_beet_make_header(skb); | |
374 | ||
375 | top_iph = ipv6_hdr(skb); | |
376 | if (unlikely(optlen)) { | |
377 | if (WARN_ON(optlen < 0)) | |
378 | return -EINVAL; | |
379 | ||
380 | ph->padlen = 4 - (optlen & 4); | |
381 | ph->hdrlen = optlen / 8; | |
382 | ph->nexthdr = top_iph->nexthdr; | |
383 | if (ph->padlen) | |
384 | memset(ph + 1, IPOPT_NOP, ph->padlen); | |
385 | ||
386 | top_iph->nexthdr = IPPROTO_BEETPH; | |
387 | } | |
388 | ||
389 | top_iph->saddr = *(struct in6_addr *)&x->props.saddr; | |
390 | top_iph->daddr = *(struct in6_addr *)&x->id.daddr; | |
391 | return 0; | |
392 | } | |
393 | #endif | |
394 | ||
395 | /* Add encapsulation header. | |
396 | * | |
397 | * On exit, the transport header will be set to the start of the | |
398 | * encapsulation header to be filled in by x->type->output and the mac | |
399 | * header will be set to the nextheader (protocol for IPv4) field of the | |
400 | * extension header directly preceding the encapsulation header, or in | |
401 | * its absence, that of the top IP header. | |
402 | * The value of the network header will always point to the top IP header | |
403 | * while skb->data will point to the payload. | |
404 | */ | |
0c620e97 FW |
405 | static int xfrm4_prepare_output(struct xfrm_state *x, struct sk_buff *skb) |
406 | { | |
407 | int err; | |
408 | ||
409 | err = xfrm_inner_extract_output(x, skb); | |
410 | if (err) | |
411 | return err; | |
412 | ||
413 | IPCB(skb)->flags |= IPSKB_XFRM_TUNNEL_SIZE; | |
414 | skb->protocol = htons(ETH_P_IP); | |
415 | ||
f4796398 | 416 | switch (x->props.mode) { |
1de70830 FW |
417 | case XFRM_MODE_BEET: |
418 | return xfrm4_beet_encap_add(x, skb); | |
419 | case XFRM_MODE_TUNNEL: | |
420 | return xfrm4_tunnel_encap_add(x, skb); | |
421 | } | |
422 | ||
423 | WARN_ON_ONCE(1); | |
424 | return -EOPNOTSUPP; | |
0c620e97 FW |
425 | } |
426 | ||
427 | static int xfrm6_prepare_output(struct xfrm_state *x, struct sk_buff *skb) | |
428 | { | |
429 | #if IS_ENABLED(CONFIG_IPV6) | |
430 | int err; | |
431 | ||
432 | err = xfrm_inner_extract_output(x, skb); | |
433 | if (err) | |
434 | return err; | |
435 | ||
436 | skb->ignore_df = 1; | |
437 | skb->protocol = htons(ETH_P_IPV6); | |
438 | ||
f4796398 | 439 | switch (x->props.mode) { |
1de70830 FW |
440 | case XFRM_MODE_BEET: |
441 | return xfrm6_beet_encap_add(x, skb); | |
442 | case XFRM_MODE_TUNNEL: | |
443 | return xfrm6_tunnel_encap_add(x, skb); | |
444 | default: | |
445 | WARN_ON_ONCE(1); | |
446 | return -EOPNOTSUPP; | |
447 | } | |
0c620e97 | 448 | #endif |
1de70830 FW |
449 | WARN_ON_ONCE(1); |
450 | return -EAFNOSUPPORT; | |
0c620e97 FW |
451 | } |
452 | ||
453 | static int xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb) | |
454 | { | |
f4796398 | 455 | switch (x->props.mode) { |
0c620e97 FW |
456 | case XFRM_MODE_BEET: |
457 | case XFRM_MODE_TUNNEL: | |
f4796398 | 458 | if (x->props.family == AF_INET) |
0c620e97 | 459 | return xfrm4_prepare_output(x, skb); |
f4796398 | 460 | if (x->props.family == AF_INET6) |
0c620e97 FW |
461 | return xfrm6_prepare_output(x, skb); |
462 | break; | |
463 | case XFRM_MODE_TRANSPORT: | |
f4796398 | 464 | if (x->props.family == AF_INET) |
0c620e97 | 465 | return xfrm4_transport_output(x, skb); |
f4796398 | 466 | if (x->props.family == AF_INET6) |
0c620e97 FW |
467 | return xfrm6_transport_output(x, skb); |
468 | break; | |
469 | case XFRM_MODE_ROUTEOPTIMIZATION: | |
f4796398 | 470 | if (x->props.family == AF_INET6) |
0c620e97 FW |
471 | return xfrm6_ro_output(x, skb); |
472 | WARN_ON_ONCE(1); | |
473 | break; | |
474 | default: | |
475 | WARN_ON_ONCE(1); | |
476 | break; | |
477 | } | |
478 | ||
479 | return -EOPNOTSUPP; | |
480 | } | |
481 | ||
482 | #if IS_ENABLED(CONFIG_NET_PKTGEN) | |
483 | int pktgen_xfrm_outer_mode_output(struct xfrm_state *x, struct sk_buff *skb) | |
484 | { | |
485 | return xfrm_outer_mode_output(x, skb); | |
486 | } | |
487 | EXPORT_SYMBOL_GPL(pktgen_xfrm_outer_mode_output); | |
488 | #endif | |
489 | ||
c6581a45 | 490 | static int xfrm_output_one(struct sk_buff *skb, int err) |
406ef77c | 491 | { |
adf30907 | 492 | struct dst_entry *dst = skb_dst(skb); |
406ef77c | 493 | struct xfrm_state *x = dst->xfrm; |
a6483b79 | 494 | struct net *net = xs_net(x); |
406ef77c | 495 | |
f8a70afa | 496 | if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET) |
c6581a45 | 497 | goto resume; |
406ef77c HX |
498 | |
499 | do { | |
26b2072e | 500 | err = xfrm_skb_check_space(skb); |
b15c4bcd | 501 | if (err) { |
59c9940e | 502 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); |
910ef70a | 503 | goto error_nolock; |
b15c4bcd | 504 | } |
910ef70a | 505 | |
9b42c1f1 | 506 | skb->mark = xfrm_smark_get(skb->mark, x); |
077fbac4 | 507 | |
0c620e97 | 508 | err = xfrm_outer_mode_output(x, skb); |
b15c4bcd | 509 | if (err) { |
59c9940e | 510 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR); |
910ef70a | 511 | goto error_nolock; |
b15c4bcd | 512 | } |
a2deb6d2 | 513 | |
406ef77c | 514 | spin_lock_bh(&x->lock); |
bb65a9cb LR |
515 | |
516 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { | |
517 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEINVALID); | |
497574c7 | 518 | err = -EINVAL; |
fa8599db | 519 | goto error; |
bb65a9cb LR |
520 | } |
521 | ||
910ef70a | 522 | err = xfrm_state_check_expire(x); |
b15c4bcd | 523 | if (err) { |
59c9940e | 524 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEEXPIRED); |
406ef77c | 525 | goto error; |
b15c4bcd | 526 | } |
406ef77c | 527 | |
b5a1d1fe | 528 | err = xfrm_replay_overflow(x, skb); |
9fdc4883 SK |
529 | if (err) { |
530 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATESEQERROR); | |
531 | goto error; | |
436a0a40 HX |
532 | } |
533 | ||
406ef77c HX |
534 | x->curlft.bytes += skb->len; |
535 | x->curlft.packets++; | |
f7fe25a6 | 536 | x->lastused = ktime_get_real_seconds(); |
406ef77c | 537 | |
406ef77c HX |
538 | spin_unlock_bh(&x->lock); |
539 | ||
3bc07321 | 540 | skb_dst_force(skb); |
9e143793 SK |
541 | if (!skb_dst(skb)) { |
542 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); | |
533555e5 | 543 | err = -EHOSTUNREACH; |
9e143793 SK |
544 | goto error_nolock; |
545 | } | |
3bc07321 | 546 | |
d77e38e6 SK |
547 | if (xfrm_offload(skb)) { |
548 | x->type_offload->encap(x, skb); | |
549 | } else { | |
73b9fc49 SK |
550 | /* Inner headers are invalid now. */ |
551 | skb->encapsulation = 0; | |
552 | ||
d77e38e6 SK |
553 | err = x->type->output(x, skb); |
554 | if (err == -EINPROGRESS) | |
555 | goto out; | |
556 | } | |
c6581a45 HX |
557 | |
558 | resume: | |
0aa64774 | 559 | if (err) { |
59c9940e | 560 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR); |
b7c6538c | 561 | goto error_nolock; |
0aa64774 | 562 | } |
b7c6538c | 563 | |
8764ab2c | 564 | dst = skb_dst_pop(skb); |
adf30907 | 565 | if (!dst) { |
59c9940e | 566 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); |
406ef77c HX |
567 | err = -EHOSTUNREACH; |
568 | goto error_nolock; | |
569 | } | |
e433430a | 570 | skb_dst_set(skb, dst); |
406ef77c | 571 | x = dst->xfrm; |
c9500d7b | 572 | } while (x && !(x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL)); |
406ef77c | 573 | |
ebd4687a | 574 | return 0; |
406ef77c | 575 | |
406ef77c HX |
576 | error: |
577 | spin_unlock_bh(&x->lock); | |
862b82c6 HX |
578 | error_nolock: |
579 | kfree_skb(skb); | |
ebd4687a JS |
580 | out: |
581 | return err; | |
862b82c6 HX |
582 | } |
583 | ||
9ab1265d | 584 | int xfrm_output_resume(struct sock *sk, struct sk_buff *skb, int err) |
862b82c6 | 585 | { |
29a26a56 | 586 | struct net *net = xs_net(skb_dst(skb)->xfrm); |
be10de0a | 587 | |
c6581a45 | 588 | while (likely((err = xfrm_output_one(skb, err)) == 0)) { |
895b5c9f | 589 | nf_reset_ct(skb); |
862b82c6 | 590 | |
9ab1265d | 591 | err = skb_dst(skb)->ops->local_out(net, sk, skb); |
862b82c6 | 592 | if (unlikely(err != 1)) |
c6581a45 | 593 | goto out; |
862b82c6 | 594 | |
adf30907 | 595 | if (!skb_dst(skb)->xfrm) |
9ab1265d | 596 | return dst_output(net, sk, skb); |
862b82c6 | 597 | |
adf30907 | 598 | err = nf_hook(skb_dst(skb)->ops->family, |
9ab1265d | 599 | NF_INET_POST_ROUTING, net, sk, skb, |
adf30907 | 600 | NULL, skb_dst(skb)->dev, xfrm_output2); |
862b82c6 | 601 | if (unlikely(err != 1)) |
c6581a45 | 602 | goto out; |
862b82c6 HX |
603 | } |
604 | ||
c6581a45 HX |
605 | if (err == -EINPROGRESS) |
606 | err = 0; | |
607 | ||
608 | out: | |
862b82c6 HX |
609 | return err; |
610 | } | |
c6581a45 | 611 | EXPORT_SYMBOL_GPL(xfrm_output_resume); |
862b82c6 | 612 | |
0c4b51f0 | 613 | static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb) |
862b82c6 | 614 | { |
9ab1265d | 615 | return xfrm_output_resume(sk, skb, 1); |
c6581a45 | 616 | } |
862b82c6 | 617 | |
0c4b51f0 | 618 | static int xfrm_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb) |
c6581a45 | 619 | { |
c3b18e0d | 620 | struct sk_buff *segs, *nskb; |
862b82c6 | 621 | |
a08e7fd9 CZ |
622 | BUILD_BUG_ON(sizeof(*IPCB(skb)) > SKB_GSO_CB_OFFSET); |
623 | BUILD_BUG_ON(sizeof(*IP6CB(skb)) > SKB_GSO_CB_OFFSET); | |
862b82c6 HX |
624 | segs = skb_gso_segment(skb, 0); |
625 | kfree_skb(skb); | |
801678c5 | 626 | if (IS_ERR(segs)) |
862b82c6 | 627 | return PTR_ERR(segs); |
330966e5 FW |
628 | if (segs == NULL) |
629 | return -EINVAL; | |
862b82c6 | 630 | |
c3b18e0d | 631 | skb_list_walk_safe(segs, segs, nskb) { |
862b82c6 HX |
632 | int err; |
633 | ||
a8305bff | 634 | skb_mark_not_on_list(segs); |
0c4b51f0 | 635 | err = xfrm_output2(net, sk, segs); |
862b82c6 HX |
636 | |
637 | if (unlikely(err)) { | |
46cfd725 | 638 | kfree_skb_list(nskb); |
862b82c6 HX |
639 | return err; |
640 | } | |
c3b18e0d | 641 | } |
862b82c6 HX |
642 | |
643 | return 0; | |
406ef77c | 644 | } |
c6581a45 | 645 | |
fa453523 HN |
646 | /* For partial checksum offload, the outer header checksum is calculated |
647 | * by software and the inner header checksum is calculated by hardware. | |
648 | * This requires hardware to know the inner packet type to calculate | |
649 | * the inner header checksum. Save inner ip protocol here to avoid | |
650 | * traversing the packet in the vendor's xmit code. | |
45a98ef4 RS |
651 | * For IPsec tunnel mode save the ip protocol from the IP header of the |
652 | * plain text packet. Otherwise If the encap type is IPIP, just save | |
653 | * skb->inner_ipproto in any other case get the ip protocol from the IP | |
654 | * header. | |
fa453523 | 655 | */ |
45a98ef4 | 656 | static void xfrm_get_inner_ipproto(struct sk_buff *skb, struct xfrm_state *x) |
fa453523 HN |
657 | { |
658 | struct xfrm_offload *xo = xfrm_offload(skb); | |
659 | const struct ethhdr *eth; | |
660 | ||
661 | if (!xo) | |
662 | return; | |
663 | ||
45a98ef4 RS |
664 | if (x->outer_mode.encap == XFRM_MODE_TUNNEL) { |
665 | switch (x->outer_mode.family) { | |
666 | case AF_INET: | |
667 | xo->inner_ipproto = ip_hdr(skb)->protocol; | |
668 | break; | |
669 | case AF_INET6: | |
670 | xo->inner_ipproto = ipv6_hdr(skb)->nexthdr; | |
671 | break; | |
672 | default: | |
673 | break; | |
674 | } | |
675 | ||
676 | return; | |
677 | } | |
678 | ||
679 | /* non-Tunnel Mode */ | |
680 | if (!skb->encapsulation) | |
681 | return; | |
682 | ||
fa453523 HN |
683 | if (skb->inner_protocol_type == ENCAP_TYPE_IPPROTO) { |
684 | xo->inner_ipproto = skb->inner_ipproto; | |
685 | return; | |
686 | } | |
687 | ||
688 | if (skb->inner_protocol_type != ENCAP_TYPE_ETHER) | |
689 | return; | |
690 | ||
691 | eth = (struct ethhdr *)skb_inner_mac_header(skb); | |
692 | ||
693 | switch (ntohs(eth->h_proto)) { | |
694 | case ETH_P_IPV6: | |
695 | xo->inner_ipproto = inner_ipv6_hdr(skb)->nexthdr; | |
696 | break; | |
697 | case ETH_P_IP: | |
698 | xo->inner_ipproto = inner_ip_hdr(skb)->protocol; | |
699 | break; | |
700 | } | |
701 | } | |
702 | ||
7026b1dd | 703 | int xfrm_output(struct sock *sk, struct sk_buff *skb) |
c6581a45 | 704 | { |
adf30907 | 705 | struct net *net = dev_net(skb_dst(skb)->dev); |
d77e38e6 | 706 | struct xfrm_state *x = skb_dst(skb)->xfrm; |
c6581a45 HX |
707 | int err; |
708 | ||
2ab6096d FW |
709 | switch (x->outer_mode.family) { |
710 | case AF_INET: | |
711 | memset(IPCB(skb), 0, sizeof(*IPCB(skb))); | |
2ab6096d | 712 | IPCB(skb)->flags |= IPSKB_XFRM_TRANSFORMED; |
2ab6096d FW |
713 | break; |
714 | case AF_INET6: | |
715 | memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); | |
716 | ||
2ab6096d | 717 | IP6CB(skb)->flags |= IP6SKB_XFRM_TRANSFORMED; |
2ab6096d FW |
718 | break; |
719 | } | |
720 | ||
f8a70afa LR |
721 | if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) { |
722 | if (!xfrm_dev_offload_ok(skb, x)) { | |
723 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); | |
724 | kfree_skb(skb); | |
725 | return -EHOSTUNREACH; | |
726 | } | |
727 | ||
728 | return xfrm_output_resume(sk, skb, 0); | |
729 | } | |
730 | ||
d77e38e6 SK |
731 | secpath_reset(skb); |
732 | ||
733 | if (xfrm_dev_offload_ok(skb, x)) { | |
734 | struct sec_path *sp; | |
735 | ||
a84e3f53 | 736 | sp = secpath_set(skb); |
d77e38e6 SK |
737 | if (!sp) { |
738 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); | |
739 | kfree_skb(skb); | |
740 | return -ENOMEM; | |
741 | } | |
d77e38e6 SK |
742 | |
743 | sp->olen++; | |
a84e3f53 | 744 | sp->xvec[sp->len++] = x; |
d77e38e6 SK |
745 | xfrm_state_hold(x); |
746 | ||
45a98ef4 | 747 | xfrm_get_inner_ipproto(skb, x); |
fa453523 HN |
748 | skb->encapsulation = 1; |
749 | ||
d77e38e6 | 750 | if (skb_is_gso(skb)) { |
a204aef9 XL |
751 | if (skb->inner_protocol) |
752 | return xfrm_output_gso(net, sk, skb); | |
d77e38e6 | 753 | |
a204aef9 XL |
754 | skb_shinfo(skb)->gso_type |= SKB_GSO_ESP; |
755 | goto out; | |
d77e38e6 SK |
756 | } |
757 | ||
758 | if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM) | |
759 | goto out; | |
a204aef9 XL |
760 | } else { |
761 | if (skb_is_gso(skb)) | |
762 | return xfrm_output_gso(net, sk, skb); | |
d77e38e6 SK |
763 | } |
764 | ||
c6581a45 HX |
765 | if (skb->ip_summed == CHECKSUM_PARTIAL) { |
766 | err = skb_checksum_help(skb); | |
767 | if (err) { | |
59c9940e | 768 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); |
c6581a45 HX |
769 | kfree_skb(skb); |
770 | return err; | |
771 | } | |
772 | } | |
773 | ||
d77e38e6 | 774 | out: |
0c4b51f0 | 775 | return xfrm_output2(net, sk, skb); |
c6581a45 | 776 | } |
fc68086c | 777 | EXPORT_SYMBOL_GPL(xfrm_output); |
df9dcb45 | 778 | |
6d64be3d FW |
779 | static int xfrm4_tunnel_check_size(struct sk_buff *skb) |
780 | { | |
781 | int mtu, ret = 0; | |
782 | ||
783 | if (IPCB(skb)->flags & IPSKB_XFRM_TUNNEL_SIZE) | |
784 | goto out; | |
785 | ||
786 | if (!(ip_hdr(skb)->frag_off & htons(IP_DF)) || skb->ignore_df) | |
787 | goto out; | |
788 | ||
789 | mtu = dst_mtu(skb_dst(skb)); | |
790 | if ((!skb_is_gso(skb) && skb->len > mtu) || | |
791 | (skb_is_gso(skb) && | |
792 | !skb_gso_validate_network_len(skb, ip_skb_dst_mtu(skb->sk, skb)))) { | |
793 | skb->protocol = htons(ETH_P_IP); | |
794 | ||
795 | if (skb->sk) | |
796 | xfrm_local_error(skb, mtu); | |
797 | else | |
798 | icmp_send(skb, ICMP_DEST_UNREACH, | |
799 | ICMP_FRAG_NEEDED, htonl(mtu)); | |
800 | ret = -EMSGSIZE; | |
801 | } | |
802 | out: | |
803 | return ret; | |
804 | } | |
805 | ||
806 | static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb) | |
807 | { | |
808 | int err; | |
809 | ||
68dc022d XL |
810 | if (x->outer_mode.encap == XFRM_MODE_BEET && |
811 | ip_is_fragment(ip_hdr(skb))) { | |
812 | net_warn_ratelimited("BEET mode doesn't support inner IPv4 fragments\n"); | |
813 | return -EAFNOSUPPORT; | |
814 | } | |
815 | ||
6d64be3d FW |
816 | err = xfrm4_tunnel_check_size(skb); |
817 | if (err) | |
818 | return err; | |
819 | ||
820 | XFRM_MODE_SKB_CB(skb)->protocol = ip_hdr(skb)->protocol; | |
821 | ||
822 | xfrm4_extract_header(skb); | |
823 | return 0; | |
824 | } | |
825 | ||
f3075f48 FW |
826 | #if IS_ENABLED(CONFIG_IPV6) |
827 | static int xfrm6_tunnel_check_size(struct sk_buff *skb) | |
828 | { | |
829 | int mtu, ret = 0; | |
830 | struct dst_entry *dst = skb_dst(skb); | |
831 | ||
832 | if (skb->ignore_df) | |
833 | goto out; | |
834 | ||
835 | mtu = dst_mtu(dst); | |
836 | if (mtu < IPV6_MIN_MTU) | |
837 | mtu = IPV6_MIN_MTU; | |
838 | ||
839 | if ((!skb_is_gso(skb) && skb->len > mtu) || | |
840 | (skb_is_gso(skb) && | |
841 | !skb_gso_validate_network_len(skb, ip6_skb_dst_mtu(skb)))) { | |
842 | skb->dev = dst->dev; | |
843 | skb->protocol = htons(ETH_P_IPV6); | |
844 | ||
845 | if (xfrm6_local_dontfrag(skb->sk)) | |
846 | ipv6_stub->xfrm6_local_rxpmtu(skb, mtu); | |
847 | else if (skb->sk) | |
848 | xfrm_local_error(skb, mtu); | |
849 | else | |
850 | icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); | |
851 | ret = -EMSGSIZE; | |
852 | } | |
853 | out: | |
854 | return ret; | |
855 | } | |
856 | #endif | |
857 | ||
858 | static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb) | |
859 | { | |
860 | #if IS_ENABLED(CONFIG_IPV6) | |
861 | int err; | |
862 | ||
863 | err = xfrm6_tunnel_check_size(skb); | |
864 | if (err) | |
865 | return err; | |
866 | ||
867 | XFRM_MODE_SKB_CB(skb)->protocol = ipv6_hdr(skb)->nexthdr; | |
868 | ||
869 | xfrm6_extract_header(skb); | |
870 | return 0; | |
871 | #else | |
872 | WARN_ON_ONCE(1); | |
873 | return -EAFNOSUPPORT; | |
874 | #endif | |
875 | } | |
876 | ||
0c620e97 | 877 | static int xfrm_inner_extract_output(struct xfrm_state *x, struct sk_buff *skb) |
df9dcb45 | 878 | { |
f4796398 HX |
879 | switch (skb->protocol) { |
880 | case htons(ETH_P_IP): | |
6d64be3d | 881 | return xfrm4_extract_output(x, skb); |
f4796398 | 882 | case htons(ETH_P_IPV6): |
f3075f48 | 883 | return xfrm6_extract_output(x, skb); |
6d64be3d | 884 | } |
733a5fac | 885 | |
f3075f48 | 886 | return -EAFNOSUPPORT; |
df9dcb45 KM |
887 | } |
888 | ||
628e341f HFS |
889 | void xfrm_local_error(struct sk_buff *skb, int mtu) |
890 | { | |
844d4874 | 891 | unsigned int proto; |
628e341f HFS |
892 | struct xfrm_state_afinfo *afinfo; |
893 | ||
844d4874 HFS |
894 | if (skb->protocol == htons(ETH_P_IP)) |
895 | proto = AF_INET; | |
f6a23d85 XL |
896 | else if (skb->protocol == htons(ETH_P_IPV6) && |
897 | skb->sk->sk_family == AF_INET6) | |
844d4874 HFS |
898 | proto = AF_INET6; |
899 | else | |
900 | return; | |
901 | ||
902 | afinfo = xfrm_state_get_afinfo(proto); | |
46c0ef6e | 903 | if (afinfo) { |
af5d27c4 | 904 | afinfo->local_error(skb, mtu); |
46c0ef6e TY |
905 | rcu_read_unlock(); |
906 | } | |
628e341f | 907 | } |
628e341f | 908 | EXPORT_SYMBOL_GPL(xfrm_local_error); |