Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * xfrm_input.c | |
4 | * | |
5 | * Changes: | |
6 | * YOSHIFUJI Hideaki @USAGI | |
7 | * Split up af-specific portion | |
a716c119 | 8 | * |
1da177e4 LT |
9 | */ |
10 | ||
acf568ee | 11 | #include <linux/bottom_half.h> |
f8c3d0dd | 12 | #include <linux/cache.h> |
acf568ee | 13 | #include <linux/interrupt.h> |
1da177e4 LT |
14 | #include <linux/slab.h> |
15 | #include <linux/module.h> | |
716062fd | 16 | #include <linux/netdevice.h> |
acf568ee | 17 | #include <linux/percpu.h> |
716062fd | 18 | #include <net/dst.h> |
1da177e4 LT |
19 | #include <net/ip.h> |
20 | #include <net/xfrm.h> | |
049f8e2e AD |
21 | #include <net/ip_tunnels.h> |
22 | #include <net/ip6_tunnel.h> | |
abc340b3 | 23 | #include <net/dst_metadata.h> |
edbc666c | 24 | #include <net/hotdata.h> |
1da177e4 | 25 | |
b3284df1 FW |
26 | #include "xfrm_inout.h" |
27 | ||
acf568ee | 28 | struct xfrm_trans_tasklet { |
4f492066 LJ |
29 | struct work_struct work; |
30 | spinlock_t queue_lock; | |
acf568ee HX |
31 | struct sk_buff_head queue; |
32 | }; | |
33 | ||
34 | struct xfrm_trans_cb { | |
9a3fb9fb SK |
35 | union { |
36 | struct inet_skb_parm h4; | |
37 | #if IS_ENABLED(CONFIG_IPV6) | |
38 | struct inet6_skb_parm h6; | |
39 | #endif | |
40 | } header; | |
acf568ee | 41 | int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb); |
7b380192 | 42 | struct net *net; |
acf568ee HX |
43 | }; |
44 | ||
45 | #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0])) | |
46 | ||
2f32b51b | 47 | static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); |
1475ee0a | 48 | static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1]; |
2f32b51b | 49 | |
1995876a SK |
50 | static struct gro_cells gro_cells; |
51 | static struct net_device xfrm_napi_dev; | |
52 | ||
acf568ee HX |
53 | static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet); |
54 | ||
960fdfde | 55 | int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
56 | { |
57 | int err = 0; | |
58 | ||
1475ee0a | 59 | if (WARN_ON(afinfo->family > AF_INET6)) |
2f32b51b | 60 | return -EAFNOSUPPORT; |
960fdfde | 61 | |
2f32b51b | 62 | spin_lock_bh(&xfrm_input_afinfo_lock); |
1475ee0a | 63 | if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) |
f31e8d4f | 64 | err = -EEXIST; |
2f32b51b | 65 | else |
1475ee0a | 66 | rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo); |
2f32b51b SK |
67 | spin_unlock_bh(&xfrm_input_afinfo_lock); |
68 | return err; | |
69 | } | |
70 | EXPORT_SYMBOL(xfrm_input_register_afinfo); | |
71 | ||
960fdfde | 72 | int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
73 | { |
74 | int err = 0; | |
75 | ||
2f32b51b | 76 | spin_lock_bh(&xfrm_input_afinfo_lock); |
1475ee0a XL |
77 | if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) { |
78 | if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo)) | |
2f32b51b SK |
79 | err = -EINVAL; |
80 | else | |
1475ee0a | 81 | RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL); |
2f32b51b SK |
82 | } |
83 | spin_unlock_bh(&xfrm_input_afinfo_lock); | |
84 | synchronize_rcu(); | |
85 | return err; | |
86 | } | |
87 | EXPORT_SYMBOL(xfrm_input_unregister_afinfo); | |
88 | ||
1475ee0a | 89 | static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip) |
2f32b51b | 90 | { |
960fdfde | 91 | const struct xfrm_input_afinfo *afinfo; |
2f32b51b | 92 | |
1475ee0a | 93 | if (WARN_ON_ONCE(family > AF_INET6)) |
2f32b51b | 94 | return NULL; |
960fdfde | 95 | |
2f32b51b | 96 | rcu_read_lock(); |
1475ee0a | 97 | afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]); |
2f32b51b SK |
98 | if (unlikely(!afinfo)) |
99 | rcu_read_unlock(); | |
100 | return afinfo; | |
101 | } | |
102 | ||
2f32b51b SK |
103 | static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, |
104 | int err) | |
105 | { | |
1475ee0a XL |
106 | bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6); |
107 | const struct xfrm_input_afinfo *afinfo; | |
2f32b51b | 108 | int ret; |
2f32b51b | 109 | |
1475ee0a | 110 | afinfo = xfrm_input_get_afinfo(family, is_ipip); |
2f32b51b SK |
111 | if (!afinfo) |
112 | return -EAFNOSUPPORT; | |
113 | ||
114 | ret = afinfo->callback(skb, protocol, err); | |
960fdfde | 115 | rcu_read_unlock(); |
2f32b51b SK |
116 | |
117 | return ret; | |
118 | } | |
119 | ||
4165079b | 120 | struct sec_path *secpath_set(struct sk_buff *skb) |
1da177e4 | 121 | { |
4165079b | 122 | struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH); |
1da177e4 | 123 | |
4165079b | 124 | sp = skb_ext_add(skb, SKB_EXT_SEC_PATH); |
1da177e4 LT |
125 | if (!sp) |
126 | return NULL; | |
127 | ||
4165079b FW |
128 | if (tmp) /* reused existing one (was COW'd if needed) */ |
129 | return sp; | |
54ef207a | 130 | |
4165079b | 131 | /* allocated new secpath */ |
f1193e91 | 132 | memset(sp->ovec, 0, sizeof(sp->ovec)); |
4165079b FW |
133 | sp->olen = 0; |
134 | sp->len = 0; | |
1f8b6df6 | 135 | sp->verified_cnt = 0; |
0ca64da1 FW |
136 | |
137 | return sp; | |
b0fcee82 SK |
138 | } |
139 | EXPORT_SYMBOL(secpath_set); | |
140 | ||
1da177e4 LT |
141 | /* Fetch spi and seq from ipsec header */ |
142 | ||
6067b2ba | 143 | int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) |
1da177e4 LT |
144 | { |
145 | int offset, offset_seq; | |
44072500 | 146 | int hlen; |
1da177e4 LT |
147 | |
148 | switch (nexthdr) { | |
149 | case IPPROTO_AH: | |
44072500 | 150 | hlen = sizeof(struct ip_auth_hdr); |
1da177e4 LT |
151 | offset = offsetof(struct ip_auth_hdr, spi); |
152 | offset_seq = offsetof(struct ip_auth_hdr, seq_no); | |
153 | break; | |
154 | case IPPROTO_ESP: | |
44072500 | 155 | hlen = sizeof(struct ip_esp_hdr); |
1da177e4 LT |
156 | offset = offsetof(struct ip_esp_hdr, spi); |
157 | offset_seq = offsetof(struct ip_esp_hdr, seq_no); | |
158 | break; | |
159 | case IPPROTO_COMP: | |
160 | if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr))) | |
161 | return -EINVAL; | |
3e94c2dc | 162 | *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); |
1da177e4 LT |
163 | *seq = 0; |
164 | return 0; | |
165 | default: | |
166 | return 1; | |
167 | } | |
168 | ||
44072500 | 169 | if (!pskb_may_pull(skb, hlen)) |
1da177e4 LT |
170 | return -EINVAL; |
171 | ||
3e94c2dc WC |
172 | *spi = *(__be32 *)(skb_transport_header(skb) + offset); |
173 | *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); | |
1da177e4 LT |
174 | return 0; |
175 | } | |
1e295370 | 176 | EXPORT_SYMBOL(xfrm_parse_spi); |
1da177e4 | 177 | |
b3284df1 FW |
178 | static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) |
179 | { | |
180 | struct iphdr *iph; | |
181 | int optlen = 0; | |
182 | int err = -EINVAL; | |
183 | ||
57010b8e HX |
184 | skb->protocol = htons(ETH_P_IP); |
185 | ||
b3284df1 FW |
186 | if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) { |
187 | struct ip_beet_phdr *ph; | |
188 | int phlen; | |
189 | ||
190 | if (!pskb_may_pull(skb, sizeof(*ph))) | |
191 | goto out; | |
192 | ||
193 | ph = (struct ip_beet_phdr *)skb->data; | |
194 | ||
195 | phlen = sizeof(*ph) + ph->padlen; | |
196 | optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen); | |
197 | if (optlen < 0 || optlen & 3 || optlen > 250) | |
198 | goto out; | |
199 | ||
200 | XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr; | |
201 | ||
202 | if (!pskb_may_pull(skb, phlen)) | |
203 | goto out; | |
204 | __skb_pull(skb, phlen); | |
205 | } | |
206 | ||
207 | skb_push(skb, sizeof(*iph)); | |
208 | skb_reset_network_header(skb); | |
209 | skb_mac_header_rebuild(skb); | |
210 | ||
211 | xfrm4_beet_make_header(skb); | |
212 | ||
213 | iph = ip_hdr(skb); | |
214 | ||
215 | iph->ihl += optlen / 4; | |
216 | iph->tot_len = htons(skb->len); | |
217 | iph->daddr = x->sel.daddr.a4; | |
218 | iph->saddr = x->sel.saddr.a4; | |
219 | iph->check = 0; | |
220 | iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl); | |
221 | err = 0; | |
222 | out: | |
223 | return err; | |
224 | } | |
225 | ||
226 | static void ipip_ecn_decapsulate(struct sk_buff *skb) | |
227 | { | |
228 | struct iphdr *inner_iph = ipip_hdr(skb); | |
229 | ||
230 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) | |
231 | IP_ECN_set_ce(inner_iph); | |
232 | } | |
233 | ||
234 | static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) | |
235 | { | |
236 | int err = -EINVAL; | |
237 | ||
57010b8e HX |
238 | skb->protocol = htons(ETH_P_IP); |
239 | ||
b3284df1 FW |
240 | if (!pskb_may_pull(skb, sizeof(struct iphdr))) |
241 | goto out; | |
242 | ||
243 | err = skb_unclone(skb, GFP_ATOMIC); | |
244 | if (err) | |
245 | goto out; | |
246 | ||
247 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) | |
248 | ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb)); | |
249 | if (!(x->props.flags & XFRM_STATE_NOECN)) | |
250 | ipip_ecn_decapsulate(skb); | |
251 | ||
252 | skb_reset_network_header(skb); | |
253 | skb_mac_header_rebuild(skb); | |
254 | if (skb->mac_len) | |
255 | eth_hdr(skb)->h_proto = skb->protocol; | |
256 | ||
257 | err = 0; | |
258 | ||
259 | out: | |
260 | return err; | |
261 | } | |
262 | ||
263 | static void ipip6_ecn_decapsulate(struct sk_buff *skb) | |
264 | { | |
265 | struct ipv6hdr *inner_iph = ipipv6_hdr(skb); | |
266 | ||
267 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) | |
268 | IP6_ECN_set_ce(skb, inner_iph); | |
269 | } | |
270 | ||
271 | static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) | |
272 | { | |
273 | int err = -EINVAL; | |
274 | ||
57010b8e HX |
275 | skb->protocol = htons(ETH_P_IPV6); |
276 | ||
b3284df1 FW |
277 | if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) |
278 | goto out; | |
279 | ||
280 | err = skb_unclone(skb, GFP_ATOMIC); | |
281 | if (err) | |
282 | goto out; | |
283 | ||
284 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) | |
6028da3f | 285 | ipv6_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipipv6_hdr(skb)); |
b3284df1 FW |
286 | if (!(x->props.flags & XFRM_STATE_NOECN)) |
287 | ipip6_ecn_decapsulate(skb); | |
288 | ||
289 | skb_reset_network_header(skb); | |
290 | skb_mac_header_rebuild(skb); | |
291 | if (skb->mac_len) | |
292 | eth_hdr(skb)->h_proto = skb->protocol; | |
293 | ||
294 | err = 0; | |
295 | ||
296 | out: | |
297 | return err; | |
298 | } | |
299 | ||
300 | static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) | |
301 | { | |
302 | struct ipv6hdr *ip6h; | |
303 | int size = sizeof(struct ipv6hdr); | |
304 | int err; | |
305 | ||
57010b8e HX |
306 | skb->protocol = htons(ETH_P_IPV6); |
307 | ||
b3284df1 FW |
308 | err = skb_cow_head(skb, size + skb->mac_len); |
309 | if (err) | |
310 | goto out; | |
311 | ||
312 | __skb_push(skb, size); | |
313 | skb_reset_network_header(skb); | |
314 | skb_mac_header_rebuild(skb); | |
315 | ||
316 | xfrm6_beet_make_header(skb); | |
317 | ||
318 | ip6h = ipv6_hdr(skb); | |
319 | ip6h->payload_len = htons(skb->len - size); | |
320 | ip6h->daddr = x->sel.daddr.in6; | |
321 | ip6h->saddr = x->sel.saddr.in6; | |
322 | err = 0; | |
323 | out: | |
324 | return err; | |
325 | } | |
326 | ||
327 | /* Remove encapsulation header. | |
328 | * | |
329 | * The IP header will be moved over the top of the encapsulation | |
330 | * header. | |
331 | * | |
332 | * On entry, the transport header shall point to where the IP header | |
333 | * should be and the network header shall be set to where the IP | |
334 | * header currently is. skb->data shall point to the start of the | |
335 | * payload. | |
336 | */ | |
337 | static int | |
338 | xfrm_inner_mode_encap_remove(struct xfrm_state *x, | |
b3284df1 FW |
339 | struct sk_buff *skb) |
340 | { | |
5f24f41e | 341 | switch (x->props.mode) { |
b3284df1 | 342 | case XFRM_MODE_BEET: |
842665a9 HX |
343 | switch (x->sel.family) { |
344 | case AF_INET: | |
b3284df1 | 345 | return xfrm4_remove_beet_encap(x, skb); |
842665a9 | 346 | case AF_INET6: |
b3284df1 | 347 | return xfrm6_remove_beet_encap(x, skb); |
5f24f41e | 348 | } |
b3284df1 FW |
349 | break; |
350 | case XFRM_MODE_TUNNEL: | |
5f24f41e HX |
351 | switch (XFRM_MODE_SKB_CB(skb)->protocol) { |
352 | case IPPROTO_IPIP: | |
b3284df1 | 353 | return xfrm4_remove_tunnel_encap(x, skb); |
5f24f41e | 354 | case IPPROTO_IPV6: |
b3284df1 FW |
355 | return xfrm6_remove_tunnel_encap(x, skb); |
356 | break; | |
5f24f41e | 357 | } |
57010b8e | 358 | return -EINVAL; |
b3284df1 FW |
359 | } |
360 | ||
361 | WARN_ON_ONCE(1); | |
362 | return -EOPNOTSUPP; | |
363 | } | |
364 | ||
c2d305e5 | 365 | static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) |
227620e2 | 366 | { |
5f24f41e | 367 | switch (x->props.family) { |
a269fbfc FW |
368 | case AF_INET: |
369 | xfrm4_extract_header(skb); | |
370 | break; | |
371 | case AF_INET6: | |
372 | xfrm6_extract_header(skb); | |
373 | break; | |
374 | default: | |
375 | WARN_ON_ONCE(1); | |
376 | return -EAFNOSUPPORT; | |
377 | } | |
227620e2 | 378 | |
5f24f41e | 379 | return xfrm_inner_mode_encap_remove(x, skb); |
227620e2 | 380 | } |
c2d305e5 FW |
381 | |
382 | /* Remove encapsulation header. | |
383 | * | |
384 | * The IP header will be moved over the top of the encapsulation header. | |
385 | * | |
386 | * On entry, skb_transport_header() shall point to where the IP header | |
387 | * should be and skb_network_header() shall be set to where the IP header | |
388 | * currently is. skb->data shall point to the start of the payload. | |
389 | */ | |
390 | static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb) | |
391 | { | |
58fbfeca | 392 | struct xfrm_offload *xo = xfrm_offload(skb); |
c2d305e5 FW |
393 | int ihl = skb->data - skb_transport_header(skb); |
394 | ||
395 | if (skb->transport_header != skb->network_header) { | |
396 | memmove(skb_transport_header(skb), | |
397 | skb_network_header(skb), ihl); | |
58fbfeca PD |
398 | if (xo) |
399 | xo->orig_mac_len = | |
400 | skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0; | |
c2d305e5 FW |
401 | skb->network_header = skb->transport_header; |
402 | } | |
403 | ip_hdr(skb)->tot_len = htons(skb->len + ihl); | |
404 | skb_reset_transport_header(skb); | |
405 | return 0; | |
c2d305e5 FW |
406 | } |
407 | ||
408 | static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb) | |
409 | { | |
4c145dce | 410 | #if IS_ENABLED(CONFIG_IPV6) |
58fbfeca | 411 | struct xfrm_offload *xo = xfrm_offload(skb); |
c2d305e5 FW |
412 | int ihl = skb->data - skb_transport_header(skb); |
413 | ||
414 | if (skb->transport_header != skb->network_header) { | |
415 | memmove(skb_transport_header(skb), | |
416 | skb_network_header(skb), ihl); | |
58fbfeca PD |
417 | if (xo) |
418 | xo->orig_mac_len = | |
419 | skb_mac_header_was_set(skb) ? skb_mac_header_len(skb) : 0; | |
c2d305e5 FW |
420 | skb->network_header = skb->transport_header; |
421 | } | |
422 | ipv6_hdr(skb)->payload_len = htons(skb->len + ihl - | |
423 | sizeof(struct ipv6hdr)); | |
424 | skb_reset_transport_header(skb); | |
425 | return 0; | |
426 | #else | |
4c145dce FW |
427 | WARN_ON_ONCE(1); |
428 | return -EAFNOSUPPORT; | |
c2d305e5 FW |
429 | #endif |
430 | } | |
431 | ||
432 | static int xfrm_inner_mode_input(struct xfrm_state *x, | |
c2d305e5 FW |
433 | struct sk_buff *skb) |
434 | { | |
5f24f41e | 435 | switch (x->props.mode) { |
c2d305e5 FW |
436 | case XFRM_MODE_BEET: |
437 | case XFRM_MODE_TUNNEL: | |
438 | return xfrm_prepare_input(x, skb); | |
439 | case XFRM_MODE_TRANSPORT: | |
5f24f41e | 440 | if (x->props.family == AF_INET) |
c2d305e5 | 441 | return xfrm4_transport_input(x, skb); |
5f24f41e | 442 | if (x->props.family == AF_INET6) |
c2d305e5 FW |
443 | return xfrm6_transport_input(x, skb); |
444 | break; | |
445 | case XFRM_MODE_ROUTEOPTIMIZATION: | |
446 | WARN_ON_ONCE(1); | |
447 | break; | |
448 | default: | |
449 | WARN_ON_ONCE(1); | |
450 | break; | |
451 | } | |
452 | ||
453 | return -EOPNOTSUPP; | |
454 | } | |
227620e2 | 455 | |
716062fd HX |
456 | int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) |
457 | { | |
733a5fac | 458 | const struct xfrm_state_afinfo *afinfo; |
bd235e3c | 459 | struct net *net = dev_net(skb->dev); |
716062fd HX |
460 | int err; |
461 | __be32 seq; | |
2cd08467 | 462 | __be32 seq_hi; |
3328715e | 463 | struct xfrm_state *x = NULL; |
1bf06cd2 | 464 | xfrm_address_t *daddr; |
049f8e2e | 465 | u32 mark = skb->mark; |
4ce3dbe3 | 466 | unsigned int family = AF_UNSPEC; |
716062fd | 467 | int decaps = 0; |
1bf06cd2 | 468 | int async = 0; |
7785bba2 | 469 | bool xfrm_gro = false; |
d77e38e6 SK |
470 | bool crypto_done = false; |
471 | struct xfrm_offload *xo = xfrm_offload(skb); | |
0ca64da1 | 472 | struct sec_path *sp; |
1bf06cd2 | 473 | |
b439475a | 474 | if (encap_type < 0 || (xo && xo->flags & XFRM_GRO)) { |
00501121 | 475 | x = xfrm_input_state(skb); |
4ce3dbe3 | 476 | |
304b44f0 AA |
477 | if (unlikely(x->dir && x->dir != XFRM_SA_DIR_IN)) { |
478 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEDIRERROR); | |
479 | goto drop; | |
480 | } | |
481 | ||
4ce3dbe3 AH |
482 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
483 | if (x->km.state == XFRM_STATE_ACQ) | |
484 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
485 | else | |
486 | XFRM_INC_STATS(net, | |
487 | LINUX_MIB_XFRMINSTATEINVALID); | |
4944a4b1 XX |
488 | |
489 | if (encap_type == -1) | |
490 | dev_put(skb->dev); | |
4ce3dbe3 AH |
491 | goto drop; |
492 | } | |
493 | ||
5f24f41e | 494 | family = x->props.family; |
7785bba2 SK |
495 | |
496 | /* An encap_type of -1 indicates async resumption. */ | |
497 | if (encap_type == -1) { | |
498 | async = 1; | |
499 | seq = XFRM_SKB_CB(skb)->seq.input.low; | |
500 | goto resume; | |
501 | } | |
b439475a | 502 | /* GRO call */ |
7785bba2 | 503 | seq = XFRM_SPI_SKB_CB(skb)->seq; |
716062fd | 504 | |
bcd1f8a4 SK |
505 | if (xo && (xo->flags & CRYPTO_DONE)) { |
506 | crypto_done = true; | |
bcd1f8a4 SK |
507 | family = XFRM_SPI_SKB_CB(skb)->family; |
508 | ||
509 | if (!(xo->status & CRYPTO_SUCCESS)) { | |
510 | if (xo->status & | |
511 | (CRYPTO_TRANSPORT_AH_AUTH_FAILED | | |
512 | CRYPTO_TRANSPORT_ESP_AUTH_FAILED | | |
513 | CRYPTO_TUNNEL_AH_AUTH_FAILED | | |
514 | CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { | |
515 | ||
516 | xfrm_audit_state_icvfail(x, skb, | |
517 | x->type->proto); | |
518 | x->stats.integrity_failed++; | |
519 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
520 | goto drop; | |
521 | } | |
522 | ||
47ebcc0b YK |
523 | if (xo->status & CRYPTO_INVALID_PROTOCOL) { |
524 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
525 | goto drop; | |
526 | } | |
527 | ||
bcd1f8a4 | 528 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
d77e38e6 SK |
529 | goto drop; |
530 | } | |
531 | ||
ad57dae8 | 532 | if (xfrm_parse_spi(skb, nexthdr, &spi, &seq)) { |
bcd1f8a4 SK |
533 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
534 | goto drop; | |
535 | } | |
d77e38e6 SK |
536 | } |
537 | ||
7785bba2 | 538 | goto lock; |
1bf06cd2 | 539 | } |
716062fd | 540 | |
3328715e SK |
541 | family = XFRM_SPI_SKB_CB(skb)->family; |
542 | ||
049f8e2e | 543 | /* if tunnel is present override skb->mark value with tunnel i_key */ |
1625f452 AK |
544 | switch (family) { |
545 | case AF_INET: | |
546 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) | |
049f8e2e | 547 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); |
1625f452 AK |
548 | break; |
549 | case AF_INET6: | |
550 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) | |
049f8e2e | 551 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); |
1625f452 | 552 | break; |
049f8e2e AD |
553 | } |
554 | ||
0ca64da1 FW |
555 | sp = secpath_set(skb); |
556 | if (!sp) { | |
b0fcee82 SK |
557 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); |
558 | goto drop; | |
b2aa5e9d HX |
559 | } |
560 | ||
716062fd | 561 | seq = 0; |
ad57dae8 | 562 | if (!spi && xfrm_parse_spi(skb, nexthdr, &spi, &seq)) { |
f203b76d | 563 | secpath_reset(skb); |
59c9940e | 564 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 565 | goto drop; |
0aa64774 | 566 | } |
716062fd | 567 | |
cb79a180 FW |
568 | daddr = (xfrm_address_t *)(skb_network_header(skb) + |
569 | XFRM_SPI_SKB_CB(skb)->daddroff); | |
716062fd | 570 | do { |
2294be0f FW |
571 | sp = skb_sec_path(skb); |
572 | ||
573 | if (sp->len == XFRM_MAX_DEPTH) { | |
f203b76d | 574 | secpath_reset(skb); |
59c9940e | 575 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
716062fd | 576 | goto drop; |
0aa64774 | 577 | } |
716062fd | 578 | |
049f8e2e | 579 | x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family); |
0aa64774 | 580 | if (x == NULL) { |
f203b76d | 581 | secpath_reset(skb); |
59c9940e | 582 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); |
afeb14b4 | 583 | xfrm_audit_state_notfound(skb, family, spi, seq); |
716062fd | 584 | goto drop; |
0aa64774 | 585 | } |
716062fd | 586 | |
304b44f0 AA |
587 | if (unlikely(x->dir && x->dir != XFRM_SA_DIR_IN)) { |
588 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEDIRERROR); | |
589 | xfrm_state_put(x); | |
590 | goto drop; | |
591 | } | |
592 | ||
9b42c1f1 SK |
593 | skb->mark = xfrm_smark_get(skb->mark, x); |
594 | ||
2294be0f | 595 | sp->xvec[sp->len++] = x; |
b2aa5e9d | 596 | |
0152eee6 SK |
597 | skb_dst_force(skb); |
598 | if (!skb_dst(skb)) { | |
599 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); | |
600 | goto drop; | |
601 | } | |
602 | ||
7785bba2 | 603 | lock: |
716062fd | 604 | spin_lock(&x->lock); |
4c4d41f2 | 605 | |
0aa64774 | 606 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
dc0565ce LR |
607 | if (x->km.state == XFRM_STATE_ACQ) |
608 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
609 | else | |
610 | XFRM_INC_STATS(net, | |
611 | LINUX_MIB_XFRMINSTATEINVALID); | |
716062fd | 612 | goto drop_unlock; |
0aa64774 | 613 | } |
716062fd | 614 | |
3de77cf2 DM |
615 | if ((x->encap ? x->encap->encap_type : 0) != encap_type) { |
616 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); | |
617 | goto drop_unlock; | |
618 | } | |
619 | ||
adfc2fdb | 620 | if (xfrm_replay_check(x, skb, seq)) { |
59c9940e | 621 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
716062fd | 622 | goto drop_unlock; |
0aa64774 | 623 | } |
716062fd | 624 | |
0aa64774 | 625 | if (xfrm_state_check_expire(x)) { |
59c9940e | 626 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); |
716062fd | 627 | goto drop_unlock; |
0aa64774 | 628 | } |
716062fd | 629 | |
0ebea8ef HX |
630 | spin_unlock(&x->lock); |
631 | ||
68c11e98 AD |
632 | if (xfrm_tunnel_check(skb, x, family)) { |
633 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); | |
634 | goto drop; | |
635 | } | |
636 | ||
2cd08467 SK |
637 | seq_hi = htonl(xfrm_replay_seqhi(x, seq)); |
638 | ||
1ce3644a | 639 | XFRM_SKB_CB(skb)->seq.input.low = seq; |
2cd08467 | 640 | XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; |
1bf06cd2 | 641 | |
071d36bf | 642 | dev_hold(skb->dev); |
3bc07321 | 643 | |
d77e38e6 SK |
644 | if (crypto_done) |
645 | nexthdr = x->type_offload->input_tail(x, skb); | |
646 | else | |
647 | nexthdr = x->type->input(x, skb); | |
0ebea8ef | 648 | |
1bf06cd2 HX |
649 | if (nexthdr == -EINPROGRESS) |
650 | return 0; | |
1bf06cd2 | 651 | resume: |
071d36bf | 652 | dev_put(skb->dev); |
653 | ||
0ebea8ef | 654 | spin_lock(&x->lock); |
afcaf61b | 655 | if (nexthdr < 0) { |
9dd3245a HX |
656 | if (nexthdr == -EBADMSG) { |
657 | xfrm_audit_state_icvfail(x, skb, | |
658 | x->type->proto); | |
668dc8af | 659 | x->stats.integrity_failed++; |
9dd3245a | 660 | } |
59c9940e | 661 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
716062fd | 662 | goto drop_unlock; |
668dc8af | 663 | } |
716062fd | 664 | |
716062fd HX |
665 | /* only the first xfrm gets the encap type */ |
666 | encap_type = 0; | |
667 | ||
25cfb8bc | 668 | if (xfrm_replay_recheck(x, skb, seq)) { |
bcf66bf5 SK |
669 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
670 | goto drop_unlock; | |
671 | } | |
672 | ||
c7f87783 | 673 | xfrm_replay_advance(x, seq); |
716062fd HX |
674 | |
675 | x->curlft.bytes += skb->len; | |
676 | x->curlft.packets++; | |
f7fe25a6 | 677 | x->lastused = ktime_get_real_seconds(); |
716062fd HX |
678 | |
679 | spin_unlock(&x->lock); | |
680 | ||
60d5fcfb HX |
681 | XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; |
682 | ||
5f24f41e | 683 | if (xfrm_inner_mode_input(x, skb)) { |
59c9940e | 684 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); |
716062fd | 685 | goto drop; |
0aa64774 | 686 | } |
716062fd | 687 | |
c9500d7b | 688 | if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) { |
716062fd HX |
689 | decaps = 1; |
690 | break; | |
691 | } | |
692 | ||
1bf06cd2 HX |
693 | /* |
694 | * We need the inner address. However, we only get here for | |
695 | * transport mode so the outer address is identical. | |
696 | */ | |
697 | daddr = &x->id.daddr; | |
5f24f41e | 698 | family = x->props.family; |
1bf06cd2 | 699 | |
716062fd | 700 | err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); |
0aa64774 | 701 | if (err < 0) { |
59c9940e | 702 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 703 | goto drop; |
0aa64774 | 704 | } |
782710e3 | 705 | crypto_done = false; |
716062fd HX |
706 | } while (!err); |
707 | ||
3328715e SK |
708 | err = xfrm_rcv_cb(skb, family, x->type->proto, 0); |
709 | if (err) | |
710 | goto drop; | |
711 | ||
895b5c9f | 712 | nf_reset_ct(skb); |
716062fd HX |
713 | |
714 | if (decaps) { | |
2294be0f FW |
715 | sp = skb_sec_path(skb); |
716 | if (sp) | |
717 | sp->olen = 0; | |
abc340b3 EB |
718 | if (skb_valid_dst(skb)) |
719 | skb_dst_drop(skb); | |
1995876a | 720 | gro_cells_receive(&gro_cells, skb); |
716062fd HX |
721 | return 0; |
722 | } else { | |
7785bba2 SK |
723 | xo = xfrm_offload(skb); |
724 | if (xo) | |
725 | xfrm_gro = xo->flags & XFRM_GRO; | |
726 | ||
733a5fac FW |
727 | err = -EAFNOSUPPORT; |
728 | rcu_read_lock(); | |
5f24f41e | 729 | afinfo = xfrm_state_afinfo_get_rcu(x->props.family); |
733a5fac FW |
730 | if (likely(afinfo)) |
731 | err = afinfo->transport_finish(skb, xfrm_gro || async); | |
732 | rcu_read_unlock(); | |
7785bba2 | 733 | if (xfrm_gro) { |
2294be0f FW |
734 | sp = skb_sec_path(skb); |
735 | if (sp) | |
736 | sp->olen = 0; | |
abc340b3 EB |
737 | if (skb_valid_dst(skb)) |
738 | skb_dst_drop(skb); | |
7785bba2 SK |
739 | gro_cells_receive(&gro_cells, skb); |
740 | return err; | |
741 | } | |
742 | ||
743 | return err; | |
716062fd HX |
744 | } |
745 | ||
746 | drop_unlock: | |
747 | spin_unlock(&x->lock); | |
716062fd | 748 | drop: |
3328715e | 749 | xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); |
716062fd HX |
750 | kfree_skb(skb); |
751 | return 0; | |
752 | } | |
753 | EXPORT_SYMBOL(xfrm_input); | |
754 | ||
1bf06cd2 HX |
755 | int xfrm_input_resume(struct sk_buff *skb, int nexthdr) |
756 | { | |
757 | return xfrm_input(skb, nexthdr, 0, -1); | |
758 | } | |
759 | EXPORT_SYMBOL(xfrm_input_resume); | |
760 | ||
4f492066 | 761 | static void xfrm_trans_reinject(struct work_struct *work) |
acf568ee | 762 | { |
4f492066 | 763 | struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work); |
acf568ee HX |
764 | struct sk_buff_head queue; |
765 | struct sk_buff *skb; | |
766 | ||
767 | __skb_queue_head_init(&queue); | |
4f492066 | 768 | spin_lock_bh(&trans->queue_lock); |
acf568ee | 769 | skb_queue_splice_init(&trans->queue, &queue); |
4f492066 | 770 | spin_unlock_bh(&trans->queue_lock); |
acf568ee | 771 | |
4f492066 | 772 | local_bh_disable(); |
acf568ee | 773 | while ((skb = __skb_dequeue(&queue))) |
7b380192 SD |
774 | XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net, |
775 | NULL, skb); | |
4f492066 | 776 | local_bh_enable(); |
acf568ee HX |
777 | } |
778 | ||
7b380192 SD |
779 | int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, |
780 | int (*finish)(struct net *, struct sock *, | |
781 | struct sk_buff *)) | |
acf568ee HX |
782 | { |
783 | struct xfrm_trans_tasklet *trans; | |
784 | ||
785 | trans = this_cpu_ptr(&xfrm_trans_tasklet); | |
786 | ||
edbc666c | 787 | if (skb_queue_len(&trans->queue) >= READ_ONCE(net_hotdata.max_backlog)) |
acf568ee HX |
788 | return -ENOBUFS; |
789 | ||
7b380192 SD |
790 | BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb)); |
791 | ||
acf568ee | 792 | XFRM_TRANS_SKB_CB(skb)->finish = finish; |
7b380192 | 793 | XFRM_TRANS_SKB_CB(skb)->net = net; |
4f492066 | 794 | spin_lock_bh(&trans->queue_lock); |
d16b46e4 | 795 | __skb_queue_tail(&trans->queue, skb); |
4f492066 LJ |
796 | spin_unlock_bh(&trans->queue_lock); |
797 | schedule_work(&trans->work); | |
acf568ee HX |
798 | return 0; |
799 | } | |
7b380192 SD |
800 | EXPORT_SYMBOL(xfrm_trans_queue_net); |
801 | ||
802 | int xfrm_trans_queue(struct sk_buff *skb, | |
803 | int (*finish)(struct net *, struct sock *, | |
804 | struct sk_buff *)) | |
805 | { | |
806 | return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish); | |
807 | } | |
acf568ee HX |
808 | EXPORT_SYMBOL(xfrm_trans_queue); |
809 | ||
1da177e4 LT |
810 | void __init xfrm_input_init(void) |
811 | { | |
1995876a | 812 | int err; |
acf568ee | 813 | int i; |
1995876a SK |
814 | |
815 | init_dummy_netdev(&xfrm_napi_dev); | |
816 | err = gro_cells_init(&gro_cells, &xfrm_napi_dev); | |
817 | if (err) | |
818 | gro_cells.cells = NULL; | |
819 | ||
acf568ee HX |
820 | for_each_possible_cpu(i) { |
821 | struct xfrm_trans_tasklet *trans; | |
822 | ||
823 | trans = &per_cpu(xfrm_trans_tasklet, i); | |
4f492066 | 824 | spin_lock_init(&trans->queue_lock); |
acf568ee | 825 | __skb_queue_head_init(&trans->queue); |
4f492066 | 826 | INIT_WORK(&trans->work, xfrm_trans_reinject); |
acf568ee | 827 | } |
1da177e4 | 828 | } |