Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * xfrm_input.c | |
4 | * | |
5 | * Changes: | |
6 | * YOSHIFUJI Hideaki @USAGI | |
7 | * Split up af-specific portion | |
a716c119 | 8 | * |
1da177e4 LT |
9 | */ |
10 | ||
acf568ee | 11 | #include <linux/bottom_half.h> |
f8c3d0dd | 12 | #include <linux/cache.h> |
acf568ee | 13 | #include <linux/interrupt.h> |
1da177e4 LT |
14 | #include <linux/slab.h> |
15 | #include <linux/module.h> | |
716062fd | 16 | #include <linux/netdevice.h> |
acf568ee | 17 | #include <linux/percpu.h> |
716062fd | 18 | #include <net/dst.h> |
1da177e4 LT |
19 | #include <net/ip.h> |
20 | #include <net/xfrm.h> | |
049f8e2e AD |
21 | #include <net/ip_tunnels.h> |
22 | #include <net/ip6_tunnel.h> | |
1da177e4 | 23 | |
b3284df1 FW |
24 | #include "xfrm_inout.h" |
25 | ||
acf568ee HX |
26 | struct xfrm_trans_tasklet { |
27 | struct tasklet_struct tasklet; | |
28 | struct sk_buff_head queue; | |
29 | }; | |
30 | ||
31 | struct xfrm_trans_cb { | |
9a3fb9fb SK |
32 | union { |
33 | struct inet_skb_parm h4; | |
34 | #if IS_ENABLED(CONFIG_IPV6) | |
35 | struct inet6_skb_parm h6; | |
36 | #endif | |
37 | } header; | |
acf568ee HX |
38 | int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb); |
39 | }; | |
40 | ||
41 | #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0])) | |
42 | ||
2f32b51b | 43 | static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); |
960fdfde | 44 | static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1]; |
2f32b51b | 45 | |
1995876a SK |
46 | static struct gro_cells gro_cells; |
47 | static struct net_device xfrm_napi_dev; | |
48 | ||
acf568ee HX |
49 | static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet); |
50 | ||
960fdfde | 51 | int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
52 | { |
53 | int err = 0; | |
54 | ||
960fdfde | 55 | if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo))) |
2f32b51b | 56 | return -EAFNOSUPPORT; |
960fdfde | 57 | |
2f32b51b SK |
58 | spin_lock_bh(&xfrm_input_afinfo_lock); |
59 | if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL)) | |
f31e8d4f | 60 | err = -EEXIST; |
2f32b51b SK |
61 | else |
62 | rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo); | |
63 | spin_unlock_bh(&xfrm_input_afinfo_lock); | |
64 | return err; | |
65 | } | |
66 | EXPORT_SYMBOL(xfrm_input_register_afinfo); | |
67 | ||
960fdfde | 68 | int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
69 | { |
70 | int err = 0; | |
71 | ||
2f32b51b SK |
72 | spin_lock_bh(&xfrm_input_afinfo_lock); |
73 | if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) { | |
74 | if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo)) | |
75 | err = -EINVAL; | |
76 | else | |
77 | RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL); | |
78 | } | |
79 | spin_unlock_bh(&xfrm_input_afinfo_lock); | |
80 | synchronize_rcu(); | |
81 | return err; | |
82 | } | |
83 | EXPORT_SYMBOL(xfrm_input_unregister_afinfo); | |
84 | ||
960fdfde | 85 | static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family) |
2f32b51b | 86 | { |
960fdfde | 87 | const struct xfrm_input_afinfo *afinfo; |
2f32b51b | 88 | |
960fdfde | 89 | if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo))) |
2f32b51b | 90 | return NULL; |
960fdfde | 91 | |
2f32b51b SK |
92 | rcu_read_lock(); |
93 | afinfo = rcu_dereference(xfrm_input_afinfo[family]); | |
94 | if (unlikely(!afinfo)) | |
95 | rcu_read_unlock(); | |
96 | return afinfo; | |
97 | } | |
98 | ||
2f32b51b SK |
99 | static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, |
100 | int err) | |
101 | { | |
102 | int ret; | |
960fdfde | 103 | const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family); |
2f32b51b SK |
104 | |
105 | if (!afinfo) | |
106 | return -EAFNOSUPPORT; | |
107 | ||
108 | ret = afinfo->callback(skb, protocol, err); | |
960fdfde | 109 | rcu_read_unlock(); |
2f32b51b SK |
110 | |
111 | return ret; | |
112 | } | |
113 | ||
4165079b | 114 | struct sec_path *secpath_set(struct sk_buff *skb) |
1da177e4 | 115 | { |
4165079b | 116 | struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH); |
1da177e4 | 117 | |
4165079b | 118 | sp = skb_ext_add(skb, SKB_EXT_SEC_PATH); |
1da177e4 LT |
119 | if (!sp) |
120 | return NULL; | |
121 | ||
4165079b FW |
122 | if (tmp) /* reused existing one (was COW'd if needed) */ |
123 | return sp; | |
54ef207a | 124 | |
4165079b | 125 | /* allocated new secpath */ |
f1193e91 | 126 | memset(sp->ovec, 0, sizeof(sp->ovec)); |
4165079b FW |
127 | sp->olen = 0; |
128 | sp->len = 0; | |
0ca64da1 FW |
129 | |
130 | return sp; | |
b0fcee82 SK |
131 | } |
132 | EXPORT_SYMBOL(secpath_set); | |
133 | ||
1da177e4 LT |
134 | /* Fetch spi and seq from ipsec header */ |
135 | ||
6067b2ba | 136 | int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) |
1da177e4 LT |
137 | { |
138 | int offset, offset_seq; | |
44072500 | 139 | int hlen; |
1da177e4 LT |
140 | |
141 | switch (nexthdr) { | |
142 | case IPPROTO_AH: | |
44072500 | 143 | hlen = sizeof(struct ip_auth_hdr); |
1da177e4 LT |
144 | offset = offsetof(struct ip_auth_hdr, spi); |
145 | offset_seq = offsetof(struct ip_auth_hdr, seq_no); | |
146 | break; | |
147 | case IPPROTO_ESP: | |
44072500 | 148 | hlen = sizeof(struct ip_esp_hdr); |
1da177e4 LT |
149 | offset = offsetof(struct ip_esp_hdr, spi); |
150 | offset_seq = offsetof(struct ip_esp_hdr, seq_no); | |
151 | break; | |
152 | case IPPROTO_COMP: | |
153 | if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr))) | |
154 | return -EINVAL; | |
3e94c2dc | 155 | *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); |
1da177e4 LT |
156 | *seq = 0; |
157 | return 0; | |
158 | default: | |
159 | return 1; | |
160 | } | |
161 | ||
44072500 | 162 | if (!pskb_may_pull(skb, hlen)) |
1da177e4 LT |
163 | return -EINVAL; |
164 | ||
3e94c2dc WC |
165 | *spi = *(__be32 *)(skb_transport_header(skb) + offset); |
166 | *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); | |
1da177e4 LT |
167 | return 0; |
168 | } | |
1e295370 | 169 | EXPORT_SYMBOL(xfrm_parse_spi); |
1da177e4 | 170 | |
b3284df1 FW |
171 | static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) |
172 | { | |
173 | struct iphdr *iph; | |
174 | int optlen = 0; | |
175 | int err = -EINVAL; | |
176 | ||
177 | if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) { | |
178 | struct ip_beet_phdr *ph; | |
179 | int phlen; | |
180 | ||
181 | if (!pskb_may_pull(skb, sizeof(*ph))) | |
182 | goto out; | |
183 | ||
184 | ph = (struct ip_beet_phdr *)skb->data; | |
185 | ||
186 | phlen = sizeof(*ph) + ph->padlen; | |
187 | optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen); | |
188 | if (optlen < 0 || optlen & 3 || optlen > 250) | |
189 | goto out; | |
190 | ||
191 | XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr; | |
192 | ||
193 | if (!pskb_may_pull(skb, phlen)) | |
194 | goto out; | |
195 | __skb_pull(skb, phlen); | |
196 | } | |
197 | ||
198 | skb_push(skb, sizeof(*iph)); | |
199 | skb_reset_network_header(skb); | |
200 | skb_mac_header_rebuild(skb); | |
201 | ||
202 | xfrm4_beet_make_header(skb); | |
203 | ||
204 | iph = ip_hdr(skb); | |
205 | ||
206 | iph->ihl += optlen / 4; | |
207 | iph->tot_len = htons(skb->len); | |
208 | iph->daddr = x->sel.daddr.a4; | |
209 | iph->saddr = x->sel.saddr.a4; | |
210 | iph->check = 0; | |
211 | iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl); | |
212 | err = 0; | |
213 | out: | |
214 | return err; | |
215 | } | |
216 | ||
217 | static void ipip_ecn_decapsulate(struct sk_buff *skb) | |
218 | { | |
219 | struct iphdr *inner_iph = ipip_hdr(skb); | |
220 | ||
221 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) | |
222 | IP_ECN_set_ce(inner_iph); | |
223 | } | |
224 | ||
225 | static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) | |
226 | { | |
227 | int err = -EINVAL; | |
228 | ||
229 | if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP) | |
230 | goto out; | |
231 | ||
232 | if (!pskb_may_pull(skb, sizeof(struct iphdr))) | |
233 | goto out; | |
234 | ||
235 | err = skb_unclone(skb, GFP_ATOMIC); | |
236 | if (err) | |
237 | goto out; | |
238 | ||
239 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) | |
240 | ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb)); | |
241 | if (!(x->props.flags & XFRM_STATE_NOECN)) | |
242 | ipip_ecn_decapsulate(skb); | |
243 | ||
244 | skb_reset_network_header(skb); | |
245 | skb_mac_header_rebuild(skb); | |
246 | if (skb->mac_len) | |
247 | eth_hdr(skb)->h_proto = skb->protocol; | |
248 | ||
249 | err = 0; | |
250 | ||
251 | out: | |
252 | return err; | |
253 | } | |
254 | ||
255 | static void ipip6_ecn_decapsulate(struct sk_buff *skb) | |
256 | { | |
257 | struct ipv6hdr *inner_iph = ipipv6_hdr(skb); | |
258 | ||
259 | if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos)) | |
260 | IP6_ECN_set_ce(skb, inner_iph); | |
261 | } | |
262 | ||
263 | static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb) | |
264 | { | |
265 | int err = -EINVAL; | |
266 | ||
267 | if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6) | |
268 | goto out; | |
269 | if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) | |
270 | goto out; | |
271 | ||
272 | err = skb_unclone(skb, GFP_ATOMIC); | |
273 | if (err) | |
274 | goto out; | |
275 | ||
276 | if (x->props.flags & XFRM_STATE_DECAP_DSCP) | |
277 | ipv6_copy_dscp(ipv6_get_dsfield(ipv6_hdr(skb)), | |
278 | ipipv6_hdr(skb)); | |
279 | if (!(x->props.flags & XFRM_STATE_NOECN)) | |
280 | ipip6_ecn_decapsulate(skb); | |
281 | ||
282 | skb_reset_network_header(skb); | |
283 | skb_mac_header_rebuild(skb); | |
284 | if (skb->mac_len) | |
285 | eth_hdr(skb)->h_proto = skb->protocol; | |
286 | ||
287 | err = 0; | |
288 | ||
289 | out: | |
290 | return err; | |
291 | } | |
292 | ||
293 | static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb) | |
294 | { | |
295 | struct ipv6hdr *ip6h; | |
296 | int size = sizeof(struct ipv6hdr); | |
297 | int err; | |
298 | ||
299 | err = skb_cow_head(skb, size + skb->mac_len); | |
300 | if (err) | |
301 | goto out; | |
302 | ||
303 | __skb_push(skb, size); | |
304 | skb_reset_network_header(skb); | |
305 | skb_mac_header_rebuild(skb); | |
306 | ||
307 | xfrm6_beet_make_header(skb); | |
308 | ||
309 | ip6h = ipv6_hdr(skb); | |
310 | ip6h->payload_len = htons(skb->len - size); | |
311 | ip6h->daddr = x->sel.daddr.in6; | |
312 | ip6h->saddr = x->sel.saddr.in6; | |
313 | err = 0; | |
314 | out: | |
315 | return err; | |
316 | } | |
317 | ||
318 | /* Remove encapsulation header. | |
319 | * | |
320 | * The IP header will be moved over the top of the encapsulation | |
321 | * header. | |
322 | * | |
323 | * On entry, the transport header shall point to where the IP header | |
324 | * should be and the network header shall be set to where the IP | |
325 | * header currently is. skb->data shall point to the start of the | |
326 | * payload. | |
327 | */ | |
328 | static int | |
329 | xfrm_inner_mode_encap_remove(struct xfrm_state *x, | |
330 | const struct xfrm_mode *inner_mode, | |
331 | struct sk_buff *skb) | |
332 | { | |
333 | switch (inner_mode->encap) { | |
334 | case XFRM_MODE_BEET: | |
335 | if (inner_mode->family == AF_INET) | |
336 | return xfrm4_remove_beet_encap(x, skb); | |
337 | if (inner_mode->family == AF_INET6) | |
338 | return xfrm6_remove_beet_encap(x, skb); | |
339 | break; | |
340 | case XFRM_MODE_TUNNEL: | |
341 | if (inner_mode->family == AF_INET) | |
342 | return xfrm4_remove_tunnel_encap(x, skb); | |
343 | if (inner_mode->family == AF_INET6) | |
344 | return xfrm6_remove_tunnel_encap(x, skb); | |
345 | break; | |
346 | } | |
347 | ||
348 | WARN_ON_ONCE(1); | |
349 | return -EOPNOTSUPP; | |
350 | } | |
351 | ||
c2d305e5 | 352 | static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) |
227620e2 | 353 | { |
c9500d7b | 354 | const struct xfrm_mode *inner_mode = &x->inner_mode; |
733a5fac FW |
355 | const struct xfrm_state_afinfo *afinfo; |
356 | int err = -EAFNOSUPPORT; | |
227620e2 | 357 | |
733a5fac | 358 | rcu_read_lock(); |
c9500d7b | 359 | afinfo = xfrm_state_afinfo_get_rcu(x->outer_mode.family); |
733a5fac FW |
360 | if (likely(afinfo)) |
361 | err = afinfo->extract_input(x, skb); | |
4c203b04 | 362 | rcu_read_unlock(); |
733a5fac | 363 | |
4c203b04 | 364 | if (err) |
227620e2 HX |
365 | return err; |
366 | ||
df9dcb45 KM |
367 | if (x->sel.family == AF_UNSPEC) { |
368 | inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); | |
4c203b04 | 369 | if (!inner_mode) |
df9dcb45 KM |
370 | return -EAFNOSUPPORT; |
371 | } | |
372 | ||
4c203b04 FW |
373 | switch (inner_mode->family) { |
374 | case AF_INET: | |
375 | skb->protocol = htons(ETH_P_IP); | |
376 | break; | |
377 | case AF_INET6: | |
378 | skb->protocol = htons(ETH_P_IPV6); | |
1be451d9 | 379 | break; |
4c203b04 FW |
380 | default: |
381 | WARN_ON_ONCE(1); | |
382 | break; | |
733a5fac FW |
383 | } |
384 | ||
b3284df1 | 385 | return xfrm_inner_mode_encap_remove(x, inner_mode, skb); |
227620e2 | 386 | } |
c2d305e5 FW |
387 | |
388 | /* Remove encapsulation header. | |
389 | * | |
390 | * The IP header will be moved over the top of the encapsulation header. | |
391 | * | |
392 | * On entry, skb_transport_header() shall point to where the IP header | |
393 | * should be and skb_network_header() shall be set to where the IP header | |
394 | * currently is. skb->data shall point to the start of the payload. | |
395 | */ | |
396 | static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb) | |
397 | { | |
c2d305e5 FW |
398 | int ihl = skb->data - skb_transport_header(skb); |
399 | ||
400 | if (skb->transport_header != skb->network_header) { | |
401 | memmove(skb_transport_header(skb), | |
402 | skb_network_header(skb), ihl); | |
403 | skb->network_header = skb->transport_header; | |
404 | } | |
405 | ip_hdr(skb)->tot_len = htons(skb->len + ihl); | |
406 | skb_reset_transport_header(skb); | |
407 | return 0; | |
c2d305e5 FW |
408 | } |
409 | ||
410 | static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb) | |
411 | { | |
4c145dce | 412 | #if IS_ENABLED(CONFIG_IPV6) |
c2d305e5 FW |
413 | int ihl = skb->data - skb_transport_header(skb); |
414 | ||
415 | if (skb->transport_header != skb->network_header) { | |
416 | memmove(skb_transport_header(skb), | |
417 | skb_network_header(skb), ihl); | |
418 | skb->network_header = skb->transport_header; | |
419 | } | |
420 | ipv6_hdr(skb)->payload_len = htons(skb->len + ihl - | |
421 | sizeof(struct ipv6hdr)); | |
422 | skb_reset_transport_header(skb); | |
423 | return 0; | |
424 | #else | |
4c145dce FW |
425 | WARN_ON_ONCE(1); |
426 | return -EAFNOSUPPORT; | |
c2d305e5 FW |
427 | #endif |
428 | } | |
429 | ||
430 | static int xfrm_inner_mode_input(struct xfrm_state *x, | |
431 | const struct xfrm_mode *inner_mode, | |
432 | struct sk_buff *skb) | |
433 | { | |
434 | switch (inner_mode->encap) { | |
435 | case XFRM_MODE_BEET: | |
436 | case XFRM_MODE_TUNNEL: | |
437 | return xfrm_prepare_input(x, skb); | |
438 | case XFRM_MODE_TRANSPORT: | |
439 | if (inner_mode->family == AF_INET) | |
440 | return xfrm4_transport_input(x, skb); | |
441 | if (inner_mode->family == AF_INET6) | |
442 | return xfrm6_transport_input(x, skb); | |
443 | break; | |
444 | case XFRM_MODE_ROUTEOPTIMIZATION: | |
445 | WARN_ON_ONCE(1); | |
446 | break; | |
447 | default: | |
448 | WARN_ON_ONCE(1); | |
449 | break; | |
450 | } | |
451 | ||
452 | return -EOPNOTSUPP; | |
453 | } | |
227620e2 | 454 | |
716062fd HX |
455 | int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) |
456 | { | |
733a5fac | 457 | const struct xfrm_state_afinfo *afinfo; |
bd235e3c | 458 | struct net *net = dev_net(skb->dev); |
4c145dce | 459 | const struct xfrm_mode *inner_mode; |
716062fd HX |
460 | int err; |
461 | __be32 seq; | |
2cd08467 | 462 | __be32 seq_hi; |
3328715e | 463 | struct xfrm_state *x = NULL; |
1bf06cd2 | 464 | xfrm_address_t *daddr; |
049f8e2e | 465 | u32 mark = skb->mark; |
4ce3dbe3 | 466 | unsigned int family = AF_UNSPEC; |
716062fd | 467 | int decaps = 0; |
1bf06cd2 | 468 | int async = 0; |
7785bba2 | 469 | bool xfrm_gro = false; |
d77e38e6 SK |
470 | bool crypto_done = false; |
471 | struct xfrm_offload *xo = xfrm_offload(skb); | |
0ca64da1 | 472 | struct sec_path *sp; |
1bf06cd2 | 473 | |
1bf06cd2 | 474 | if (encap_type < 0) { |
00501121 | 475 | x = xfrm_input_state(skb); |
4ce3dbe3 AH |
476 | |
477 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { | |
478 | if (x->km.state == XFRM_STATE_ACQ) | |
479 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
480 | else | |
481 | XFRM_INC_STATS(net, | |
482 | LINUX_MIB_XFRMINSTATEINVALID); | |
483 | goto drop; | |
484 | } | |
485 | ||
c9500d7b | 486 | family = x->outer_mode.family; |
7785bba2 SK |
487 | |
488 | /* An encap_type of -1 indicates async resumption. */ | |
489 | if (encap_type == -1) { | |
490 | async = 1; | |
491 | seq = XFRM_SKB_CB(skb)->seq.input.low; | |
492 | goto resume; | |
493 | } | |
bcd1f8a4 | 494 | |
7785bba2 SK |
495 | /* encap_type < -1 indicates a GRO call. */ |
496 | encap_type = 0; | |
497 | seq = XFRM_SPI_SKB_CB(skb)->seq; | |
716062fd | 498 | |
bcd1f8a4 SK |
499 | if (xo && (xo->flags & CRYPTO_DONE)) { |
500 | crypto_done = true; | |
bcd1f8a4 SK |
501 | family = XFRM_SPI_SKB_CB(skb)->family; |
502 | ||
503 | if (!(xo->status & CRYPTO_SUCCESS)) { | |
504 | if (xo->status & | |
505 | (CRYPTO_TRANSPORT_AH_AUTH_FAILED | | |
506 | CRYPTO_TRANSPORT_ESP_AUTH_FAILED | | |
507 | CRYPTO_TUNNEL_AH_AUTH_FAILED | | |
508 | CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { | |
509 | ||
510 | xfrm_audit_state_icvfail(x, skb, | |
511 | x->type->proto); | |
512 | x->stats.integrity_failed++; | |
513 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
514 | goto drop; | |
515 | } | |
516 | ||
47ebcc0b YK |
517 | if (xo->status & CRYPTO_INVALID_PROTOCOL) { |
518 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
519 | goto drop; | |
520 | } | |
521 | ||
bcd1f8a4 | 522 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
d77e38e6 SK |
523 | goto drop; |
524 | } | |
525 | ||
bcd1f8a4 SK |
526 | if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { |
527 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); | |
528 | goto drop; | |
529 | } | |
d77e38e6 SK |
530 | } |
531 | ||
7785bba2 | 532 | goto lock; |
1bf06cd2 | 533 | } |
716062fd | 534 | |
3328715e SK |
535 | family = XFRM_SPI_SKB_CB(skb)->family; |
536 | ||
049f8e2e | 537 | /* if tunnel is present override skb->mark value with tunnel i_key */ |
1625f452 AK |
538 | switch (family) { |
539 | case AF_INET: | |
540 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) | |
049f8e2e | 541 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); |
1625f452 AK |
542 | break; |
543 | case AF_INET6: | |
544 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) | |
049f8e2e | 545 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); |
1625f452 | 546 | break; |
049f8e2e AD |
547 | } |
548 | ||
0ca64da1 FW |
549 | sp = secpath_set(skb); |
550 | if (!sp) { | |
b0fcee82 SK |
551 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); |
552 | goto drop; | |
b2aa5e9d HX |
553 | } |
554 | ||
716062fd | 555 | seq = 0; |
0aa64774 | 556 | if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { |
f203b76d | 557 | secpath_reset(skb); |
59c9940e | 558 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 559 | goto drop; |
0aa64774 | 560 | } |
716062fd | 561 | |
cb79a180 FW |
562 | daddr = (xfrm_address_t *)(skb_network_header(skb) + |
563 | XFRM_SPI_SKB_CB(skb)->daddroff); | |
716062fd | 564 | do { |
2294be0f FW |
565 | sp = skb_sec_path(skb); |
566 | ||
567 | if (sp->len == XFRM_MAX_DEPTH) { | |
f203b76d | 568 | secpath_reset(skb); |
59c9940e | 569 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
716062fd | 570 | goto drop; |
0aa64774 | 571 | } |
716062fd | 572 | |
049f8e2e | 573 | x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family); |
0aa64774 | 574 | if (x == NULL) { |
f203b76d | 575 | secpath_reset(skb); |
59c9940e | 576 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); |
afeb14b4 | 577 | xfrm_audit_state_notfound(skb, family, spi, seq); |
716062fd | 578 | goto drop; |
0aa64774 | 579 | } |
716062fd | 580 | |
9b42c1f1 SK |
581 | skb->mark = xfrm_smark_get(skb->mark, x); |
582 | ||
2294be0f | 583 | sp->xvec[sp->len++] = x; |
b2aa5e9d | 584 | |
0152eee6 SK |
585 | skb_dst_force(skb); |
586 | if (!skb_dst(skb)) { | |
587 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); | |
588 | goto drop; | |
589 | } | |
590 | ||
7785bba2 | 591 | lock: |
716062fd | 592 | spin_lock(&x->lock); |
4c4d41f2 | 593 | |
0aa64774 | 594 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
dc0565ce LR |
595 | if (x->km.state == XFRM_STATE_ACQ) |
596 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
597 | else | |
598 | XFRM_INC_STATS(net, | |
599 | LINUX_MIB_XFRMINSTATEINVALID); | |
716062fd | 600 | goto drop_unlock; |
0aa64774 | 601 | } |
716062fd | 602 | |
3de77cf2 DM |
603 | if ((x->encap ? x->encap->encap_type : 0) != encap_type) { |
604 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); | |
605 | goto drop_unlock; | |
606 | } | |
607 | ||
36ae0148 | 608 | if (x->repl->check(x, skb, seq)) { |
59c9940e | 609 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
716062fd | 610 | goto drop_unlock; |
0aa64774 | 611 | } |
716062fd | 612 | |
0aa64774 | 613 | if (xfrm_state_check_expire(x)) { |
59c9940e | 614 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); |
716062fd | 615 | goto drop_unlock; |
0aa64774 | 616 | } |
716062fd | 617 | |
0ebea8ef HX |
618 | spin_unlock(&x->lock); |
619 | ||
68c11e98 AD |
620 | if (xfrm_tunnel_check(skb, x, family)) { |
621 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); | |
622 | goto drop; | |
623 | } | |
624 | ||
2cd08467 SK |
625 | seq_hi = htonl(xfrm_replay_seqhi(x, seq)); |
626 | ||
1ce3644a | 627 | XFRM_SKB_CB(skb)->seq.input.low = seq; |
2cd08467 | 628 | XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; |
1bf06cd2 | 629 | |
071d36bf | 630 | dev_hold(skb->dev); |
3bc07321 | 631 | |
d77e38e6 SK |
632 | if (crypto_done) |
633 | nexthdr = x->type_offload->input_tail(x, skb); | |
634 | else | |
635 | nexthdr = x->type->input(x, skb); | |
0ebea8ef | 636 | |
1bf06cd2 HX |
637 | if (nexthdr == -EINPROGRESS) |
638 | return 0; | |
1bf06cd2 | 639 | resume: |
071d36bf | 640 | dev_put(skb->dev); |
641 | ||
0ebea8ef | 642 | spin_lock(&x->lock); |
668dc8af | 643 | if (nexthdr <= 0) { |
9dd3245a HX |
644 | if (nexthdr == -EBADMSG) { |
645 | xfrm_audit_state_icvfail(x, skb, | |
646 | x->type->proto); | |
668dc8af | 647 | x->stats.integrity_failed++; |
9dd3245a | 648 | } |
59c9940e | 649 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
716062fd | 650 | goto drop_unlock; |
668dc8af | 651 | } |
716062fd | 652 | |
716062fd HX |
653 | /* only the first xfrm gets the encap type */ |
654 | encap_type = 0; | |
655 | ||
3b59df46 | 656 | if (async && x->repl->recheck(x, skb, seq)) { |
bcf66bf5 SK |
657 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
658 | goto drop_unlock; | |
659 | } | |
660 | ||
9fdc4883 | 661 | x->repl->advance(x, seq); |
716062fd HX |
662 | |
663 | x->curlft.bytes += skb->len; | |
664 | x->curlft.packets++; | |
665 | ||
666 | spin_unlock(&x->lock); | |
667 | ||
60d5fcfb HX |
668 | XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; |
669 | ||
c9500d7b | 670 | inner_mode = &x->inner_mode; |
df9dcb45 KM |
671 | |
672 | if (x->sel.family == AF_UNSPEC) { | |
673 | inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); | |
cb866e32 SK |
674 | if (inner_mode == NULL) { |
675 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); | |
df9dcb45 | 676 | goto drop; |
cb866e32 | 677 | } |
df9dcb45 KM |
678 | } |
679 | ||
c2d305e5 | 680 | if (xfrm_inner_mode_input(x, inner_mode, skb)) { |
59c9940e | 681 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); |
716062fd | 682 | goto drop; |
0aa64774 | 683 | } |
716062fd | 684 | |
c9500d7b | 685 | if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) { |
716062fd HX |
686 | decaps = 1; |
687 | break; | |
688 | } | |
689 | ||
1bf06cd2 HX |
690 | /* |
691 | * We need the inner address. However, we only get here for | |
692 | * transport mode so the outer address is identical. | |
693 | */ | |
694 | daddr = &x->id.daddr; | |
c9500d7b | 695 | family = x->outer_mode.family; |
1bf06cd2 | 696 | |
716062fd | 697 | err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); |
0aa64774 | 698 | if (err < 0) { |
59c9940e | 699 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 700 | goto drop; |
0aa64774 | 701 | } |
782710e3 | 702 | crypto_done = false; |
716062fd HX |
703 | } while (!err); |
704 | ||
3328715e SK |
705 | err = xfrm_rcv_cb(skb, family, x->type->proto, 0); |
706 | if (err) | |
707 | goto drop; | |
708 | ||
716062fd HX |
709 | nf_reset(skb); |
710 | ||
711 | if (decaps) { | |
2294be0f FW |
712 | sp = skb_sec_path(skb); |
713 | if (sp) | |
714 | sp->olen = 0; | |
adf30907 | 715 | skb_dst_drop(skb); |
1995876a | 716 | gro_cells_receive(&gro_cells, skb); |
716062fd HX |
717 | return 0; |
718 | } else { | |
7785bba2 SK |
719 | xo = xfrm_offload(skb); |
720 | if (xo) | |
721 | xfrm_gro = xo->flags & XFRM_GRO; | |
722 | ||
733a5fac FW |
723 | err = -EAFNOSUPPORT; |
724 | rcu_read_lock(); | |
c9500d7b | 725 | afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family); |
733a5fac FW |
726 | if (likely(afinfo)) |
727 | err = afinfo->transport_finish(skb, xfrm_gro || async); | |
728 | rcu_read_unlock(); | |
7785bba2 | 729 | if (xfrm_gro) { |
2294be0f FW |
730 | sp = skb_sec_path(skb); |
731 | if (sp) | |
732 | sp->olen = 0; | |
7785bba2 SK |
733 | skb_dst_drop(skb); |
734 | gro_cells_receive(&gro_cells, skb); | |
735 | return err; | |
736 | } | |
737 | ||
738 | return err; | |
716062fd HX |
739 | } |
740 | ||
741 | drop_unlock: | |
742 | spin_unlock(&x->lock); | |
716062fd | 743 | drop: |
3328715e | 744 | xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); |
716062fd HX |
745 | kfree_skb(skb); |
746 | return 0; | |
747 | } | |
748 | EXPORT_SYMBOL(xfrm_input); | |
749 | ||
1bf06cd2 HX |
750 | int xfrm_input_resume(struct sk_buff *skb, int nexthdr) |
751 | { | |
752 | return xfrm_input(skb, nexthdr, 0, -1); | |
753 | } | |
754 | EXPORT_SYMBOL(xfrm_input_resume); | |
755 | ||
acf568ee HX |
756 | static void xfrm_trans_reinject(unsigned long data) |
757 | { | |
758 | struct xfrm_trans_tasklet *trans = (void *)data; | |
759 | struct sk_buff_head queue; | |
760 | struct sk_buff *skb; | |
761 | ||
762 | __skb_queue_head_init(&queue); | |
763 | skb_queue_splice_init(&trans->queue, &queue); | |
764 | ||
765 | while ((skb = __skb_dequeue(&queue))) | |
766 | XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb); | |
767 | } | |
768 | ||
769 | int xfrm_trans_queue(struct sk_buff *skb, | |
770 | int (*finish)(struct net *, struct sock *, | |
771 | struct sk_buff *)) | |
772 | { | |
773 | struct xfrm_trans_tasklet *trans; | |
774 | ||
775 | trans = this_cpu_ptr(&xfrm_trans_tasklet); | |
776 | ||
777 | if (skb_queue_len(&trans->queue) >= netdev_max_backlog) | |
778 | return -ENOBUFS; | |
779 | ||
780 | XFRM_TRANS_SKB_CB(skb)->finish = finish; | |
d16b46e4 | 781 | __skb_queue_tail(&trans->queue, skb); |
acf568ee HX |
782 | tasklet_schedule(&trans->tasklet); |
783 | return 0; | |
784 | } | |
785 | EXPORT_SYMBOL(xfrm_trans_queue); | |
786 | ||
1da177e4 LT |
787 | void __init xfrm_input_init(void) |
788 | { | |
1995876a | 789 | int err; |
acf568ee | 790 | int i; |
1995876a SK |
791 | |
792 | init_dummy_netdev(&xfrm_napi_dev); | |
793 | err = gro_cells_init(&gro_cells, &xfrm_napi_dev); | |
794 | if (err) | |
795 | gro_cells.cells = NULL; | |
796 | ||
acf568ee HX |
797 | for_each_possible_cpu(i) { |
798 | struct xfrm_trans_tasklet *trans; | |
799 | ||
800 | trans = &per_cpu(xfrm_trans_tasklet, i); | |
801 | __skb_queue_head_init(&trans->queue); | |
802 | tasklet_init(&trans->tasklet, xfrm_trans_reinject, | |
803 | (unsigned long)trans); | |
804 | } | |
1da177e4 | 805 | } |