projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xfrm: state: remove extract_input indirection from xfrm_state_afinfo
[linux-block.git] / net / xfrm / xfrm_input.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
1da177e4
LT		 2/*
3 * xfrm_input.c
4 *
5 * Changes:
6 * YOSHIFUJI Hideaki @USAGI
7 * Split up af-specific portion
a716c119 8 *
1da177e4
LT		 9 */
10
acf568ee 11#include <linux/bottom_half.h>
f8c3d0dd 12#include <linux/cache.h>
acf568ee 13#include <linux/interrupt.h>
1da177e4
LT		 14#include <linux/slab.h>
15#include <linux/module.h>
716062fd 16#include <linux/netdevice.h>
acf568ee 17#include <linux/percpu.h>
716062fd 18#include <net/dst.h>
1da177e4
LT		 19#include <net/ip.h>
20#include <net/xfrm.h>
049f8e2e
AD		 21#include <net/ip_tunnels.h>
22#include <net/ip6_tunnel.h>
1da177e4 23
b3284df1
FW		 24#include "xfrm_inout.h"
25
acf568ee
HX		 26struct xfrm_trans_tasklet {
27 struct tasklet_struct tasklet;
28 struct sk_buff_head queue;
29};
30
31struct xfrm_trans_cb {
9a3fb9fb
SK		 32 union {
33 struct inet_skb_parm h4;
34#if IS_ENABLED(CONFIG_IPV6)
35 struct inet6_skb_parm h6;
36#endif
37 } header;
acf568ee 38 int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
7b380192 39 struct net *net;
acf568ee
HX		 40};
41
42#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
43
2f32b51b 44static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
960fdfde 45static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];
2f32b51b 46
1995876a
SK		 47static struct gro_cells gro_cells;
48static struct net_device xfrm_napi_dev;
49
acf568ee
HX		 50static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
51
960fdfde 52int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b
SK		 53{
54 int err = 0;
55
960fdfde 56 if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b 57 return -EAFNOSUPPORT;
960fdfde 58
2f32b51b
SK		 59 spin_lock_bh(&xfrm_input_afinfo_lock);
60 if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
f31e8d4f 61 err = -EEXIST;
2f32b51b
SK		 62 else
63 rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
64 spin_unlock_bh(&xfrm_input_afinfo_lock);
65 return err;
66}
67EXPORT_SYMBOL(xfrm_input_register_afinfo);
68
960fdfde 69int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b
SK		 70{
71 int err = 0;
72
2f32b51b
SK		 73 spin_lock_bh(&xfrm_input_afinfo_lock);
74 if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
75 if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
76 err = -EINVAL;
77 else
78 RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
79 }
80 spin_unlock_bh(&xfrm_input_afinfo_lock);
81 synchronize_rcu();
82 return err;
83}
84EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
85
960fdfde 86static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
2f32b51b 87{
960fdfde 88 const struct xfrm_input_afinfo *afinfo;
2f32b51b 89
960fdfde 90 if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b 91 return NULL;
960fdfde 92
2f32b51b
SK		 93 rcu_read_lock();
94 afinfo = rcu_dereference(xfrm_input_afinfo[family]);
95 if (unlikely(!afinfo))
96 rcu_read_unlock();
97 return afinfo;
98}
99
2f32b51b
SK		 100static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
101 int err)
102{
103 int ret;
960fdfde 104 const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
2f32b51b
SK		 105
106 if (!afinfo)
107 return -EAFNOSUPPORT;
108
109 ret = afinfo->callback(skb, protocol, err);
960fdfde 110 rcu_read_unlock();
2f32b51b
SK		 111
112 return ret;
113}
114
4165079b 115struct sec_path *secpath_set(struct sk_buff *skb)
1da177e4 116{
4165079b 117 struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
1da177e4 118
4165079b 119 sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
1da177e4
LT		 120 if (!sp)
121 return NULL;
122
4165079b
FW		 123 if (tmp) /* reused existing one (was COW'd if needed) */
124 return sp;
54ef207a 125
4165079b 126 /* allocated new secpath */
f1193e91 127 memset(sp->ovec, 0, sizeof(sp->ovec));
4165079b
FW		 128 sp->olen = 0;
129 sp->len = 0;
0ca64da1
FW		 130
131 return sp;
b0fcee82
SK		 132}
133EXPORT_SYMBOL(secpath_set);
134
1da177e4
LT		 135/* Fetch spi and seq from ipsec header */
136
6067b2ba 137int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
1da177e4
LT		 138{
139 int offset, offset_seq;
44072500 140 int hlen;
1da177e4
LT		 141
142 switch (nexthdr) {
143 case IPPROTO_AH:
44072500 144 hlen = sizeof(struct ip_auth_hdr);
1da177e4
LT		 145 offset = offsetof(struct ip_auth_hdr, spi);
146 offset_seq = offsetof(struct ip_auth_hdr, seq_no);
147 break;
148 case IPPROTO_ESP:
44072500 149 hlen = sizeof(struct ip_esp_hdr);
1da177e4
LT		 150 offset = offsetof(struct ip_esp_hdr, spi);
151 offset_seq = offsetof(struct ip_esp_hdr, seq_no);
152 break;
153 case IPPROTO_COMP:
154 if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
155 return -EINVAL;
3e94c2dc 156 *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
1da177e4
LT		 157 *seq = 0;
158 return 0;
159 default:
160 return 1;
161 }
162
44072500 163 if (!pskb_may_pull(skb, hlen))
1da177e4
LT		 164 return -EINVAL;
165
3e94c2dc
WC		 166 *spi = *(__be32 *)(skb_transport_header(skb) + offset);
167 *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
1da177e4
LT		 168 return 0;
169}
1e295370 170EXPORT_SYMBOL(xfrm_parse_spi);
1da177e4 171
b3284df1
FW		 172static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
173{
174 struct iphdr *iph;
175 int optlen = 0;
176 int err = -EINVAL;
177
178 if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
179 struct ip_beet_phdr *ph;
180 int phlen;
181
182 if (!pskb_may_pull(skb, sizeof(*ph)))
183 goto out;
184
185 ph = (struct ip_beet_phdr *)skb->data;
186
187 phlen = sizeof(*ph) + ph->padlen;
188 optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
189 if (optlen < 0 || optlen & 3 || optlen > 250)
190 goto out;
191
192 XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;
193
194 if (!pskb_may_pull(skb, phlen))
195 goto out;
196 __skb_pull(skb, phlen);
197 }
198
199 skb_push(skb, sizeof(*iph));
200 skb_reset_network_header(skb);
201 skb_mac_header_rebuild(skb);
202
203 xfrm4_beet_make_header(skb);
204
205 iph = ip_hdr(skb);
206
207 iph->ihl += optlen / 4;
208 iph->tot_len = htons(skb->len);
209 iph->daddr = x->sel.daddr.a4;
210 iph->saddr = x->sel.saddr.a4;
211 iph->check = 0;
212 iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
213 err = 0;
214out:
215 return err;
216}
217
218static void ipip_ecn_decapsulate(struct sk_buff *skb)
219{
220 struct iphdr *inner_iph = ipip_hdr(skb);
221
222 if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
223 IP_ECN_set_ce(inner_iph);
224}
225
226static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
227{
228 int err = -EINVAL;
229
230 if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
231 goto out;
232
233 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
234 goto out;
235
236 err = skb_unclone(skb, GFP_ATOMIC);
237 if (err)
238 goto out;
239
240 if (x->props.flags & XFRM_STATE_DECAP_DSCP)
241 ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
242 if (!(x->props.flags & XFRM_STATE_NOECN))
243 ipip_ecn_decapsulate(skb);
244
245 skb_reset_network_header(skb);
246 skb_mac_header_rebuild(skb);
247 if (skb->mac_len)
248 eth_hdr(skb)->h_proto = skb->protocol;
249
250 err = 0;
251
252out:
253 return err;
254}
255
256static void ipip6_ecn_decapsulate(struct sk_buff *skb)
257{
258 struct ipv6hdr *inner_iph = ipipv6_hdr(skb);
259
260 if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
261 IP6_ECN_set_ce(skb, inner_iph);
262}
263
264static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
265{
266 int err = -EINVAL;
267
268 if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
269 goto out;
270 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
271 goto out;
272
273 err = skb_unclone(skb, GFP_ATOMIC);
274 if (err)
275 goto out;
276
277 if (x->props.flags & XFRM_STATE_DECAP_DSCP)
278 ipv6_copy_dscp(ipv6_get_dsfield(ipv6_hdr(skb)),
279 ipipv6_hdr(skb));
280 if (!(x->props.flags & XFRM_STATE_NOECN))
281 ipip6_ecn_decapsulate(skb);
282
283 skb_reset_network_header(skb);
284 skb_mac_header_rebuild(skb);
285 if (skb->mac_len)
286 eth_hdr(skb)->h_proto = skb->protocol;
287
288 err = 0;
289
290out:
291 return err;
292}
293
294static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
295{
296 struct ipv6hdr *ip6h;
297 int size = sizeof(struct ipv6hdr);
298 int err;
299
300 err = skb_cow_head(skb, size + skb->mac_len);
301 if (err)
302 goto out;
303
304 __skb_push(skb, size);
305 skb_reset_network_header(skb);
306 skb_mac_header_rebuild(skb);
307
308 xfrm6_beet_make_header(skb);
309
310 ip6h = ipv6_hdr(skb);
311 ip6h->payload_len = htons(skb->len - size);
312 ip6h->daddr = x->sel.daddr.in6;
313 ip6h->saddr = x->sel.saddr.in6;
314 err = 0;
315out:
316 return err;
317}
318
319/* Remove encapsulation header.
320 *
321 * The IP header will be moved over the top of the encapsulation
322 * header.
323 *
324 * On entry, the transport header shall point to where the IP header
325 * should be and the network header shall be set to where the IP
326 * header currently is. skb->data shall point to the start of the
327 * payload.
328 */
329static int
330xfrm_inner_mode_encap_remove(struct xfrm_state *x,
331 const struct xfrm_mode *inner_mode,
332 struct sk_buff *skb)
333{
334 switch (inner_mode->encap) {
335 case XFRM_MODE_BEET:
336 if (inner_mode->family == AF_INET)
337 return xfrm4_remove_beet_encap(x, skb);
338 if (inner_mode->family == AF_INET6)
339 return xfrm6_remove_beet_encap(x, skb);
340 break;
341 case XFRM_MODE_TUNNEL:
342 if (inner_mode->family == AF_INET)
343 return xfrm4_remove_tunnel_encap(x, skb);
344 if (inner_mode->family == AF_INET6)
345 return xfrm6_remove_tunnel_encap(x, skb);
346 break;
347 }
348
349 WARN_ON_ONCE(1);
350 return -EOPNOTSUPP;
351}
352
c2d305e5 353static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
227620e2 354{
c9500d7b 355 const struct xfrm_mode *inner_mode = &x->inner_mode;
227620e2 356
a269fbfc
FW		 357 switch (x->outer_mode.family) {
358 case AF_INET:
359 xfrm4_extract_header(skb);
360 break;
361 case AF_INET6:
362 xfrm6_extract_header(skb);
363 break;
364 default:
365 WARN_ON_ONCE(1);
366 return -EAFNOSUPPORT;
367 }
227620e2 368
df9dcb45
KM		 369 if (x->sel.family == AF_UNSPEC) {
370 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
4c203b04 371 if (!inner_mode)
df9dcb45
KM		 372 return -EAFNOSUPPORT;
373 }
374
4c203b04
FW		 375 switch (inner_mode->family) {
376 case AF_INET:
377 skb->protocol = htons(ETH_P_IP);
378 break;
379 case AF_INET6:
380 skb->protocol = htons(ETH_P_IPV6);
1be451d9 381 break;
4c203b04
FW		 382 default:
383 WARN_ON_ONCE(1);
384 break;
733a5fac
FW		 385 }
386
b3284df1 387 return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
227620e2 388}
c2d305e5
FW		 389
390/* Remove encapsulation header.
391 *
392 * The IP header will be moved over the top of the encapsulation header.
393 *
394 * On entry, skb_transport_header() shall point to where the IP header
395 * should be and skb_network_header() shall be set to where the IP header
396 * currently is. skb->data shall point to the start of the payload.
397 */
398static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
399{
c2d305e5
FW		 400 int ihl = skb->data - skb_transport_header(skb);
401
402 if (skb->transport_header != skb->network_header) {
403 memmove(skb_transport_header(skb),
404 skb_network_header(skb), ihl);
405 skb->network_header = skb->transport_header;
406 }
407 ip_hdr(skb)->tot_len = htons(skb->len + ihl);
408 skb_reset_transport_header(skb);
409 return 0;
c2d305e5
FW		 410}
411
412static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
413{
4c145dce 414#if IS_ENABLED(CONFIG_IPV6)
c2d305e5
FW		 415 int ihl = skb->data - skb_transport_header(skb);
416
417 if (skb->transport_header != skb->network_header) {
418 memmove(skb_transport_header(skb),
419 skb_network_header(skb), ihl);
420 skb->network_header = skb->transport_header;
421 }
422 ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
423 sizeof(struct ipv6hdr));
424 skb_reset_transport_header(skb);
425 return 0;
426#else
4c145dce
FW		 427 WARN_ON_ONCE(1);
428 return -EAFNOSUPPORT;
c2d305e5
FW		 429#endif
430}
431
432static int xfrm_inner_mode_input(struct xfrm_state *x,
433 const struct xfrm_mode *inner_mode,
434 struct sk_buff *skb)
435{
436 switch (inner_mode->encap) {
437 case XFRM_MODE_BEET:
438 case XFRM_MODE_TUNNEL:
439 return xfrm_prepare_input(x, skb);
440 case XFRM_MODE_TRANSPORT:
441 if (inner_mode->family == AF_INET)
442 return xfrm4_transport_input(x, skb);
443 if (inner_mode->family == AF_INET6)
444 return xfrm6_transport_input(x, skb);
445 break;
446 case XFRM_MODE_ROUTEOPTIMIZATION:
447 WARN_ON_ONCE(1);
448 break;
449 default:
450 WARN_ON_ONCE(1);
451 break;
452 }
453
454 return -EOPNOTSUPP;
455}
227620e2 456
716062fd
HX		 457int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
458{
733a5fac 459 const struct xfrm_state_afinfo *afinfo;
bd235e3c 460 struct net *net = dev_net(skb->dev);
4c145dce 461 const struct xfrm_mode *inner_mode;
716062fd
HX		 462 int err;
463 __be32 seq;
2cd08467 464 __be32 seq_hi;
3328715e 465 struct xfrm_state *x = NULL;
1bf06cd2 466 xfrm_address_t *daddr;
049f8e2e 467 u32 mark = skb->mark;
4ce3dbe3 468 unsigned int family = AF_UNSPEC;
716062fd 469 int decaps = 0;
1bf06cd2 470 int async = 0;
7785bba2 471 bool xfrm_gro = false;
d77e38e6
SK		 472 bool crypto_done = false;
473 struct xfrm_offload *xo = xfrm_offload(skb);
0ca64da1 474 struct sec_path *sp;
1bf06cd2 475
1bf06cd2 476 if (encap_type < 0) {
00501121 477 x = xfrm_input_state(skb);
4ce3dbe3
AH		 478
479 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
480 if (x->km.state == XFRM_STATE_ACQ)
481 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
482 else
483 XFRM_INC_STATS(net,
484 LINUX_MIB_XFRMINSTATEINVALID);
4944a4b1
XX		 485
486 if (encap_type == -1)
487 dev_put(skb->dev);
4ce3dbe3
AH		 488 goto drop;
489 }
490
c9500d7b 491 family = x->outer_mode.family;
7785bba2
SK		 492
493 /* An encap_type of -1 indicates async resumption. */
494 if (encap_type == -1) {
495 async = 1;
496 seq = XFRM_SKB_CB(skb)->seq.input.low;
497 goto resume;
498 }
bcd1f8a4 499
7785bba2
SK		 500 /* encap_type < -1 indicates a GRO call. */
501 encap_type = 0;
502 seq = XFRM_SPI_SKB_CB(skb)->seq;
716062fd 503
bcd1f8a4
SK		 504 if (xo && (xo->flags & CRYPTO_DONE)) {
505 crypto_done = true;
bcd1f8a4
SK		 506 family = XFRM_SPI_SKB_CB(skb)->family;
507
508 if (!(xo->status & CRYPTO_SUCCESS)) {
509 if (xo->status &
510 (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
511 CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
512 CRYPTO_TUNNEL_AH_AUTH_FAILED |
513 CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
514
515 xfrm_audit_state_icvfail(x, skb,
516 x->type->proto);
517 x->stats.integrity_failed++;
518 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
519 goto drop;
520 }
521
47ebcc0b
YK		 522 if (xo->status & CRYPTO_INVALID_PROTOCOL) {
523 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
524 goto drop;
525 }
526
bcd1f8a4 527 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
d77e38e6
SK		 528 goto drop;
529 }
530
bcd1f8a4
SK		 531 if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
532 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
533 goto drop;
534 }
d77e38e6
SK		 535 }
536
7785bba2 537 goto lock;
1bf06cd2 538 }
716062fd 539
3328715e
SK		 540 family = XFRM_SPI_SKB_CB(skb)->family;
541
049f8e2e 542 /* if tunnel is present override skb->mark value with tunnel i_key */
1625f452
AK		 543 switch (family) {
544 case AF_INET:
545 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
049f8e2e 546 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
1625f452
AK		 547 break;
548 case AF_INET6:
549 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
049f8e2e 550 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
1625f452 551 break;
049f8e2e
AD		 552 }
553
0ca64da1
FW		 554 sp = secpath_set(skb);
555 if (!sp) {
b0fcee82
SK		 556 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
557 goto drop;
b2aa5e9d
HX		 558 }
559
716062fd 560 seq = 0;
0aa64774 561 if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
f203b76d 562 secpath_reset(skb);
59c9940e 563 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd 564 goto drop;
0aa64774 565 }
716062fd 566
cb79a180
FW		 567 daddr = (xfrm_address_t *)(skb_network_header(skb) +
568 XFRM_SPI_SKB_CB(skb)->daddroff);
716062fd 569 do {
2294be0f
FW		 570 sp = skb_sec_path(skb);
571
572 if (sp->len == XFRM_MAX_DEPTH) {
f203b76d 573 secpath_reset(skb);
59c9940e 574 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
716062fd 575 goto drop;
0aa64774 576 }
716062fd 577
049f8e2e 578 x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
0aa64774 579 if (x == NULL) {
f203b76d 580 secpath_reset(skb);
59c9940e 581 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
afeb14b4 582 xfrm_audit_state_notfound(skb, family, spi, seq);
716062fd 583 goto drop;
0aa64774 584 }
716062fd 585
9b42c1f1
SK		 586 skb->mark = xfrm_smark_get(skb->mark, x);
587
2294be0f 588 sp->xvec[sp->len++] = x;
b2aa5e9d 589
0152eee6
SK		 590 skb_dst_force(skb);
591 if (!skb_dst(skb)) {
592 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
593 goto drop;
594 }
595
7785bba2 596lock:
716062fd 597 spin_lock(&x->lock);
4c4d41f2 598
0aa64774 599 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
dc0565ce
LR		 600 if (x->km.state == XFRM_STATE_ACQ)
601 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
602 else
603 XFRM_INC_STATS(net,
604 LINUX_MIB_XFRMINSTATEINVALID);
716062fd 605 goto drop_unlock;
0aa64774 606 }
716062fd 607
3de77cf2
DM		 608 if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
609 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
610 goto drop_unlock;
611 }
612
36ae0148 613 if (x->repl->check(x, skb, seq)) {
59c9940e 614 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
716062fd 615 goto drop_unlock;
0aa64774 616 }
716062fd 617
0aa64774 618 if (xfrm_state_check_expire(x)) {
59c9940e 619 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
716062fd 620 goto drop_unlock;
0aa64774 621 }
716062fd 622
0ebea8ef
HX		 623 spin_unlock(&x->lock);
624
68c11e98
AD		 625 if (xfrm_tunnel_check(skb, x, family)) {
626 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
627 goto drop;
628 }
629
2cd08467
SK		 630 seq_hi = htonl(xfrm_replay_seqhi(x, seq));
631
1ce3644a 632 XFRM_SKB_CB(skb)->seq.input.low = seq;
2cd08467 633 XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
1bf06cd2 634
071d36bf 635 dev_hold(skb->dev);
3bc07321 636
d77e38e6
SK		 637 if (crypto_done)
638 nexthdr = x->type_offload->input_tail(x, skb);
639 else
640 nexthdr = x->type->input(x, skb);
0ebea8ef 641
1bf06cd2
HX		 642 if (nexthdr == -EINPROGRESS)
643 return 0;
1bf06cd2 644resume:
071d36bf 645 dev_put(skb->dev);
646
0ebea8ef 647 spin_lock(&x->lock);
668dc8af 648 if (nexthdr <= 0) {
9dd3245a
HX		 649 if (nexthdr == -EBADMSG) {
650 xfrm_audit_state_icvfail(x, skb,
651 x->type->proto);
668dc8af 652 x->stats.integrity_failed++;
9dd3245a 653 }
59c9940e 654 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
716062fd 655 goto drop_unlock;
668dc8af 656 }
716062fd 657
716062fd
HX		 658 /* only the first xfrm gets the encap type */
659 encap_type = 0;
660
3b59df46 661 if (async && x->repl->recheck(x, skb, seq)) {
bcf66bf5
SK		 662 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
663 goto drop_unlock;
664 }
665
9fdc4883 666 x->repl->advance(x, seq);
716062fd
HX		 667
668 x->curlft.bytes += skb->len;
669 x->curlft.packets++;
670
671 spin_unlock(&x->lock);
672
60d5fcfb
HX		 673 XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
674
c9500d7b 675 inner_mode = &x->inner_mode;
df9dcb45
KM		 676
677 if (x->sel.family == AF_UNSPEC) {
678 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
cb866e32
SK		 679 if (inner_mode == NULL) {
680 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
df9dcb45 681 goto drop;
cb866e32 682 }
df9dcb45
KM		 683 }
684
c2d305e5 685 if (xfrm_inner_mode_input(x, inner_mode, skb)) {
59c9940e 686 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
716062fd 687 goto drop;
0aa64774 688 }
716062fd 689
c9500d7b 690 if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
716062fd
HX		 691 decaps = 1;
692 break;
693 }
694
1bf06cd2
HX		 695 /*
696 * We need the inner address. However, we only get here for
697 * transport mode so the outer address is identical.
698 */
699 daddr = &x->id.daddr;
c9500d7b 700 family = x->outer_mode.family;
1bf06cd2 701
716062fd 702 err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
0aa64774 703 if (err < 0) {
59c9940e 704 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd 705 goto drop;
0aa64774 706 }
782710e3 707 crypto_done = false;
716062fd
HX		 708 } while (!err);
709
3328715e
SK		 710 err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
711 if (err)
712 goto drop;
713
895b5c9f 714 nf_reset_ct(skb);
716062fd
HX		 715
716 if (decaps) {
2294be0f
FW		 717 sp = skb_sec_path(skb);
718 if (sp)
719 sp->olen = 0;
adf30907 720 skb_dst_drop(skb);
1995876a 721 gro_cells_receive(&gro_cells, skb);
716062fd
HX		 722 return 0;
723 } else {
7785bba2
SK		 724 xo = xfrm_offload(skb);
725 if (xo)
726 xfrm_gro = xo->flags & XFRM_GRO;
727
733a5fac
FW		 728 err = -EAFNOSUPPORT;
729 rcu_read_lock();
c9500d7b 730 afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
733a5fac
FW		 731 if (likely(afinfo))
732 err = afinfo->transport_finish(skb, xfrm_gro || async);
733 rcu_read_unlock();
7785bba2 734 if (xfrm_gro) {
2294be0f
FW		 735 sp = skb_sec_path(skb);
736 if (sp)
737 sp->olen = 0;
7785bba2
SK		 738 skb_dst_drop(skb);
739 gro_cells_receive(&gro_cells, skb);
740 return err;
741 }
742
743 return err;
716062fd
HX		 744 }
745
746drop_unlock:
747 spin_unlock(&x->lock);
716062fd 748drop:
3328715e 749 xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
716062fd
HX		 750 kfree_skb(skb);
751 return 0;
752}
753EXPORT_SYMBOL(xfrm_input);
754
1bf06cd2
HX		 755int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
756{
757 return xfrm_input(skb, nexthdr, 0, -1);
758}
759EXPORT_SYMBOL(xfrm_input_resume);
760
acf568ee
HX		 761static void xfrm_trans_reinject(unsigned long data)
762{
763 struct xfrm_trans_tasklet *trans = (void *)data;
764 struct sk_buff_head queue;
765 struct sk_buff *skb;
766
767 __skb_queue_head_init(&queue);
768 skb_queue_splice_init(&trans->queue, &queue);
769
770 while ((skb = __skb_dequeue(&queue)))
7b380192
SD		 771 XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
772 NULL, skb);
acf568ee
HX		 773}
774
7b380192
SD		 775int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
776 int (*finish)(struct net *, struct sock *,
777 struct sk_buff *))
acf568ee
HX		 778{
779 struct xfrm_trans_tasklet *trans;
780
781 trans = this_cpu_ptr(&xfrm_trans_tasklet);
782
783 if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
784 return -ENOBUFS;
785
7b380192
SD		 786 BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
787
acf568ee 788 XFRM_TRANS_SKB_CB(skb)->finish = finish;
7b380192 789 XFRM_TRANS_SKB_CB(skb)->net = net;
d16b46e4 790 __skb_queue_tail(&trans->queue, skb);
acf568ee
HX		 791 tasklet_schedule(&trans->tasklet);
792 return 0;
793}
7b380192
SD		 794EXPORT_SYMBOL(xfrm_trans_queue_net);
795
796int xfrm_trans_queue(struct sk_buff *skb,
797 int (*finish)(struct net *, struct sock *,
798 struct sk_buff *))
799{
800 return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
801}
acf568ee
HX		 802EXPORT_SYMBOL(xfrm_trans_queue);
803
1da177e4
LT		 804void __init xfrm_input_init(void)
805{
1995876a 806 int err;
acf568ee 807 int i;
1995876a
SK		 808
809 init_dummy_netdev(&xfrm_napi_dev);
810 err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
811 if (err)
812 gro_cells.cells = NULL;
813
acf568ee
HX		 814 for_each_possible_cpu(i) {
815 struct xfrm_trans_tasklet *trans;
816
817 trans = &per_cpu(xfrm_trans_tasklet, i);
818 __skb_queue_head_init(&trans->queue);
819 tasklet_init(&trans->tasklet, xfrm_trans_reinject,
820 (unsigned long)trans);
821 }
1da177e4 822}
Linux 6.x block layer and io_uring tree(s)