projects / linux-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
xfrm: state: remove extract_input indirection from xfrm_state_afinfo
[linux-block.git] / net / xfrm / xfrm_input.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * xfrm_input.c
4 *
5 * Changes:
6 * YOSHIFUJI Hideaki @USAGI
7 * Split up af-specific portion
8 *
9 */
10
11#include <linux/bottom_half.h>
12#include <linux/cache.h>
13#include <linux/interrupt.h>
14#include <linux/slab.h>
15#include <linux/module.h>
16#include <linux/netdevice.h>
17#include <linux/percpu.h>
18#include <net/dst.h>
19#include <net/ip.h>
20#include <net/xfrm.h>
21#include <net/ip_tunnels.h>
22#include <net/ip6_tunnel.h>
23
24#include "xfrm_inout.h"
25
26struct xfrm_trans_tasklet {
27 struct tasklet_struct tasklet;
28 struct sk_buff_head queue;
29};
30
31struct xfrm_trans_cb {
32 union {
33 struct inet_skb_parm h4;
34#if IS_ENABLED(CONFIG_IPV6)
35 struct inet6_skb_parm h6;
36#endif
37 } header;
38 int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
39 struct net *net;
40};
41
42#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
43
44static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
45static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];
46
47static struct gro_cells gro_cells;
48static struct net_device xfrm_napi_dev;
49
50static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
51
52int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
53{
54 int err = 0;
55
56 if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
57 return -EAFNOSUPPORT;
58
59 spin_lock_bh(&xfrm_input_afinfo_lock);
60 if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
61 err = -EEXIST;
62 else
63 rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
64 spin_unlock_bh(&xfrm_input_afinfo_lock);
65 return err;
66}
67EXPORT_SYMBOL(xfrm_input_register_afinfo);
68
69int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
70{
71 int err = 0;
72
73 spin_lock_bh(&xfrm_input_afinfo_lock);
74 if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
75 if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
76 err = -EINVAL;
77 else
78 RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
79 }
80 spin_unlock_bh(&xfrm_input_afinfo_lock);
81 synchronize_rcu();
82 return err;
83}
84EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
85
86static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
87{
88 const struct xfrm_input_afinfo *afinfo;
89
90 if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
91 return NULL;
92
93 rcu_read_lock();
94 afinfo = rcu_dereference(xfrm_input_afinfo[family]);
95 if (unlikely(!afinfo))
96 rcu_read_unlock();
97 return afinfo;
98}
99
100static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
101 int err)
102{
103 int ret;
104 const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
105
106 if (!afinfo)
107 return -EAFNOSUPPORT;
108
109 ret = afinfo->callback(skb, protocol, err);
110 rcu_read_unlock();
111
112 return ret;
113}
114
115struct sec_path *secpath_set(struct sk_buff *skb)
116{
117 struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
118
119 sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
120 if (!sp)
121 return NULL;
122
123 if (tmp) /* reused existing one (was COW'd if needed) */
124 return sp;
125
126 /* allocated new secpath */
127 memset(sp->ovec, 0, sizeof(sp->ovec));
128 sp->olen = 0;
129 sp->len = 0;
130
131 return sp;
132}
133EXPORT_SYMBOL(secpath_set);
134
135/* Fetch spi and seq from ipsec header */
136
137int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
138{
139 int offset, offset_seq;
140 int hlen;
141
142 switch (nexthdr) {
143 case IPPROTO_AH:
144 hlen = sizeof(struct ip_auth_hdr);
145 offset = offsetof(struct ip_auth_hdr, spi);
146 offset_seq = offsetof(struct ip_auth_hdr, seq_no);
147 break;
148 case IPPROTO_ESP:
149 hlen = sizeof(struct ip_esp_hdr);
150 offset = offsetof(struct ip_esp_hdr, spi);
151 offset_seq = offsetof(struct ip_esp_hdr, seq_no);
152 break;
153 case IPPROTO_COMP:
154 if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
155 return -EINVAL;
156 *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
157 *seq = 0;
158 return 0;
159 default:
160 return 1;
161 }
162
163 if (!pskb_may_pull(skb, hlen))
164 return -EINVAL;
165
166 *spi = *(__be32 *)(skb_transport_header(skb) + offset);
167 *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
168 return 0;
169}
170EXPORT_SYMBOL(xfrm_parse_spi);
171
172static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
173{
174 struct iphdr *iph;
175 int optlen = 0;
176 int err = -EINVAL;
177
178 if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
179 struct ip_beet_phdr *ph;
180 int phlen;
181
182 if (!pskb_may_pull(skb, sizeof(*ph)))
183 goto out;
184
185 ph = (struct ip_beet_phdr *)skb->data;
186
187 phlen = sizeof(*ph) + ph->padlen;
188 optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
189 if (optlen < 0 || optlen & 3 || optlen > 250)
190 goto out;
191
192 XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;
193
194 if (!pskb_may_pull(skb, phlen))
195 goto out;
196 __skb_pull(skb, phlen);
197 }
198
199 skb_push(skb, sizeof(*iph));
200 skb_reset_network_header(skb);
201 skb_mac_header_rebuild(skb);
202
203 xfrm4_beet_make_header(skb);
204
205 iph = ip_hdr(skb);
206
207 iph->ihl += optlen / 4;
208 iph->tot_len = htons(skb->len);
209 iph->daddr = x->sel.daddr.a4;
210 iph->saddr = x->sel.saddr.a4;
211 iph->check = 0;
212 iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
213 err = 0;
214out:
215 return err;
216}
217
218static void ipip_ecn_decapsulate(struct sk_buff *skb)
219{
220 struct iphdr *inner_iph = ipip_hdr(skb);
221
222 if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
223 IP_ECN_set_ce(inner_iph);
224}
225
226static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
227{
228 int err = -EINVAL;
229
230 if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
231 goto out;
232
233 if (!pskb_may_pull(skb, sizeof(struct iphdr)))
234 goto out;
235
236 err = skb_unclone(skb, GFP_ATOMIC);
237 if (err)
238 goto out;
239
240 if (x->props.flags & XFRM_STATE_DECAP_DSCP)
241 ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
242 if (!(x->props.flags & XFRM_STATE_NOECN))
243 ipip_ecn_decapsulate(skb);
244
245 skb_reset_network_header(skb);
246 skb_mac_header_rebuild(skb);
247 if (skb->mac_len)
248 eth_hdr(skb)->h_proto = skb->protocol;
249
250 err = 0;
251
252out:
253 return err;
254}
255
256static void ipip6_ecn_decapsulate(struct sk_buff *skb)
257{
258 struct ipv6hdr *inner_iph = ipipv6_hdr(skb);
259
260 if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
261 IP6_ECN_set_ce(skb, inner_iph);
262}
263
264static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
265{
266 int err = -EINVAL;
267
268 if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
269 goto out;
270 if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
271 goto out;
272
273 err = skb_unclone(skb, GFP_ATOMIC);
274 if (err)
275 goto out;
276
277 if (x->props.flags & XFRM_STATE_DECAP_DSCP)
278 ipv6_copy_dscp(ipv6_get_dsfield(ipv6_hdr(skb)),
279 ipipv6_hdr(skb));
280 if (!(x->props.flags & XFRM_STATE_NOECN))
281 ipip6_ecn_decapsulate(skb);
282
283 skb_reset_network_header(skb);
284 skb_mac_header_rebuild(skb);
285 if (skb->mac_len)
286 eth_hdr(skb)->h_proto = skb->protocol;
287
288 err = 0;
289
290out:
291 return err;
292}
293
294static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
295{
296 struct ipv6hdr *ip6h;
297 int size = sizeof(struct ipv6hdr);
298 int err;
299
300 err = skb_cow_head(skb, size + skb->mac_len);
301 if (err)
302 goto out;
303
304 __skb_push(skb, size);
305 skb_reset_network_header(skb);
306 skb_mac_header_rebuild(skb);
307
308 xfrm6_beet_make_header(skb);
309
310 ip6h = ipv6_hdr(skb);
311 ip6h->payload_len = htons(skb->len - size);
312 ip6h->daddr = x->sel.daddr.in6;
313 ip6h->saddr = x->sel.saddr.in6;
314 err = 0;
315out:
316 return err;
317}
318
319/* Remove encapsulation header.
320 *
321 * The IP header will be moved over the top of the encapsulation
322 * header.
323 *
324 * On entry, the transport header shall point to where the IP header
325 * should be and the network header shall be set to where the IP
326 * header currently is. skb->data shall point to the start of the
327 * payload.
328 */
329static int
330xfrm_inner_mode_encap_remove(struct xfrm_state *x,
331 const struct xfrm_mode *inner_mode,
332 struct sk_buff *skb)
333{
334 switch (inner_mode->encap) {
335 case XFRM_MODE_BEET:
336 if (inner_mode->family == AF_INET)
337 return xfrm4_remove_beet_encap(x, skb);
338 if (inner_mode->family == AF_INET6)
339 return xfrm6_remove_beet_encap(x, skb);
340 break;
341 case XFRM_MODE_TUNNEL:
342 if (inner_mode->family == AF_INET)
343 return xfrm4_remove_tunnel_encap(x, skb);
344 if (inner_mode->family == AF_INET6)
345 return xfrm6_remove_tunnel_encap(x, skb);
346 break;
347 }
348
349 WARN_ON_ONCE(1);
350 return -EOPNOTSUPP;
351}
352
353static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
354{
355 const struct xfrm_mode *inner_mode = &x->inner_mode;
356
357 switch (x->outer_mode.family) {
358 case AF_INET:
359 xfrm4_extract_header(skb);
360 break;
361 case AF_INET6:
362 xfrm6_extract_header(skb);
363 break;
364 default:
365 WARN_ON_ONCE(1);
366 return -EAFNOSUPPORT;
367 }
368
369 if (x->sel.family == AF_UNSPEC) {
370 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
371 if (!inner_mode)
372 return -EAFNOSUPPORT;
373 }
374
375 switch (inner_mode->family) {
376 case AF_INET:
377 skb->protocol = htons(ETH_P_IP);
378 break;
379 case AF_INET6:
380 skb->protocol = htons(ETH_P_IPV6);
381 break;
382 default:
383 WARN_ON_ONCE(1);
384 break;
385 }
386
387 return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
388}
389
390/* Remove encapsulation header.
391 *
392 * The IP header will be moved over the top of the encapsulation header.
393 *
394 * On entry, skb_transport_header() shall point to where the IP header
395 * should be and skb_network_header() shall be set to where the IP header
396 * currently is. skb->data shall point to the start of the payload.
397 */
398static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
399{
400 int ihl = skb->data - skb_transport_header(skb);
401
402 if (skb->transport_header != skb->network_header) {
403 memmove(skb_transport_header(skb),
404 skb_network_header(skb), ihl);
405 skb->network_header = skb->transport_header;
406 }
407 ip_hdr(skb)->tot_len = htons(skb->len + ihl);
408 skb_reset_transport_header(skb);
409 return 0;
410}
411
412static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
413{
414#if IS_ENABLED(CONFIG_IPV6)
415 int ihl = skb->data - skb_transport_header(skb);
416
417 if (skb->transport_header != skb->network_header) {
418 memmove(skb_transport_header(skb),
419 skb_network_header(skb), ihl);
420 skb->network_header = skb->transport_header;
421 }
422 ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
423 sizeof(struct ipv6hdr));
424 skb_reset_transport_header(skb);
425 return 0;
426#else
427 WARN_ON_ONCE(1);
428 return -EAFNOSUPPORT;
429#endif
430}
431
432static int xfrm_inner_mode_input(struct xfrm_state *x,
433 const struct xfrm_mode *inner_mode,
434 struct sk_buff *skb)
435{
436 switch (inner_mode->encap) {
437 case XFRM_MODE_BEET:
438 case XFRM_MODE_TUNNEL:
439 return xfrm_prepare_input(x, skb);
440 case XFRM_MODE_TRANSPORT:
441 if (inner_mode->family == AF_INET)
442 return xfrm4_transport_input(x, skb);
443 if (inner_mode->family == AF_INET6)
444 return xfrm6_transport_input(x, skb);
445 break;
446 case XFRM_MODE_ROUTEOPTIMIZATION:
447 WARN_ON_ONCE(1);
448 break;
449 default:
450 WARN_ON_ONCE(1);
451 break;
452 }
453
454 return -EOPNOTSUPP;
455}
456
457int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
458{
459 const struct xfrm_state_afinfo *afinfo;
460 struct net *net = dev_net(skb->dev);
461 const struct xfrm_mode *inner_mode;
462 int err;
463 __be32 seq;
464 __be32 seq_hi;
465 struct xfrm_state *x = NULL;
466 xfrm_address_t *daddr;
467 u32 mark = skb->mark;
468 unsigned int family = AF_UNSPEC;
469 int decaps = 0;
470 int async = 0;
471 bool xfrm_gro = false;
472 bool crypto_done = false;
473 struct xfrm_offload *xo = xfrm_offload(skb);
474 struct sec_path *sp;
475
476 if (encap_type < 0) {
477 x = xfrm_input_state(skb);
478
479 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
480 if (x->km.state == XFRM_STATE_ACQ)
481 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
482 else
483 XFRM_INC_STATS(net,
484 LINUX_MIB_XFRMINSTATEINVALID);
485
486 if (encap_type == -1)
487 dev_put(skb->dev);
488 goto drop;
489 }
490
491 family = x->outer_mode.family;
492
493 /* An encap_type of -1 indicates async resumption. */
494 if (encap_type == -1) {
495 async = 1;
496 seq = XFRM_SKB_CB(skb)->seq.input.low;
497 goto resume;
498 }
499
500 /* encap_type < -1 indicates a GRO call. */
501 encap_type = 0;
502 seq = XFRM_SPI_SKB_CB(skb)->seq;
503
504 if (xo && (xo->flags & CRYPTO_DONE)) {
505 crypto_done = true;
506 family = XFRM_SPI_SKB_CB(skb)->family;
507
508 if (!(xo->status & CRYPTO_SUCCESS)) {
509 if (xo->status &
510 (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
511 CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
512 CRYPTO_TUNNEL_AH_AUTH_FAILED |
513 CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
514
515 xfrm_audit_state_icvfail(x, skb,
516 x->type->proto);
517 x->stats.integrity_failed++;
518 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
519 goto drop;
520 }
521
522 if (xo->status & CRYPTO_INVALID_PROTOCOL) {
523 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
524 goto drop;
525 }
526
527 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
528 goto drop;
529 }
530
531 if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
532 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
533 goto drop;
534 }
535 }
536
537 goto lock;
538 }
539
540 family = XFRM_SPI_SKB_CB(skb)->family;
541
542 /* if tunnel is present override skb->mark value with tunnel i_key */
543 switch (family) {
544 case AF_INET:
545 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
546 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
547 break;
548 case AF_INET6:
549 if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
550 mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
551 break;
552 }
553
554 sp = secpath_set(skb);
555 if (!sp) {
556 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
557 goto drop;
558 }
559
560 seq = 0;
561 if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
562 secpath_reset(skb);
563 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
564 goto drop;
565 }
566
567 daddr = (xfrm_address_t *)(skb_network_header(skb) +
568 XFRM_SPI_SKB_CB(skb)->daddroff);
569 do {
570 sp = skb_sec_path(skb);
571
572 if (sp->len == XFRM_MAX_DEPTH) {
573 secpath_reset(skb);
574 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
575 goto drop;
576 }
577
578 x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
579 if (x == NULL) {
580 secpath_reset(skb);
581 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
582 xfrm_audit_state_notfound(skb, family, spi, seq);
583 goto drop;
584 }
585
586 skb->mark = xfrm_smark_get(skb->mark, x);
587
588 sp->xvec[sp->len++] = x;
589
590 skb_dst_force(skb);
591 if (!skb_dst(skb)) {
592 XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
593 goto drop;
594 }
595
596lock:
597 spin_lock(&x->lock);
598
599 if (unlikely(x->km.state != XFRM_STATE_VALID)) {
600 if (x->km.state == XFRM_STATE_ACQ)
601 XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
602 else
603 XFRM_INC_STATS(net,
604 LINUX_MIB_XFRMINSTATEINVALID);
605 goto drop_unlock;
606 }
607
608 if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
609 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
610 goto drop_unlock;
611 }
612
613 if (x->repl->check(x, skb, seq)) {
614 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
615 goto drop_unlock;
616 }
617
618 if (xfrm_state_check_expire(x)) {
619 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
620 goto drop_unlock;
621 }
622
623 spin_unlock(&x->lock);
624
625 if (xfrm_tunnel_check(skb, x, family)) {
626 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
627 goto drop;
628 }
629
630 seq_hi = htonl(xfrm_replay_seqhi(x, seq));
631
632 XFRM_SKB_CB(skb)->seq.input.low = seq;
633 XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
634
635 dev_hold(skb->dev);
636
637 if (crypto_done)
638 nexthdr = x->type_offload->input_tail(x, skb);
639 else
640 nexthdr = x->type->input(x, skb);
641
642 if (nexthdr == -EINPROGRESS)
643 return 0;
644resume:
645 dev_put(skb->dev);
646
647 spin_lock(&x->lock);
648 if (nexthdr <= 0) {
649 if (nexthdr == -EBADMSG) {
650 xfrm_audit_state_icvfail(x, skb,
651 x->type->proto);
652 x->stats.integrity_failed++;
653 }
654 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
655 goto drop_unlock;
656 }
657
658 /* only the first xfrm gets the encap type */
659 encap_type = 0;
660
661 if (async && x->repl->recheck(x, skb, seq)) {
662 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
663 goto drop_unlock;
664 }
665
666 x->repl->advance(x, seq);
667
668 x->curlft.bytes += skb->len;
669 x->curlft.packets++;
670
671 spin_unlock(&x->lock);
672
673 XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
674
675 inner_mode = &x->inner_mode;
676
677 if (x->sel.family == AF_UNSPEC) {
678 inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
679 if (inner_mode == NULL) {
680 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
681 goto drop;
682 }
683 }
684
685 if (xfrm_inner_mode_input(x, inner_mode, skb)) {
686 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
687 goto drop;
688 }
689
690 if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
691 decaps = 1;
692 break;
693 }
694
695 /*
696 * We need the inner address. However, we only get here for
697 * transport mode so the outer address is identical.
698 */
699 daddr = &x->id.daddr;
700 family = x->outer_mode.family;
701
702 err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
703 if (err < 0) {
704 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
705 goto drop;
706 }
707 crypto_done = false;
708 } while (!err);
709
710 err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
711 if (err)
712 goto drop;
713
714 nf_reset_ct(skb);
715
716 if (decaps) {
717 sp = skb_sec_path(skb);
718 if (sp)
719 sp->olen = 0;
720 skb_dst_drop(skb);
721 gro_cells_receive(&gro_cells, skb);
722 return 0;
723 } else {
724 xo = xfrm_offload(skb);
725 if (xo)
726 xfrm_gro = xo->flags & XFRM_GRO;
727
728 err = -EAFNOSUPPORT;
729 rcu_read_lock();
730 afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
731 if (likely(afinfo))
732 err = afinfo->transport_finish(skb, xfrm_gro || async);
733 rcu_read_unlock();
734 if (xfrm_gro) {
735 sp = skb_sec_path(skb);
736 if (sp)
737 sp->olen = 0;
738 skb_dst_drop(skb);
739 gro_cells_receive(&gro_cells, skb);
740 return err;
741 }
742
743 return err;
744 }
745
746drop_unlock:
747 spin_unlock(&x->lock);
748drop:
749 xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
750 kfree_skb(skb);
751 return 0;
752}
753EXPORT_SYMBOL(xfrm_input);
754
755int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
756{
757 return xfrm_input(skb, nexthdr, 0, -1);
758}
759EXPORT_SYMBOL(xfrm_input_resume);
760
761static void xfrm_trans_reinject(unsigned long data)
762{
763 struct xfrm_trans_tasklet *trans = (void *)data;
764 struct sk_buff_head queue;
765 struct sk_buff *skb;
766
767 __skb_queue_head_init(&queue);
768 skb_queue_splice_init(&trans->queue, &queue);
769
770 while ((skb = __skb_dequeue(&queue)))
771 XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
772 NULL, skb);
773}
774
775int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
776 int (*finish)(struct net *, struct sock *,
777 struct sk_buff *))
778{
779 struct xfrm_trans_tasklet *trans;
780
781 trans = this_cpu_ptr(&xfrm_trans_tasklet);
782
783 if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
784 return -ENOBUFS;
785
786 BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
787
788 XFRM_TRANS_SKB_CB(skb)->finish = finish;
789 XFRM_TRANS_SKB_CB(skb)->net = net;
790 __skb_queue_tail(&trans->queue, skb);
791 tasklet_schedule(&trans->tasklet);
792 return 0;
793}
794EXPORT_SYMBOL(xfrm_trans_queue_net);
795
796int xfrm_trans_queue(struct sk_buff *skb,
797 int (*finish)(struct net *, struct sock *,
798 struct sk_buff *))
799{
800 return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
801}
802EXPORT_SYMBOL(xfrm_trans_queue);
803
804void __init xfrm_input_init(void)
805{
806 int err;
807 int i;
808
809 init_dummy_netdev(&xfrm_napi_dev);
810 err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
811 if (err)
812 gro_cells.cells = NULL;
813
814 for_each_possible_cpu(i) {
815 struct xfrm_trans_tasklet *trans;
816
817 trans = &per_cpu(xfrm_trans_tasklet, i);
818 __skb_queue_head_init(&trans->queue);
819 tasklet_init(&trans->tasklet, xfrm_trans_reinject,
820 (unsigned long)trans);
821 }
822}
Linux 6.x block layer and io_uring tree(s)