Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * xfrm_input.c | |
4 | * | |
5 | * Changes: | |
6 | * YOSHIFUJI Hideaki @USAGI | |
7 | * Split up af-specific portion | |
a716c119 | 8 | * |
1da177e4 LT |
9 | */ |
10 | ||
acf568ee | 11 | #include <linux/bottom_half.h> |
f8c3d0dd | 12 | #include <linux/cache.h> |
acf568ee | 13 | #include <linux/interrupt.h> |
1da177e4 LT |
14 | #include <linux/slab.h> |
15 | #include <linux/module.h> | |
716062fd | 16 | #include <linux/netdevice.h> |
acf568ee | 17 | #include <linux/percpu.h> |
716062fd | 18 | #include <net/dst.h> |
1da177e4 LT |
19 | #include <net/ip.h> |
20 | #include <net/xfrm.h> | |
049f8e2e AD |
21 | #include <net/ip_tunnels.h> |
22 | #include <net/ip6_tunnel.h> | |
1da177e4 | 23 | |
acf568ee HX |
24 | struct xfrm_trans_tasklet { |
25 | struct tasklet_struct tasklet; | |
26 | struct sk_buff_head queue; | |
27 | }; | |
28 | ||
29 | struct xfrm_trans_cb { | |
9a3fb9fb SK |
30 | union { |
31 | struct inet_skb_parm h4; | |
32 | #if IS_ENABLED(CONFIG_IPV6) | |
33 | struct inet6_skb_parm h6; | |
34 | #endif | |
35 | } header; | |
acf568ee HX |
36 | int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb); |
37 | }; | |
38 | ||
39 | #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0])) | |
40 | ||
f8c3d0dd | 41 | static struct kmem_cache *secpath_cachep __ro_after_init; |
1da177e4 | 42 | |
2f32b51b | 43 | static DEFINE_SPINLOCK(xfrm_input_afinfo_lock); |
960fdfde | 44 | static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1]; |
2f32b51b | 45 | |
1995876a SK |
46 | static struct gro_cells gro_cells; |
47 | static struct net_device xfrm_napi_dev; | |
48 | ||
acf568ee HX |
49 | static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet); |
50 | ||
960fdfde | 51 | int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
52 | { |
53 | int err = 0; | |
54 | ||
960fdfde | 55 | if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo))) |
2f32b51b | 56 | return -EAFNOSUPPORT; |
960fdfde | 57 | |
2f32b51b SK |
58 | spin_lock_bh(&xfrm_input_afinfo_lock); |
59 | if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL)) | |
f31e8d4f | 60 | err = -EEXIST; |
2f32b51b SK |
61 | else |
62 | rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo); | |
63 | spin_unlock_bh(&xfrm_input_afinfo_lock); | |
64 | return err; | |
65 | } | |
66 | EXPORT_SYMBOL(xfrm_input_register_afinfo); | |
67 | ||
960fdfde | 68 | int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo) |
2f32b51b SK |
69 | { |
70 | int err = 0; | |
71 | ||
2f32b51b SK |
72 | spin_lock_bh(&xfrm_input_afinfo_lock); |
73 | if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) { | |
74 | if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo)) | |
75 | err = -EINVAL; | |
76 | else | |
77 | RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL); | |
78 | } | |
79 | spin_unlock_bh(&xfrm_input_afinfo_lock); | |
80 | synchronize_rcu(); | |
81 | return err; | |
82 | } | |
83 | EXPORT_SYMBOL(xfrm_input_unregister_afinfo); | |
84 | ||
960fdfde | 85 | static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family) |
2f32b51b | 86 | { |
960fdfde | 87 | const struct xfrm_input_afinfo *afinfo; |
2f32b51b | 88 | |
960fdfde | 89 | if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo))) |
2f32b51b | 90 | return NULL; |
960fdfde | 91 | |
2f32b51b SK |
92 | rcu_read_lock(); |
93 | afinfo = rcu_dereference(xfrm_input_afinfo[family]); | |
94 | if (unlikely(!afinfo)) | |
95 | rcu_read_unlock(); | |
96 | return afinfo; | |
97 | } | |
98 | ||
2f32b51b SK |
99 | static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol, |
100 | int err) | |
101 | { | |
102 | int ret; | |
960fdfde | 103 | const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family); |
2f32b51b SK |
104 | |
105 | if (!afinfo) | |
106 | return -EAFNOSUPPORT; | |
107 | ||
108 | ret = afinfo->callback(skb, protocol, err); | |
960fdfde | 109 | rcu_read_unlock(); |
2f32b51b SK |
110 | |
111 | return ret; | |
112 | } | |
113 | ||
1da177e4 LT |
114 | void __secpath_destroy(struct sec_path *sp) |
115 | { | |
116 | int i; | |
117 | for (i = 0; i < sp->len; i++) | |
dbe5b4aa | 118 | xfrm_state_put(sp->xvec[i]); |
1da177e4 LT |
119 | kmem_cache_free(secpath_cachep, sp); |
120 | } | |
121 | EXPORT_SYMBOL(__secpath_destroy); | |
122 | ||
123 | struct sec_path *secpath_dup(struct sec_path *src) | |
124 | { | |
125 | struct sec_path *sp; | |
126 | ||
54e6ecb2 | 127 | sp = kmem_cache_alloc(secpath_cachep, GFP_ATOMIC); |
1da177e4 LT |
128 | if (!sp) |
129 | return NULL; | |
130 | ||
131 | sp->len = 0; | |
54ef207a SK |
132 | sp->olen = 0; |
133 | ||
f1193e91 | 134 | memset(sp->ovec, 0, sizeof(sp->ovec)); |
d77e38e6 | 135 | |
1da177e4 LT |
136 | if (src) { |
137 | int i; | |
138 | ||
139 | memcpy(sp, src, sizeof(*sp)); | |
140 | for (i = 0; i < sp->len; i++) | |
dbe5b4aa | 141 | xfrm_state_hold(sp->xvec[i]); |
1da177e4 | 142 | } |
55eabed6 | 143 | refcount_set(&sp->refcnt, 1); |
1da177e4 LT |
144 | return sp; |
145 | } | |
146 | EXPORT_SYMBOL(secpath_dup); | |
147 | ||
0ca64da1 | 148 | struct sec_path *secpath_set(struct sk_buff *skb) |
b0fcee82 | 149 | { |
0ca64da1 | 150 | struct sec_path *sp = skb->sp; |
b0fcee82 SK |
151 | |
152 | /* Allocate new secpath or COW existing one. */ | |
0ca64da1 | 153 | if (!sp || refcount_read(&sp->refcnt) != 1) { |
b0fcee82 SK |
154 | sp = secpath_dup(skb->sp); |
155 | if (!sp) | |
0ca64da1 | 156 | return NULL; |
b0fcee82 SK |
157 | |
158 | if (skb->sp) | |
159 | secpath_put(skb->sp); | |
160 | skb->sp = sp; | |
161 | } | |
0ca64da1 FW |
162 | |
163 | return sp; | |
b0fcee82 SK |
164 | } |
165 | EXPORT_SYMBOL(secpath_set); | |
166 | ||
1da177e4 LT |
167 | /* Fetch spi and seq from ipsec header */ |
168 | ||
6067b2ba | 169 | int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq) |
1da177e4 LT |
170 | { |
171 | int offset, offset_seq; | |
44072500 | 172 | int hlen; |
1da177e4 LT |
173 | |
174 | switch (nexthdr) { | |
175 | case IPPROTO_AH: | |
44072500 | 176 | hlen = sizeof(struct ip_auth_hdr); |
1da177e4 LT |
177 | offset = offsetof(struct ip_auth_hdr, spi); |
178 | offset_seq = offsetof(struct ip_auth_hdr, seq_no); | |
179 | break; | |
180 | case IPPROTO_ESP: | |
44072500 | 181 | hlen = sizeof(struct ip_esp_hdr); |
1da177e4 LT |
182 | offset = offsetof(struct ip_esp_hdr, spi); |
183 | offset_seq = offsetof(struct ip_esp_hdr, seq_no); | |
184 | break; | |
185 | case IPPROTO_COMP: | |
186 | if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr))) | |
187 | return -EINVAL; | |
3e94c2dc | 188 | *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2))); |
1da177e4 LT |
189 | *seq = 0; |
190 | return 0; | |
191 | default: | |
192 | return 1; | |
193 | } | |
194 | ||
44072500 | 195 | if (!pskb_may_pull(skb, hlen)) |
1da177e4 LT |
196 | return -EINVAL; |
197 | ||
3e94c2dc WC |
198 | *spi = *(__be32 *)(skb_transport_header(skb) + offset); |
199 | *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq); | |
1da177e4 LT |
200 | return 0; |
201 | } | |
1e295370 | 202 | EXPORT_SYMBOL(xfrm_parse_spi); |
1da177e4 | 203 | |
227620e2 HX |
204 | int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb) |
205 | { | |
df9dcb45 | 206 | struct xfrm_mode *inner_mode = x->inner_mode; |
227620e2 HX |
207 | int err; |
208 | ||
209 | err = x->outer_mode->afinfo->extract_input(x, skb); | |
210 | if (err) | |
211 | return err; | |
212 | ||
df9dcb45 KM |
213 | if (x->sel.family == AF_UNSPEC) { |
214 | inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); | |
215 | if (inner_mode == NULL) | |
216 | return -EAFNOSUPPORT; | |
217 | } | |
218 | ||
219 | skb->protocol = inner_mode->afinfo->eth_proto; | |
220 | return inner_mode->input2(x, skb); | |
227620e2 HX |
221 | } |
222 | EXPORT_SYMBOL(xfrm_prepare_input); | |
223 | ||
716062fd HX |
224 | int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type) |
225 | { | |
bd235e3c | 226 | struct net *net = dev_net(skb->dev); |
716062fd HX |
227 | int err; |
228 | __be32 seq; | |
2cd08467 | 229 | __be32 seq_hi; |
3328715e | 230 | struct xfrm_state *x = NULL; |
1bf06cd2 | 231 | xfrm_address_t *daddr; |
df9dcb45 | 232 | struct xfrm_mode *inner_mode; |
049f8e2e | 233 | u32 mark = skb->mark; |
4ce3dbe3 | 234 | unsigned int family = AF_UNSPEC; |
716062fd | 235 | int decaps = 0; |
1bf06cd2 | 236 | int async = 0; |
7785bba2 | 237 | bool xfrm_gro = false; |
d77e38e6 SK |
238 | bool crypto_done = false; |
239 | struct xfrm_offload *xo = xfrm_offload(skb); | |
0ca64da1 | 240 | struct sec_path *sp; |
1bf06cd2 | 241 | |
1bf06cd2 | 242 | if (encap_type < 0) { |
00501121 | 243 | x = xfrm_input_state(skb); |
4ce3dbe3 AH |
244 | |
245 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { | |
246 | if (x->km.state == XFRM_STATE_ACQ) | |
247 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
248 | else | |
249 | XFRM_INC_STATS(net, | |
250 | LINUX_MIB_XFRMINSTATEINVALID); | |
251 | goto drop; | |
252 | } | |
253 | ||
3328715e | 254 | family = x->outer_mode->afinfo->family; |
7785bba2 SK |
255 | |
256 | /* An encap_type of -1 indicates async resumption. */ | |
257 | if (encap_type == -1) { | |
258 | async = 1; | |
259 | seq = XFRM_SKB_CB(skb)->seq.input.low; | |
260 | goto resume; | |
261 | } | |
bcd1f8a4 | 262 | |
7785bba2 SK |
263 | /* encap_type < -1 indicates a GRO call. */ |
264 | encap_type = 0; | |
265 | seq = XFRM_SPI_SKB_CB(skb)->seq; | |
716062fd | 266 | |
bcd1f8a4 SK |
267 | if (xo && (xo->flags & CRYPTO_DONE)) { |
268 | crypto_done = true; | |
bcd1f8a4 SK |
269 | family = XFRM_SPI_SKB_CB(skb)->family; |
270 | ||
271 | if (!(xo->status & CRYPTO_SUCCESS)) { | |
272 | if (xo->status & | |
273 | (CRYPTO_TRANSPORT_AH_AUTH_FAILED | | |
274 | CRYPTO_TRANSPORT_ESP_AUTH_FAILED | | |
275 | CRYPTO_TUNNEL_AH_AUTH_FAILED | | |
276 | CRYPTO_TUNNEL_ESP_AUTH_FAILED)) { | |
277 | ||
278 | xfrm_audit_state_icvfail(x, skb, | |
279 | x->type->proto); | |
280 | x->stats.integrity_failed++; | |
281 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
282 | goto drop; | |
283 | } | |
284 | ||
47ebcc0b YK |
285 | if (xo->status & CRYPTO_INVALID_PROTOCOL) { |
286 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); | |
287 | goto drop; | |
288 | } | |
289 | ||
bcd1f8a4 | 290 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
d77e38e6 SK |
291 | goto drop; |
292 | } | |
293 | ||
bcd1f8a4 SK |
294 | if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { |
295 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); | |
296 | goto drop; | |
297 | } | |
d77e38e6 SK |
298 | } |
299 | ||
7785bba2 | 300 | goto lock; |
1bf06cd2 | 301 | } |
716062fd | 302 | |
3328715e SK |
303 | family = XFRM_SPI_SKB_CB(skb)->family; |
304 | ||
049f8e2e | 305 | /* if tunnel is present override skb->mark value with tunnel i_key */ |
1625f452 AK |
306 | switch (family) { |
307 | case AF_INET: | |
308 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4) | |
049f8e2e | 309 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key); |
1625f452 AK |
310 | break; |
311 | case AF_INET6: | |
312 | if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6) | |
049f8e2e | 313 | mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key); |
1625f452 | 314 | break; |
049f8e2e AD |
315 | } |
316 | ||
0ca64da1 FW |
317 | sp = secpath_set(skb); |
318 | if (!sp) { | |
b0fcee82 SK |
319 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR); |
320 | goto drop; | |
b2aa5e9d HX |
321 | } |
322 | ||
716062fd | 323 | seq = 0; |
0aa64774 | 324 | if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) { |
f203b76d | 325 | secpath_reset(skb); |
59c9940e | 326 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 327 | goto drop; |
0aa64774 | 328 | } |
716062fd | 329 | |
cb79a180 FW |
330 | daddr = (xfrm_address_t *)(skb_network_header(skb) + |
331 | XFRM_SPI_SKB_CB(skb)->daddroff); | |
716062fd | 332 | do { |
2294be0f FW |
333 | sp = skb_sec_path(skb); |
334 | ||
335 | if (sp->len == XFRM_MAX_DEPTH) { | |
f203b76d | 336 | secpath_reset(skb); |
59c9940e | 337 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
716062fd | 338 | goto drop; |
0aa64774 | 339 | } |
716062fd | 340 | |
049f8e2e | 341 | x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family); |
0aa64774 | 342 | if (x == NULL) { |
f203b76d | 343 | secpath_reset(skb); |
59c9940e | 344 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES); |
afeb14b4 | 345 | xfrm_audit_state_notfound(skb, family, spi, seq); |
716062fd | 346 | goto drop; |
0aa64774 | 347 | } |
716062fd | 348 | |
9b42c1f1 SK |
349 | skb->mark = xfrm_smark_get(skb->mark, x); |
350 | ||
2294be0f | 351 | sp->xvec[sp->len++] = x; |
b2aa5e9d | 352 | |
7785bba2 | 353 | lock: |
716062fd | 354 | spin_lock(&x->lock); |
4c4d41f2 | 355 | |
0aa64774 | 356 | if (unlikely(x->km.state != XFRM_STATE_VALID)) { |
dc0565ce LR |
357 | if (x->km.state == XFRM_STATE_ACQ) |
358 | XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR); | |
359 | else | |
360 | XFRM_INC_STATS(net, | |
361 | LINUX_MIB_XFRMINSTATEINVALID); | |
716062fd | 362 | goto drop_unlock; |
0aa64774 | 363 | } |
716062fd | 364 | |
3de77cf2 DM |
365 | if ((x->encap ? x->encap->encap_type : 0) != encap_type) { |
366 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); | |
367 | goto drop_unlock; | |
368 | } | |
369 | ||
36ae0148 | 370 | if (x->repl->check(x, skb, seq)) { |
59c9940e | 371 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
716062fd | 372 | goto drop_unlock; |
0aa64774 | 373 | } |
716062fd | 374 | |
0aa64774 | 375 | if (xfrm_state_check_expire(x)) { |
59c9940e | 376 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED); |
716062fd | 377 | goto drop_unlock; |
0aa64774 | 378 | } |
716062fd | 379 | |
0ebea8ef HX |
380 | spin_unlock(&x->lock); |
381 | ||
68c11e98 AD |
382 | if (xfrm_tunnel_check(skb, x, family)) { |
383 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); | |
384 | goto drop; | |
385 | } | |
386 | ||
2cd08467 SK |
387 | seq_hi = htonl(xfrm_replay_seqhi(x, seq)); |
388 | ||
1ce3644a | 389 | XFRM_SKB_CB(skb)->seq.input.low = seq; |
2cd08467 | 390 | XFRM_SKB_CB(skb)->seq.input.hi = seq_hi; |
1bf06cd2 | 391 | |
3bc07321 | 392 | skb_dst_force(skb); |
071d36bf | 393 | dev_hold(skb->dev); |
3bc07321 | 394 | |
d77e38e6 SK |
395 | if (crypto_done) |
396 | nexthdr = x->type_offload->input_tail(x, skb); | |
397 | else | |
398 | nexthdr = x->type->input(x, skb); | |
0ebea8ef | 399 | |
1bf06cd2 HX |
400 | if (nexthdr == -EINPROGRESS) |
401 | return 0; | |
1bf06cd2 | 402 | resume: |
071d36bf | 403 | dev_put(skb->dev); |
404 | ||
0ebea8ef | 405 | spin_lock(&x->lock); |
668dc8af | 406 | if (nexthdr <= 0) { |
9dd3245a HX |
407 | if (nexthdr == -EBADMSG) { |
408 | xfrm_audit_state_icvfail(x, skb, | |
409 | x->type->proto); | |
668dc8af | 410 | x->stats.integrity_failed++; |
9dd3245a | 411 | } |
59c9940e | 412 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR); |
716062fd | 413 | goto drop_unlock; |
668dc8af | 414 | } |
716062fd | 415 | |
716062fd HX |
416 | /* only the first xfrm gets the encap type */ |
417 | encap_type = 0; | |
418 | ||
3b59df46 | 419 | if (async && x->repl->recheck(x, skb, seq)) { |
bcf66bf5 SK |
420 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR); |
421 | goto drop_unlock; | |
422 | } | |
423 | ||
9fdc4883 | 424 | x->repl->advance(x, seq); |
716062fd HX |
425 | |
426 | x->curlft.bytes += skb->len; | |
427 | x->curlft.packets++; | |
428 | ||
429 | spin_unlock(&x->lock); | |
430 | ||
60d5fcfb HX |
431 | XFRM_MODE_SKB_CB(skb)->protocol = nexthdr; |
432 | ||
df9dcb45 KM |
433 | inner_mode = x->inner_mode; |
434 | ||
435 | if (x->sel.family == AF_UNSPEC) { | |
436 | inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol); | |
cb866e32 SK |
437 | if (inner_mode == NULL) { |
438 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); | |
df9dcb45 | 439 | goto drop; |
cb866e32 | 440 | } |
df9dcb45 KM |
441 | } |
442 | ||
443 | if (inner_mode->input(x, skb)) { | |
59c9940e | 444 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR); |
716062fd | 445 | goto drop; |
0aa64774 | 446 | } |
716062fd HX |
447 | |
448 | if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) { | |
449 | decaps = 1; | |
450 | break; | |
451 | } | |
452 | ||
1bf06cd2 HX |
453 | /* |
454 | * We need the inner address. However, we only get here for | |
455 | * transport mode so the outer address is identical. | |
456 | */ | |
457 | daddr = &x->id.daddr; | |
2fcb45b6 | 458 | family = x->outer_mode->afinfo->family; |
1bf06cd2 | 459 | |
716062fd | 460 | err = xfrm_parse_spi(skb, nexthdr, &spi, &seq); |
0aa64774 | 461 | if (err < 0) { |
59c9940e | 462 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
716062fd | 463 | goto drop; |
0aa64774 | 464 | } |
782710e3 | 465 | crypto_done = false; |
716062fd HX |
466 | } while (!err); |
467 | ||
3328715e SK |
468 | err = xfrm_rcv_cb(skb, family, x->type->proto, 0); |
469 | if (err) | |
470 | goto drop; | |
471 | ||
716062fd HX |
472 | nf_reset(skb); |
473 | ||
474 | if (decaps) { | |
2294be0f FW |
475 | sp = skb_sec_path(skb); |
476 | if (sp) | |
477 | sp->olen = 0; | |
adf30907 | 478 | skb_dst_drop(skb); |
1995876a | 479 | gro_cells_receive(&gro_cells, skb); |
716062fd HX |
480 | return 0; |
481 | } else { | |
7785bba2 SK |
482 | xo = xfrm_offload(skb); |
483 | if (xo) | |
484 | xfrm_gro = xo->flags & XFRM_GRO; | |
485 | ||
cfcf99f9 | 486 | err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async); |
7785bba2 | 487 | if (xfrm_gro) { |
2294be0f FW |
488 | sp = skb_sec_path(skb); |
489 | if (sp) | |
490 | sp->olen = 0; | |
7785bba2 SK |
491 | skb_dst_drop(skb); |
492 | gro_cells_receive(&gro_cells, skb); | |
493 | return err; | |
494 | } | |
495 | ||
496 | return err; | |
716062fd HX |
497 | } |
498 | ||
499 | drop_unlock: | |
500 | spin_unlock(&x->lock); | |
716062fd | 501 | drop: |
3328715e | 502 | xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1); |
716062fd HX |
503 | kfree_skb(skb); |
504 | return 0; | |
505 | } | |
506 | EXPORT_SYMBOL(xfrm_input); | |
507 | ||
1bf06cd2 HX |
508 | int xfrm_input_resume(struct sk_buff *skb, int nexthdr) |
509 | { | |
510 | return xfrm_input(skb, nexthdr, 0, -1); | |
511 | } | |
512 | EXPORT_SYMBOL(xfrm_input_resume); | |
513 | ||
acf568ee HX |
514 | static void xfrm_trans_reinject(unsigned long data) |
515 | { | |
516 | struct xfrm_trans_tasklet *trans = (void *)data; | |
517 | struct sk_buff_head queue; | |
518 | struct sk_buff *skb; | |
519 | ||
520 | __skb_queue_head_init(&queue); | |
521 | skb_queue_splice_init(&trans->queue, &queue); | |
522 | ||
523 | while ((skb = __skb_dequeue(&queue))) | |
524 | XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb); | |
525 | } | |
526 | ||
527 | int xfrm_trans_queue(struct sk_buff *skb, | |
528 | int (*finish)(struct net *, struct sock *, | |
529 | struct sk_buff *)) | |
530 | { | |
531 | struct xfrm_trans_tasklet *trans; | |
532 | ||
533 | trans = this_cpu_ptr(&xfrm_trans_tasklet); | |
534 | ||
535 | if (skb_queue_len(&trans->queue) >= netdev_max_backlog) | |
536 | return -ENOBUFS; | |
537 | ||
538 | XFRM_TRANS_SKB_CB(skb)->finish = finish; | |
d16b46e4 | 539 | __skb_queue_tail(&trans->queue, skb); |
acf568ee HX |
540 | tasklet_schedule(&trans->tasklet); |
541 | return 0; | |
542 | } | |
543 | EXPORT_SYMBOL(xfrm_trans_queue); | |
544 | ||
1da177e4 LT |
545 | void __init xfrm_input_init(void) |
546 | { | |
1995876a | 547 | int err; |
acf568ee | 548 | int i; |
1995876a SK |
549 | |
550 | init_dummy_netdev(&xfrm_napi_dev); | |
551 | err = gro_cells_init(&gro_cells, &xfrm_napi_dev); | |
552 | if (err) | |
553 | gro_cells.cells = NULL; | |
554 | ||
1da177e4 LT |
555 | secpath_cachep = kmem_cache_create("secpath_cache", |
556 | sizeof(struct sec_path), | |
e5d679f3 | 557 | 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, |
20c2df83 | 558 | NULL); |
acf568ee HX |
559 | |
560 | for_each_possible_cpu(i) { | |
561 | struct xfrm_trans_tasklet *trans; | |
562 | ||
563 | trans = &per_cpu(xfrm_trans_tasklet, i); | |
564 | __skb_queue_head_init(&trans->queue); | |
565 | tasklet_init(&trans->tasklet, xfrm_trans_reinject, | |
566 | (unsigned long)trans); | |
567 | } | |
1da177e4 | 568 | } |