Commit | Line | Data |
---|---|---|
75a6faf6 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7785bba2 SK |
2 | /* |
3 | * IPV4 GSO/GRO offload support | |
4 | * Linux INET implementation | |
5 | * | |
6 | * Copyright (C) 2016 secunet Security Networks AG | |
7 | * Author: Steffen Klassert <steffen.klassert@secunet.com> | |
8 | * | |
7785bba2 SK |
9 | * ESP GRO support |
10 | */ | |
11 | ||
12 | #include <linux/skbuff.h> | |
13 | #include <linux/init.h> | |
14 | #include <net/protocol.h> | |
15 | #include <crypto/aead.h> | |
16 | #include <crypto/authenc.h> | |
17 | #include <linux/err.h> | |
18 | #include <linux/module.h> | |
4721031c | 19 | #include <net/gro.h> |
d457a0e3 | 20 | #include <net/gso.h> |
7785bba2 SK |
21 | #include <net/ip.h> |
22 | #include <net/xfrm.h> | |
23 | #include <net/esp.h> | |
24 | #include <linux/scatterlist.h> | |
25 | #include <linux/kernel.h> | |
26 | #include <linux/slab.h> | |
27 | #include <linux/spinlock.h> | |
28 | #include <net/udp.h> | |
29 | ||
d4546c25 DM |
30 | static struct sk_buff *esp4_gro_receive(struct list_head *head, |
31 | struct sk_buff *skb) | |
7785bba2 SK |
32 | { |
33 | int offset = skb_gro_offset(skb); | |
34 | struct xfrm_offload *xo; | |
35 | struct xfrm_state *x; | |
172bf009 | 36 | int encap_type = 0; |
7785bba2 SK |
37 | __be32 seq; |
38 | __be32 spi; | |
7785bba2 | 39 | |
374d1b5a SK |
40 | if (!pskb_pull(skb, offset)) |
41 | return NULL; | |
7785bba2 | 42 | |
335a2a1f | 43 | if (xfrm_parse_spi(skb, IPPROTO_ESP, &spi, &seq) != 0) |
7785bba2 SK |
44 | goto out; |
45 | ||
bcd1f8a4 SK |
46 | xo = xfrm_offload(skb); |
47 | if (!xo || !(xo->flags & CRYPTO_DONE)) { | |
0ca64da1 FW |
48 | struct sec_path *sp = secpath_set(skb); |
49 | ||
50 | if (!sp) | |
bcd1f8a4 | 51 | goto out; |
7785bba2 | 52 | |
0ca64da1 | 53 | if (sp->len == XFRM_MAX_DEPTH) |
6ed69184 | 54 | goto out_reset; |
7785bba2 | 55 | |
bcd1f8a4 SK |
56 | x = xfrm_state_lookup(dev_net(skb->dev), skb->mark, |
57 | (xfrm_address_t *)&ip_hdr(skb)->daddr, | |
58 | spi, IPPROTO_ESP, AF_INET); | |
15f5fe9e AA |
59 | |
60 | if (unlikely(x && x->dir && x->dir != XFRM_SA_DIR_IN)) { | |
61 | /* non-offload path will record the error and audit log */ | |
62 | xfrm_state_put(x); | |
63 | x = NULL; | |
64 | } | |
65 | ||
bcd1f8a4 | 66 | if (!x) |
6ed69184 | 67 | goto out_reset; |
7785bba2 | 68 | |
4e4362d2 UW |
69 | skb->mark = xfrm_smark_get(skb->mark, x); |
70 | ||
0ca64da1 FW |
71 | sp->xvec[sp->len++] = x; |
72 | sp->olen++; | |
7785bba2 | 73 | |
bcd1f8a4 | 74 | xo = xfrm_offload(skb); |
db87668a | 75 | if (!xo) |
6ed69184 | 76 | goto out_reset; |
7785bba2 | 77 | } |
bcd1f8a4 | 78 | |
7785bba2 SK |
79 | xo->flags |= XFRM_GRO; |
80 | ||
172bf009 SK |
81 | if (NAPI_GRO_CB(skb)->proto == IPPROTO_UDP) |
82 | encap_type = UDP_ENCAP_ESPINUDP; | |
83 | ||
7785bba2 SK |
84 | XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4 = NULL; |
85 | XFRM_SPI_SKB_CB(skb)->family = AF_INET; | |
86 | XFRM_SPI_SKB_CB(skb)->daddroff = offsetof(struct iphdr, daddr); | |
87 | XFRM_SPI_SKB_CB(skb)->seq = seq; | |
88 | ||
89 | /* We don't need to handle errors from xfrm_input, it does all | |
90 | * the error handling and frees the resources on error. */ | |
172bf009 | 91 | xfrm_input(skb, IPPROTO_ESP, spi, encap_type); |
7785bba2 SK |
92 | |
93 | return ERR_PTR(-EINPROGRESS); | |
6ed69184 MJ |
94 | out_reset: |
95 | secpath_reset(skb); | |
7785bba2 SK |
96 | out: |
97 | skb_push(skb, offset); | |
98 | NAPI_GRO_CB(skb)->same_flow = 0; | |
99 | NAPI_GRO_CB(skb)->flush = 1; | |
100 | ||
101 | return NULL; | |
102 | } | |
103 | ||
7862b405 SK |
104 | static void esp4_gso_encap(struct xfrm_state *x, struct sk_buff *skb) |
105 | { | |
106 | struct ip_esp_hdr *esph; | |
107 | struct iphdr *iph = ip_hdr(skb); | |
108 | struct xfrm_offload *xo = xfrm_offload(skb); | |
109 | int proto = iph->protocol; | |
110 | ||
111 | skb_push(skb, -skb_network_offset(skb)); | |
112 | esph = ip_esp_hdr(skb); | |
113 | *skb_mac_header(skb) = IPPROTO_ESP; | |
114 | ||
115 | esph->spi = x->id.spi; | |
116 | esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low); | |
117 | ||
118 | xo->proto = proto; | |
119 | } | |
120 | ||
7613b92b FW |
121 | static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x, |
122 | struct sk_buff *skb, | |
123 | netdev_features_t features) | |
124 | { | |
26dbd66e SD |
125 | __be16 type = x->inner_mode.family == AF_INET6 ? htons(ETH_P_IPV6) |
126 | : htons(ETH_P_IP); | |
127 | ||
128 | return skb_eth_gso_segment(skb, features, type); | |
7613b92b FW |
129 | } |
130 | ||
131 | static struct sk_buff *xfrm4_transport_gso_segment(struct xfrm_state *x, | |
132 | struct sk_buff *skb, | |
133 | netdev_features_t features) | |
134 | { | |
135 | const struct net_offload *ops; | |
136 | struct sk_buff *segs = ERR_PTR(-EINVAL); | |
137 | struct xfrm_offload *xo = xfrm_offload(skb); | |
138 | ||
139 | skb->transport_header += x->props.header_len; | |
140 | ops = rcu_dereference(inet_offloads[xo->proto]); | |
141 | if (likely(ops && ops->callbacks.gso_segment)) | |
142 | segs = ops->callbacks.gso_segment(skb, features); | |
143 | ||
144 | return segs; | |
145 | } | |
146 | ||
384a46ea XL |
147 | static struct sk_buff *xfrm4_beet_gso_segment(struct xfrm_state *x, |
148 | struct sk_buff *skb, | |
149 | netdev_features_t features) | |
150 | { | |
151 | struct xfrm_offload *xo = xfrm_offload(skb); | |
152 | struct sk_buff *segs = ERR_PTR(-EINVAL); | |
153 | const struct net_offload *ops; | |
6f297068 | 154 | u8 proto = xo->proto; |
384a46ea XL |
155 | |
156 | skb->transport_header += x->props.header_len; | |
157 | ||
3ffb93ba XL |
158 | if (x->sel.family != AF_INET6) { |
159 | if (proto == IPPROTO_BEETPH) { | |
160 | struct ip_beet_phdr *ph = | |
161 | (struct ip_beet_phdr *)skb->data; | |
162 | ||
163 | skb->transport_header += ph->hdrlen * 8; | |
164 | proto = ph->nexthdr; | |
165 | } else { | |
166 | skb->transport_header -= IPV4_BEET_PHMAXLEN; | |
167 | } | |
168 | } else { | |
6f297068 XL |
169 | __be16 frag; |
170 | ||
171 | skb->transport_header += | |
172 | ipv6_skip_exthdr(skb, 0, &proto, &frag); | |
173 | if (proto == IPPROTO_TCP) | |
174 | skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4; | |
384a46ea XL |
175 | } |
176 | ||
053c8fdf SK |
177 | if (proto == IPPROTO_IPV6) |
178 | skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4; | |
179 | ||
384a46ea XL |
180 | __skb_pull(skb, skb_transport_offset(skb)); |
181 | ops = rcu_dereference(inet_offloads[proto]); | |
182 | if (likely(ops && ops->callbacks.gso_segment)) | |
183 | segs = ops->callbacks.gso_segment(skb, features); | |
184 | ||
185 | return segs; | |
186 | } | |
187 | ||
7613b92b FW |
188 | static struct sk_buff *xfrm4_outer_mode_gso_segment(struct xfrm_state *x, |
189 | struct sk_buff *skb, | |
190 | netdev_features_t features) | |
191 | { | |
c9500d7b | 192 | switch (x->outer_mode.encap) { |
7613b92b FW |
193 | case XFRM_MODE_TUNNEL: |
194 | return xfrm4_tunnel_gso_segment(x, skb, features); | |
195 | case XFRM_MODE_TRANSPORT: | |
196 | return xfrm4_transport_gso_segment(x, skb, features); | |
384a46ea XL |
197 | case XFRM_MODE_BEET: |
198 | return xfrm4_beet_gso_segment(x, skb, features); | |
7613b92b FW |
199 | } |
200 | ||
201 | return ERR_PTR(-EOPNOTSUPP); | |
202 | } | |
203 | ||
7862b405 SK |
204 | static struct sk_buff *esp4_gso_segment(struct sk_buff *skb, |
205 | netdev_features_t features) | |
206 | { | |
7862b405 SK |
207 | struct xfrm_state *x; |
208 | struct ip_esp_hdr *esph; | |
209 | struct crypto_aead *aead; | |
7862b405 SK |
210 | netdev_features_t esp_features = features; |
211 | struct xfrm_offload *xo = xfrm_offload(skb); | |
2294be0f | 212 | struct sec_path *sp; |
7862b405 SK |
213 | |
214 | if (!xo) | |
3dca3f38 | 215 | return ERR_PTR(-EINVAL); |
7862b405 | 216 | |
121d57af | 217 | if (!(skb_shinfo(skb)->gso_type & SKB_GSO_ESP)) |
5ca11440 | 218 | return ERR_PTR(-EINVAL); |
7862b405 | 219 | |
2294be0f FW |
220 | sp = skb_sec_path(skb); |
221 | x = sp->xvec[sp->len - 1]; | |
7862b405 SK |
222 | aead = x->data; |
223 | esph = ip_esp_hdr(skb); | |
224 | ||
225 | if (esph->spi != x->id.spi) | |
3dca3f38 | 226 | return ERR_PTR(-EINVAL); |
7862b405 SK |
227 | |
228 | if (!pskb_may_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead))) | |
3dca3f38 | 229 | return ERR_PTR(-EINVAL); |
7862b405 SK |
230 | |
231 | __skb_pull(skb, sizeof(*esph) + crypto_aead_ivsize(aead)); | |
232 | ||
233 | skb->encap_hdr_csum = 1; | |
234 | ||
65fd2c2a BP |
235 | if ((!(skb->dev->gso_partial_features & NETIF_F_HW_ESP) && |
236 | !(features & NETIF_F_HW_ESP)) || x->xso.dev != skb->dev) | |
154deab6 XL |
237 | esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK | |
238 | NETIF_F_SCTP_CRC); | |
65fd2c2a BP |
239 | else if (!(features & NETIF_F_HW_ESP_TX_CSUM) && |
240 | !(skb->dev->gso_partial_features & NETIF_F_HW_ESP_TX_CSUM)) | |
154deab6 XL |
241 | esp_features = features & ~(NETIF_F_CSUM_MASK | |
242 | NETIF_F_SCTP_CRC); | |
7862b405 | 243 | |
3dca3f38 | 244 | xo->flags |= XFRM_GSO_SEGMENT; |
7862b405 | 245 | |
7613b92b | 246 | return xfrm4_outer_mode_gso_segment(x, skb, esp_features); |
7862b405 SK |
247 | } |
248 | ||
fca11ebd SK |
249 | static int esp_input_tail(struct xfrm_state *x, struct sk_buff *skb) |
250 | { | |
251 | struct crypto_aead *aead = x->data; | |
ec9567a9 | 252 | struct xfrm_offload *xo = xfrm_offload(skb); |
fca11ebd SK |
253 | |
254 | if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead))) | |
255 | return -EINVAL; | |
256 | ||
ec9567a9 IT |
257 | if (!(xo->flags & CRYPTO_DONE)) |
258 | skb->ip_summed = CHECKSUM_NONE; | |
fca11ebd SK |
259 | |
260 | return esp_input_done2(skb, 0); | |
261 | } | |
262 | ||
263 | static int esp_xmit(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) | |
264 | { | |
265 | int err; | |
266 | int alen; | |
267 | int blksize; | |
268 | struct xfrm_offload *xo; | |
269 | struct ip_esp_hdr *esph; | |
270 | struct crypto_aead *aead; | |
271 | struct esp_info esp; | |
272 | bool hw_offload = true; | |
3dca3f38 | 273 | __u32 seq; |
447bc4b1 | 274 | int encap_type = 0; |
fca11ebd SK |
275 | |
276 | esp.inplace = true; | |
277 | ||
278 | xo = xfrm_offload(skb); | |
279 | ||
280 | if (!xo) | |
281 | return -EINVAL; | |
282 | ||
65fd2c2a BP |
283 | if ((!(features & NETIF_F_HW_ESP) && |
284 | !(skb->dev->gso_partial_features & NETIF_F_HW_ESP)) || | |
285 | x->xso.dev != skb->dev) { | |
fca11ebd SK |
286 | xo->flags |= CRYPTO_FALLBACK; |
287 | hw_offload = false; | |
288 | } | |
289 | ||
290 | esp.proto = xo->proto; | |
291 | ||
292 | /* skb is pure payload to encrypt */ | |
293 | ||
294 | aead = x->data; | |
295 | alen = crypto_aead_authsize(aead); | |
296 | ||
297 | esp.tfclen = 0; | |
298 | /* XXX: Add support for tfc padding here. */ | |
299 | ||
300 | blksize = ALIGN(crypto_aead_blocksize(aead), 4); | |
301 | esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize); | |
302 | esp.plen = esp.clen - skb->len - esp.tfclen; | |
303 | esp.tailen = esp.tfclen + esp.plen + alen; | |
304 | ||
305 | esp.esph = ip_esp_hdr(skb); | |
306 | ||
447bc4b1 MY |
307 | if (x->encap) |
308 | encap_type = x->encap->encap_type; | |
fca11ebd | 309 | |
447bc4b1 | 310 | if (!hw_offload || !skb_is_gso(skb) || (hw_offload && encap_type == UDP_ENCAP_ESPINUDP)) { |
fca11ebd SK |
311 | esp.nfrags = esp_output_head(x, skb, &esp); |
312 | if (esp.nfrags < 0) | |
313 | return esp.nfrags; | |
314 | } | |
315 | ||
3dca3f38 SK |
316 | seq = xo->seq.low; |
317 | ||
fca11ebd SK |
318 | esph = esp.esph; |
319 | esph->spi = x->id.spi; | |
320 | ||
321 | skb_push(skb, -skb_network_offset(skb)); | |
322 | ||
323 | if (xo->flags & XFRM_GSO_SEGMENT) { | |
3dca3f38 SK |
324 | esph->seq_no = htonl(seq); |
325 | ||
326 | if (!skb_is_gso(skb)) | |
327 | xo->seq.low++; | |
328 | else | |
329 | xo->seq.low += skb_shinfo(skb)->gso_segs; | |
fca11ebd SK |
330 | } |
331 | ||
4b549ccc CL |
332 | if (xo->seq.low < seq) |
333 | xo->seq.hi++; | |
334 | ||
3dca3f38 SK |
335 | esp.seqno = cpu_to_be64(seq + ((u64)xo->seq.hi << 32)); |
336 | ||
447bc4b1 MY |
337 | if (hw_offload && encap_type == UDP_ENCAP_ESPINUDP) { |
338 | /* In the XFRM stack, the encapsulation protocol is set to iphdr->protocol by | |
339 | * setting *skb_mac_header(skb) (see esp_output_udp_encap()) where skb->mac_header | |
340 | * points to iphdr->protocol (see xfrm4_tunnel_encap_add()). | |
341 | * However, in esp_xmit(), skb->mac_header doesn't point to iphdr->protocol. | |
342 | * Therefore, the protocol field needs to be corrected. | |
343 | */ | |
344 | ip_hdr(skb)->protocol = IPPROTO_UDP; | |
345 | ||
346 | esph->seq_no = htonl(seq); | |
347 | } | |
348 | ||
3dca3f38 SK |
349 | ip_hdr(skb)->tot_len = htons(skb->len); |
350 | ip_send_check(ip_hdr(skb)); | |
351 | ||
c7dbf4c0 SK |
352 | if (hw_offload) { |
353 | if (!skb_ext_add(skb, SKB_EXT_SEC_PATH)) | |
354 | return -ENOMEM; | |
355 | ||
356 | xo = xfrm_offload(skb); | |
357 | if (!xo) | |
358 | return -EINVAL; | |
359 | ||
360 | xo->flags |= XFRM_XMIT; | |
fca11ebd | 361 | return 0; |
c7dbf4c0 | 362 | } |
fca11ebd | 363 | |
fca11ebd | 364 | err = esp_output_tail(x, skb, &esp); |
4ff0308f | 365 | if (err) |
fca11ebd SK |
366 | return err; |
367 | ||
368 | secpath_reset(skb); | |
369 | ||
f015b900 SAS |
370 | if (skb_needs_linearize(skb, skb->dev->features) && |
371 | __skb_linearize(skb)) | |
372 | return -ENOMEM; | |
fca11ebd SK |
373 | return 0; |
374 | } | |
375 | ||
7785bba2 SK |
376 | static const struct net_offload esp4_offload = { |
377 | .callbacks = { | |
378 | .gro_receive = esp4_gro_receive, | |
7862b405 | 379 | .gso_segment = esp4_gso_segment, |
7785bba2 SK |
380 | }, |
381 | }; | |
382 | ||
fca11ebd | 383 | static const struct xfrm_type_offload esp_type_offload = { |
fca11ebd SK |
384 | .owner = THIS_MODULE, |
385 | .proto = IPPROTO_ESP, | |
386 | .input_tail = esp_input_tail, | |
387 | .xmit = esp_xmit, | |
7862b405 | 388 | .encap = esp4_gso_encap, |
fca11ebd SK |
389 | }; |
390 | ||
7785bba2 SK |
391 | static int __init esp4_offload_init(void) |
392 | { | |
fca11ebd SK |
393 | if (xfrm_register_type_offload(&esp_type_offload, AF_INET) < 0) { |
394 | pr_info("%s: can't add xfrm type offload\n", __func__); | |
395 | return -EAGAIN; | |
396 | } | |
397 | ||
7785bba2 SK |
398 | return inet_add_offload(&esp4_offload, IPPROTO_ESP); |
399 | } | |
400 | ||
401 | static void __exit esp4_offload_exit(void) | |
402 | { | |
4f518e80 | 403 | xfrm_unregister_type_offload(&esp_type_offload, AF_INET); |
7785bba2 SK |
404 | inet_del_offload(&esp4_offload, IPPROTO_ESP); |
405 | } | |
406 | ||
407 | module_init(esp4_offload_init); | |
408 | module_exit(esp4_offload_exit); | |
409 | MODULE_LICENSE("GPL"); | |
410 | MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); | |
ffdb5211 | 411 | MODULE_ALIAS_XFRM_OFFLOAD_TYPE(AF_INET, XFRM_PROTO_ESP); |
67c20de3 | 412 | MODULE_DESCRIPTION("IPV4 GSO/GRO offload support"); |