Commit | Line | Data |
---|---|---|
21f42cc9 SK |
1 | /* |
2 | * xfrm_device.c - IPsec device offloading code. | |
3 | * | |
4 | * Copyright (c) 2015 secunet Security Networks AG | |
5 | * | |
6 | * Author: | |
7 | * Steffen Klassert <steffen.klassert@secunet.com> | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU General Public License | |
11 | * as published by the Free Software Foundation; either version | |
12 | * 2 of the License, or (at your option) any later version. | |
13 | */ | |
14 | ||
15 | #include <linux/errno.h> | |
16 | #include <linux/module.h> | |
17 | #include <linux/netdevice.h> | |
18 | #include <linux/skbuff.h> | |
19 | #include <linux/slab.h> | |
20 | #include <linux/spinlock.h> | |
21 | #include <net/dst.h> | |
22 | #include <net/xfrm.h> | |
23 | #include <linux/notifier.h> | |
24 | ||
b81f884a | 25 | #ifdef CONFIG_XFRM_OFFLOAD |
f53c7239 | 26 | struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again) |
f6e27114 SK |
27 | { |
28 | int err; | |
f53c7239 | 29 | unsigned long flags; |
f6e27114 | 30 | struct xfrm_state *x; |
3dca3f38 | 31 | struct sk_buff *skb2; |
f53c7239 | 32 | struct softnet_data *sd; |
3dca3f38 | 33 | netdev_features_t esp_features = features; |
f6e27114 SK |
34 | struct xfrm_offload *xo = xfrm_offload(skb); |
35 | ||
3dca3f38 SK |
36 | if (!xo) |
37 | return skb; | |
f6e27114 | 38 | |
3dca3f38 SK |
39 | if (!(features & NETIF_F_HW_ESP)) |
40 | esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK); | |
f6e27114 | 41 | |
3dca3f38 SK |
42 | x = skb->sp->xvec[skb->sp->len - 1]; |
43 | if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND) | |
44 | return skb; | |
45 | ||
f53c7239 SK |
46 | local_irq_save(flags); |
47 | sd = this_cpu_ptr(&softnet_data); | |
48 | err = !skb_queue_empty(&sd->xfrm_backlog); | |
49 | local_irq_restore(flags); | |
50 | ||
51 | if (err) { | |
52 | *again = true; | |
53 | return skb; | |
54 | } | |
55 | ||
3dca3f38 SK |
56 | if (skb_is_gso(skb)) { |
57 | struct net_device *dev = skb->dev; | |
58 | ||
59 | if (unlikely(!x->xso.offload_handle || (x->xso.dev != dev))) { | |
60 | struct sk_buff *segs; | |
61 | ||
62 | /* Packet got rerouted, fixup features and segment it. */ | |
63 | esp_features = esp_features & ~(NETIF_F_HW_ESP | |
64 | | NETIF_F_GSO_ESP); | |
65 | ||
66 | segs = skb_gso_segment(skb, esp_features); | |
67 | if (IS_ERR(segs)) { | |
3dca3f38 | 68 | kfree_skb(skb); |
f53c7239 | 69 | atomic_long_inc(&dev->tx_dropped); |
3dca3f38 SK |
70 | return NULL; |
71 | } else { | |
72 | consume_skb(skb); | |
73 | skb = segs; | |
74 | } | |
3dca3f38 SK |
75 | } |
76 | } | |
77 | ||
78 | if (!skb->next) { | |
f6e27114 SK |
79 | x->outer_mode->xmit(x, skb); |
80 | ||
f53c7239 SK |
81 | xo->flags |= XFRM_DEV_RESUME; |
82 | ||
3dca3f38 | 83 | err = x->type_offload->xmit(x, skb, esp_features); |
f6e27114 | 84 | if (err) { |
f53c7239 SK |
85 | if (err == -EINPROGRESS) |
86 | return NULL; | |
87 | ||
f6e27114 | 88 | XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); |
3dca3f38 SK |
89 | kfree_skb(skb); |
90 | return NULL; | |
f6e27114 SK |
91 | } |
92 | ||
93 | skb_push(skb, skb->data - skb_mac_header(skb)); | |
3dca3f38 SK |
94 | |
95 | return skb; | |
f6e27114 SK |
96 | } |
97 | ||
3dca3f38 | 98 | skb2 = skb; |
3dca3f38 SK |
99 | |
100 | do { | |
101 | struct sk_buff *nskb = skb2->next; | |
f53c7239 | 102 | skb2->next = NULL; |
3dca3f38 SK |
103 | |
104 | xo = xfrm_offload(skb2); | |
f53c7239 | 105 | xo->flags |= XFRM_DEV_RESUME; |
3dca3f38 SK |
106 | |
107 | x->outer_mode->xmit(x, skb2); | |
108 | ||
109 | err = x->type_offload->xmit(x, skb2, esp_features); | |
f53c7239 SK |
110 | if (!err) { |
111 | skb2->next = nskb; | |
112 | } else if (err != -EINPROGRESS) { | |
3dca3f38 SK |
113 | XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); |
114 | skb2->next = nskb; | |
115 | kfree_skb_list(skb2); | |
116 | return NULL; | |
f53c7239 SK |
117 | } else { |
118 | if (skb == skb2) | |
119 | skb = nskb; | |
120 | ||
121 | if (!skb) | |
122 | return NULL; | |
3dca3f38 | 123 | |
f53c7239 SK |
124 | goto skip_push; |
125 | } | |
3dca3f38 SK |
126 | |
127 | skb_push(skb2, skb2->data - skb_mac_header(skb2)); | |
128 | ||
f53c7239 | 129 | skip_push: |
3dca3f38 SK |
130 | skb2 = nskb; |
131 | } while (skb2); | |
132 | ||
133 | return skb; | |
f6e27114 SK |
134 | } |
135 | EXPORT_SYMBOL_GPL(validate_xmit_xfrm); | |
136 | ||
d77e38e6 SK |
137 | int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, |
138 | struct xfrm_user_offload *xuo) | |
139 | { | |
140 | int err; | |
141 | struct dst_entry *dst; | |
142 | struct net_device *dev; | |
143 | struct xfrm_state_offload *xso = &x->xso; | |
144 | xfrm_address_t *saddr; | |
145 | xfrm_address_t *daddr; | |
146 | ||
147 | if (!x->type_offload) | |
ffdb5211 | 148 | return -EINVAL; |
d77e38e6 | 149 | |
50bd870a YE |
150 | /* We don't yet support UDP encapsulation and TFC padding. */ |
151 | if (x->encap || x->tfcpad) | |
43024b9c | 152 | return -EINVAL; |
d77e38e6 SK |
153 | |
154 | dev = dev_get_by_index(net, xuo->ifindex); | |
155 | if (!dev) { | |
156 | if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) { | |
157 | saddr = &x->props.saddr; | |
158 | daddr = &x->id.daddr; | |
159 | } else { | |
160 | saddr = &x->id.daddr; | |
161 | daddr = &x->props.saddr; | |
162 | } | |
163 | ||
077fbac4 LC |
164 | dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr, |
165 | x->props.family, x->props.output_mark); | |
d77e38e6 SK |
166 | if (IS_ERR(dst)) |
167 | return 0; | |
168 | ||
169 | dev = dst->dev; | |
170 | ||
171 | dev_hold(dev); | |
172 | dst_release(dst); | |
173 | } | |
174 | ||
175 | if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) { | |
67a63387 | 176 | xso->dev = NULL; |
d77e38e6 SK |
177 | dev_put(dev); |
178 | return 0; | |
179 | } | |
180 | ||
50bd870a YE |
181 | if (x->props.flags & XFRM_STATE_ESN && |
182 | !dev->xfrmdev_ops->xdo_dev_state_advance_esn) { | |
183 | xso->dev = NULL; | |
184 | dev_put(dev); | |
185 | return -EINVAL; | |
186 | } | |
187 | ||
d77e38e6 SK |
188 | xso->dev = dev; |
189 | xso->num_exthdrs = 1; | |
190 | xso->flags = xuo->flags; | |
191 | ||
192 | err = dev->xfrmdev_ops->xdo_dev_state_add(x); | |
193 | if (err) { | |
aa5dd6fa | 194 | xso->dev = NULL; |
d77e38e6 SK |
195 | dev_put(dev); |
196 | return err; | |
197 | } | |
198 | ||
199 | return 0; | |
200 | } | |
201 | EXPORT_SYMBOL_GPL(xfrm_dev_state_add); | |
202 | ||
203 | bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x) | |
204 | { | |
205 | int mtu; | |
206 | struct dst_entry *dst = skb_dst(skb); | |
207 | struct xfrm_dst *xdst = (struct xfrm_dst *)dst; | |
208 | struct net_device *dev = x->xso.dev; | |
209 | ||
210 | if (!x->type_offload || x->encap) | |
211 | return false; | |
212 | ||
95bff4b5 SK |
213 | if ((!dev || (x->xso.offload_handle && (dev == xfrm_dst_path(dst)->dev))) && |
214 | (!xdst->child->xfrm && x->type->get_mtu)) { | |
d77e38e6 SK |
215 | mtu = x->type->get_mtu(x, xdst->child_mtu_cached); |
216 | ||
217 | if (skb->len <= mtu) | |
218 | goto ok; | |
219 | ||
779b7931 | 220 | if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) |
d77e38e6 SK |
221 | goto ok; |
222 | } | |
223 | ||
224 | return false; | |
225 | ||
226 | ok: | |
227 | if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok) | |
228 | return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x); | |
229 | ||
230 | return true; | |
231 | } | |
232 | EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok); | |
f53c7239 SK |
233 | |
234 | void xfrm_dev_resume(struct sk_buff *skb) | |
235 | { | |
236 | struct net_device *dev = skb->dev; | |
237 | int ret = NETDEV_TX_BUSY; | |
238 | struct netdev_queue *txq; | |
239 | struct softnet_data *sd; | |
240 | unsigned long flags; | |
241 | ||
242 | rcu_read_lock(); | |
243 | txq = netdev_pick_tx(dev, skb, NULL); | |
244 | ||
245 | HARD_TX_LOCK(dev, txq, smp_processor_id()); | |
246 | if (!netif_xmit_frozen_or_stopped(txq)) | |
247 | skb = dev_hard_start_xmit(skb, dev, txq, &ret); | |
248 | HARD_TX_UNLOCK(dev, txq); | |
249 | ||
250 | if (!dev_xmit_complete(ret)) { | |
251 | local_irq_save(flags); | |
252 | sd = this_cpu_ptr(&softnet_data); | |
253 | skb_queue_tail(&sd->xfrm_backlog, skb); | |
254 | raise_softirq_irqoff(NET_TX_SOFTIRQ); | |
255 | local_irq_restore(flags); | |
256 | } | |
257 | rcu_read_unlock(); | |
258 | } | |
259 | EXPORT_SYMBOL_GPL(xfrm_dev_resume); | |
260 | ||
261 | void xfrm_dev_backlog(struct softnet_data *sd) | |
262 | { | |
263 | struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog; | |
264 | struct sk_buff_head list; | |
265 | struct sk_buff *skb; | |
266 | ||
267 | if (skb_queue_empty(xfrm_backlog)) | |
268 | return; | |
269 | ||
270 | __skb_queue_head_init(&list); | |
271 | ||
272 | spin_lock(&xfrm_backlog->lock); | |
273 | skb_queue_splice_init(xfrm_backlog, &list); | |
274 | spin_unlock(&xfrm_backlog->lock); | |
275 | ||
276 | while (!skb_queue_empty(&list)) { | |
277 | skb = __skb_dequeue(&list); | |
278 | xfrm_dev_resume(skb); | |
279 | } | |
280 | ||
281 | } | |
b81f884a | 282 | #endif |
d77e38e6 | 283 | |
92a23206 | 284 | static int xfrm_api_check(struct net_device *dev) |
d77e38e6 | 285 | { |
92a23206 | 286 | #ifdef CONFIG_XFRM_OFFLOAD |
d77e38e6 SK |
287 | if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) && |
288 | !(dev->features & NETIF_F_HW_ESP)) | |
289 | return NOTIFY_BAD; | |
290 | ||
92a23206 SN |
291 | if ((dev->features & NETIF_F_HW_ESP) && |
292 | (!(dev->xfrmdev_ops && | |
293 | dev->xfrmdev_ops->xdo_dev_state_add && | |
294 | dev->xfrmdev_ops->xdo_dev_state_delete))) | |
295 | return NOTIFY_BAD; | |
296 | #else | |
297 | if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM)) | |
298 | return NOTIFY_BAD; | |
299 | #endif | |
300 | ||
d77e38e6 SK |
301 | return NOTIFY_DONE; |
302 | } | |
303 | ||
92a23206 SN |
304 | static int xfrm_dev_register(struct net_device *dev) |
305 | { | |
306 | return xfrm_api_check(dev); | |
307 | } | |
308 | ||
d77e38e6 SK |
309 | static int xfrm_dev_unregister(struct net_device *dev) |
310 | { | |
ec30d78c | 311 | xfrm_policy_cache_flush(); |
d77e38e6 SK |
312 | return NOTIFY_DONE; |
313 | } | |
314 | ||
315 | static int xfrm_dev_feat_change(struct net_device *dev) | |
316 | { | |
92a23206 | 317 | return xfrm_api_check(dev); |
d77e38e6 SK |
318 | } |
319 | ||
320 | static int xfrm_dev_down(struct net_device *dev) | |
321 | { | |
2c1497bb | 322 | if (dev->features & NETIF_F_HW_ESP) |
d77e38e6 SK |
323 | xfrm_dev_state_flush(dev_net(dev), dev, true); |
324 | ||
ec30d78c | 325 | xfrm_policy_cache_flush(); |
d77e38e6 SK |
326 | return NOTIFY_DONE; |
327 | } | |
328 | ||
21f42cc9 SK |
329 | static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr) |
330 | { | |
331 | struct net_device *dev = netdev_notifier_info_to_dev(ptr); | |
332 | ||
333 | switch (event) { | |
d77e38e6 SK |
334 | case NETDEV_REGISTER: |
335 | return xfrm_dev_register(dev); | |
336 | ||
337 | case NETDEV_UNREGISTER: | |
338 | return xfrm_dev_unregister(dev); | |
339 | ||
340 | case NETDEV_FEAT_CHANGE: | |
341 | return xfrm_dev_feat_change(dev); | |
342 | ||
21f42cc9 | 343 | case NETDEV_DOWN: |
d77e38e6 | 344 | return xfrm_dev_down(dev); |
21f42cc9 SK |
345 | } |
346 | return NOTIFY_DONE; | |
347 | } | |
348 | ||
349 | static struct notifier_block xfrm_dev_notifier = { | |
350 | .notifier_call = xfrm_dev_event, | |
351 | }; | |
352 | ||
353 | void __net_init xfrm_dev_init(void) | |
354 | { | |
355 | register_netdevice_notifier(&xfrm_dev_notifier); | |
356 | } |