Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a716c119 | 2 | /* |
1da177e4 LT |
3 | * xfrm_policy.c |
4 | * | |
5 | * Changes: | |
6 | * Mitsuru KANDA @USAGI | |
7 | * Kazunori MIYAZAWA @USAGI | |
8 | * Kunihiro Ishiguro <kunihiro@ipinfusion.com> | |
9 | * IPv6 support | |
10 | * Kazunori MIYAZAWA @USAGI | |
11 | * YOSHIFUJI Hideaki | |
12 | * Split up af-specific portion | |
13 | * Derek Atkins <derek@ihtfp.com> Add the post_input processor | |
df71837d | 14 | * |
1da177e4 LT |
15 | */ |
16 | ||
66cdb3ca | 17 | #include <linux/err.h> |
1da177e4 LT |
18 | #include <linux/slab.h> |
19 | #include <linux/kmod.h> | |
20 | #include <linux/list.h> | |
21 | #include <linux/spinlock.h> | |
22 | #include <linux/workqueue.h> | |
23 | #include <linux/notifier.h> | |
24 | #include <linux/netdevice.h> | |
eb9c7ebe | 25 | #include <linux/netfilter.h> |
1da177e4 | 26 | #include <linux/module.h> |
2518c7c2 | 27 | #include <linux/cache.h> |
ec30d78c | 28 | #include <linux/cpu.h> |
68277acc | 29 | #include <linux/audit.h> |
24969fac | 30 | #include <linux/rhashtable.h> |
c53ac41e | 31 | #include <linux/if_tunnel.h> |
25ee3286 | 32 | #include <net/dst.h> |
6ce74ec7 | 33 | #include <net/flow.h> |
23e7b1bf | 34 | #include <net/inet_ecn.h> |
1da177e4 LT |
35 | #include <net/xfrm.h> |
36 | #include <net/ip.h> | |
bcf141b2 | 37 | #include <net/gre.h> |
c53ac41e FW |
38 | #if IS_ENABLED(CONFIG_IPV6_MIP6) |
39 | #include <net/mip6.h> | |
40 | #endif | |
558f82ef MN |
41 | #ifdef CONFIG_XFRM_STATISTICS |
42 | #include <net/snmp.h> | |
43 | #endif | |
95a35b42 | 44 | #ifdef CONFIG_XFRM_ESPINTCP |
e27cca96 SD |
45 | #include <net/espintcp.h> |
46 | #endif | |
1da177e4 | 47 | |
44e36b42 DM |
48 | #include "xfrm_hash.h" |
49 | ||
a0073fe1 SK |
50 | #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10)) |
51 | #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ)) | |
52 | #define XFRM_MAX_QUEUE_LEN 100 | |
53 | ||
b8c203b2 SK |
54 | struct xfrm_flo { |
55 | struct dst_entry *dst_orig; | |
56 | u8 flags; | |
57 | }; | |
58 | ||
6be3b0db FW |
59 | /* prefixes smaller than this are stored in lists, not trees. */ |
60 | #define INEXACT_PREFIXLEN_IPV4 16 | |
61 | #define INEXACT_PREFIXLEN_IPV6 48 | |
9cf545eb FW |
62 | |
63 | struct xfrm_pol_inexact_node { | |
64 | struct rb_node node; | |
65 | union { | |
66 | xfrm_address_t addr; | |
67 | struct rcu_head rcu; | |
68 | }; | |
69 | u8 prefixlen; | |
70 | ||
6ac098b2 FW |
71 | struct rb_root root; |
72 | ||
9cf545eb FW |
73 | /* the policies matching this node, can be empty list */ |
74 | struct hlist_head hhead; | |
75 | }; | |
76 | ||
77 | /* xfrm inexact policy search tree: | |
78 | * xfrm_pol_inexact_bin = hash(dir,type,family,if_id); | |
79 | * | | |
80 | * +---- root_d: sorted by daddr:prefix | |
81 | * | | | |
82 | * | xfrm_pol_inexact_node | |
83 | * | | | |
6ac098b2 FW |
84 | * | +- root: sorted by saddr/prefix |
85 | * | | | | |
86 | * | | xfrm_pol_inexact_node | |
87 | * | | | | |
88 | * | | + root: unused | |
89 | * | | | | |
90 | * | | + hhead: saddr:daddr policies | |
91 | * | | | |
9cf545eb FW |
92 | * | +- coarse policies and all any:daddr policies |
93 | * | | |
64a09a7b FW |
94 | * +---- root_s: sorted by saddr:prefix |
95 | * | | | |
96 | * | xfrm_pol_inexact_node | |
97 | * | | | |
98 | * | + root: unused | |
99 | * | | | |
100 | * | + hhead: saddr:any policies | |
101 | * | | |
9cf545eb FW |
102 | * +---- coarse policies and all any:any policies |
103 | * | |
6ac098b2 | 104 | * Lookups return four candidate lists: |
9cf545eb FW |
105 | * 1. any:any list from top-level xfrm_pol_inexact_bin |
106 | * 2. any:daddr list from daddr tree | |
6ac098b2 FW |
107 | * 3. saddr:daddr list from 2nd level daddr tree |
108 | * 4. saddr:any list from saddr tree | |
9cf545eb FW |
109 | * |
110 | * This result set then needs to be searched for the policy with | |
111 | * the lowest priority. If two results have same prio, youngest one wins. | |
112 | */ | |
113 | ||
24969fac FW |
114 | struct xfrm_pol_inexact_key { |
115 | possible_net_t net; | |
b5fe22e2 | 116 | u32 if_id; |
24969fac FW |
117 | u16 family; |
118 | u8 dir, type; | |
119 | }; | |
120 | ||
121 | struct xfrm_pol_inexact_bin { | |
122 | struct xfrm_pol_inexact_key k; | |
123 | struct rhash_head head; | |
6be3b0db | 124 | /* list containing '*:*' policies */ |
24969fac FW |
125 | struct hlist_head hhead; |
126 | ||
77cc278f | 127 | seqcount_spinlock_t count; |
9cf545eb FW |
128 | /* tree sorted by daddr/prefix */ |
129 | struct rb_root root_d; | |
130 | ||
64a09a7b FW |
131 | /* tree sorted by saddr/prefix */ |
132 | struct rb_root root_s; | |
133 | ||
24969fac FW |
134 | /* slow path below */ |
135 | struct list_head inexact_bins; | |
136 | struct rcu_head rcu; | |
137 | }; | |
138 | ||
6be3b0db | 139 | enum xfrm_pol_inexact_candidate_type { |
6ac098b2 | 140 | XFRM_POL_CAND_BOTH, |
64a09a7b | 141 | XFRM_POL_CAND_SADDR, |
9cf545eb | 142 | XFRM_POL_CAND_DADDR, |
6be3b0db FW |
143 | XFRM_POL_CAND_ANY, |
144 | ||
145 | XFRM_POL_CAND_MAX, | |
146 | }; | |
147 | ||
148 | struct xfrm_pol_inexact_candidates { | |
149 | struct hlist_head *res[XFRM_POL_CAND_MAX]; | |
150 | }; | |
151 | ||
f203b76d SK |
152 | static DEFINE_SPINLOCK(xfrm_if_cb_lock); |
153 | static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly; | |
154 | ||
418a99ac | 155 | static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock); |
37b10383 | 156 | static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1] |
418a99ac | 157 | __read_mostly; |
1da177e4 | 158 | |
f8c3d0dd | 159 | static struct kmem_cache *xfrm_dst_cache __ro_after_init; |
1da177e4 | 160 | |
24969fac FW |
161 | static struct rhashtable xfrm_policy_inexact_table; |
162 | static const struct rhashtable_params xfrm_pol_inexact_params; | |
163 | ||
5492093d | 164 | static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr); |
80c802f3 | 165 | static int stale_bundle(struct dst_entry *dst); |
12fdb4d3 | 166 | static int xfrm_bundle_ok(struct xfrm_dst *xdst); |
c3aed709 | 167 | static void xfrm_policy_queue_process(struct timer_list *t); |
1da177e4 | 168 | |
12bfa8bd | 169 | static void __xfrm_policy_link(struct xfrm_policy *pol, int dir); |
29fa0b30 WY |
170 | static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, |
171 | int dir); | |
172 | ||
24969fac | 173 | static struct xfrm_pol_inexact_bin * |
b5fe22e2 FW |
174 | xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir, |
175 | u32 if_id); | |
24969fac FW |
176 | |
177 | static struct xfrm_pol_inexact_bin * | |
178 | xfrm_policy_inexact_lookup_rcu(struct net *net, | |
b5fe22e2 | 179 | u8 type, u16 family, u8 dir, u32 if_id); |
24969fac FW |
180 | static struct xfrm_policy * |
181 | xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy, | |
182 | bool excl); | |
183 | static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, | |
184 | struct xfrm_policy *policy); | |
185 | ||
6be3b0db FW |
186 | static bool |
187 | xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand, | |
188 | struct xfrm_pol_inexact_bin *b, | |
189 | const xfrm_address_t *saddr, | |
190 | const xfrm_address_t *daddr); | |
191 | ||
e37cc8ad FW |
192 | static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy) |
193 | { | |
850a6212 | 194 | return refcount_inc_not_zero(&policy->refcnt); |
e37cc8ad FW |
195 | } |
196 | ||
bc9b35ad | 197 | static inline bool |
200ce96e | 198 | __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) |
77681021 | 199 | { |
7e1dc7b6 DM |
200 | const struct flowi4 *fl4 = &fl->u.ip4; |
201 | ||
26bff940 AD |
202 | return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) && |
203 | addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) && | |
7e1dc7b6 DM |
204 | !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) && |
205 | !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) && | |
206 | (fl4->flowi4_proto == sel->proto || !sel->proto) && | |
207 | (fl4->flowi4_oif == sel->ifindex || !sel->ifindex); | |
77681021 AM |
208 | } |
209 | ||
bc9b35ad | 210 | static inline bool |
200ce96e | 211 | __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl) |
77681021 | 212 | { |
7e1dc7b6 DM |
213 | const struct flowi6 *fl6 = &fl->u.ip6; |
214 | ||
215 | return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) && | |
216 | addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) && | |
217 | !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) && | |
218 | !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) && | |
219 | (fl6->flowi6_proto == sel->proto || !sel->proto) && | |
220 | (fl6->flowi6_oif == sel->ifindex || !sel->ifindex); | |
77681021 AM |
221 | } |
222 | ||
bc9b35ad DM |
223 | bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl, |
224 | unsigned short family) | |
77681021 AM |
225 | { |
226 | switch (family) { | |
227 | case AF_INET: | |
228 | return __xfrm4_selector_match(sel, fl); | |
229 | case AF_INET6: | |
230 | return __xfrm6_selector_match(sel, fl); | |
231 | } | |
bc9b35ad | 232 | return false; |
77681021 AM |
233 | } |
234 | ||
a2817d8b | 235 | static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family) |
ef8531b6 | 236 | { |
a2817d8b | 237 | const struct xfrm_policy_afinfo *afinfo; |
ef8531b6 | 238 | |
a2817d8b | 239 | if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo))) |
ef8531b6 ED |
240 | return NULL; |
241 | rcu_read_lock(); | |
242 | afinfo = rcu_dereference(xfrm_policy_afinfo[family]); | |
243 | if (unlikely(!afinfo)) | |
244 | rcu_read_unlock(); | |
245 | return afinfo; | |
246 | } | |
247 | ||
f203b76d SK |
248 | /* Called with rcu_read_lock(). */ |
249 | static const struct xfrm_if_cb *xfrm_if_get_cb(void) | |
250 | { | |
251 | return rcu_dereference(xfrm_if_cb); | |
252 | } | |
253 | ||
d77e38e6 SK |
254 | struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif, |
255 | const xfrm_address_t *saddr, | |
256 | const xfrm_address_t *daddr, | |
077fbac4 | 257 | int family, u32 mark) |
9bb182a7 | 258 | { |
37b10383 | 259 | const struct xfrm_policy_afinfo *afinfo; |
9bb182a7 YH |
260 | struct dst_entry *dst; |
261 | ||
262 | afinfo = xfrm_policy_get_afinfo(family); | |
263 | if (unlikely(afinfo == NULL)) | |
264 | return ERR_PTR(-EAFNOSUPPORT); | |
265 | ||
077fbac4 | 266 | dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark); |
9bb182a7 | 267 | |
bdba9fe0 | 268 | rcu_read_unlock(); |
9bb182a7 YH |
269 | |
270 | return dst; | |
271 | } | |
d77e38e6 | 272 | EXPORT_SYMBOL(__xfrm_dst_lookup); |
9bb182a7 | 273 | |
42a7b32b DA |
274 | static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, |
275 | int tos, int oif, | |
9bb182a7 YH |
276 | xfrm_address_t *prev_saddr, |
277 | xfrm_address_t *prev_daddr, | |
077fbac4 | 278 | int family, u32 mark) |
1da177e4 | 279 | { |
c5b3cf46 | 280 | struct net *net = xs_net(x); |
66cdb3ca HX |
281 | xfrm_address_t *saddr = &x->props.saddr; |
282 | xfrm_address_t *daddr = &x->id.daddr; | |
66cdb3ca HX |
283 | struct dst_entry *dst; |
284 | ||
9bb182a7 | 285 | if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) { |
66cdb3ca | 286 | saddr = x->coaddr; |
9bb182a7 YH |
287 | daddr = prev_daddr; |
288 | } | |
289 | if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) { | |
290 | saddr = prev_saddr; | |
66cdb3ca | 291 | daddr = x->coaddr; |
9bb182a7 | 292 | } |
1da177e4 | 293 | |
077fbac4 | 294 | dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark); |
9bb182a7 YH |
295 | |
296 | if (!IS_ERR(dst)) { | |
297 | if (prev_saddr != saddr) | |
298 | memcpy(prev_saddr, saddr, sizeof(*prev_saddr)); | |
299 | if (prev_daddr != daddr) | |
300 | memcpy(prev_daddr, daddr, sizeof(*prev_daddr)); | |
301 | } | |
1da177e4 | 302 | |
66cdb3ca | 303 | return dst; |
1da177e4 | 304 | } |
1da177e4 | 305 | |
1da177e4 LT |
306 | static inline unsigned long make_jiffies(long secs) |
307 | { | |
308 | if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ) | |
309 | return MAX_SCHEDULE_TIMEOUT-1; | |
310 | else | |
a716c119 | 311 | return secs*HZ; |
1da177e4 LT |
312 | } |
313 | ||
c3aed709 | 314 | static void xfrm_policy_timer(struct timer_list *t) |
1da177e4 | 315 | { |
c3aed709 | 316 | struct xfrm_policy *xp = from_timer(xp, t, timer); |
386c5680 AB |
317 | time64_t now = ktime_get_real_seconds(); |
318 | time64_t next = TIME64_MAX; | |
1da177e4 LT |
319 | int warn = 0; |
320 | int dir; | |
321 | ||
322 | read_lock(&xp->lock); | |
323 | ||
ea2dea9d | 324 | if (unlikely(xp->walk.dead)) |
1da177e4 LT |
325 | goto out; |
326 | ||
77d8d7a6 | 327 | dir = xfrm_policy_id2dir(xp->index); |
1da177e4 LT |
328 | |
329 | if (xp->lft.hard_add_expires_seconds) { | |
386c5680 | 330 | time64_t tmo = xp->lft.hard_add_expires_seconds + |
1da177e4 LT |
331 | xp->curlft.add_time - now; |
332 | if (tmo <= 0) | |
333 | goto expired; | |
334 | if (tmo < next) | |
335 | next = tmo; | |
336 | } | |
337 | if (xp->lft.hard_use_expires_seconds) { | |
386c5680 | 338 | time64_t tmo = xp->lft.hard_use_expires_seconds + |
0a9e5794 | 339 | (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; |
1da177e4 LT |
340 | if (tmo <= 0) |
341 | goto expired; | |
342 | if (tmo < next) | |
343 | next = tmo; | |
344 | } | |
345 | if (xp->lft.soft_add_expires_seconds) { | |
386c5680 | 346 | time64_t tmo = xp->lft.soft_add_expires_seconds + |
1da177e4 LT |
347 | xp->curlft.add_time - now; |
348 | if (tmo <= 0) { | |
349 | warn = 1; | |
350 | tmo = XFRM_KM_TIMEOUT; | |
351 | } | |
352 | if (tmo < next) | |
353 | next = tmo; | |
354 | } | |
355 | if (xp->lft.soft_use_expires_seconds) { | |
386c5680 | 356 | time64_t tmo = xp->lft.soft_use_expires_seconds + |
0a9e5794 | 357 | (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now; |
1da177e4 LT |
358 | if (tmo <= 0) { |
359 | warn = 1; | |
360 | tmo = XFRM_KM_TIMEOUT; | |
361 | } | |
362 | if (tmo < next) | |
363 | next = tmo; | |
364 | } | |
365 | ||
366 | if (warn) | |
6c5c8ca7 | 367 | km_policy_expired(xp, dir, 0, 0); |
386c5680 | 368 | if (next != TIME64_MAX && |
1da177e4 LT |
369 | !mod_timer(&xp->timer, jiffies + make_jiffies(next))) |
370 | xfrm_pol_hold(xp); | |
371 | ||
372 | out: | |
373 | read_unlock(&xp->lock); | |
374 | xfrm_pol_put(xp); | |
375 | return; | |
376 | ||
377 | expired: | |
378 | read_unlock(&xp->lock); | |
4666faab | 379 | if (!xfrm_policy_delete(xp, dir)) |
6c5c8ca7 | 380 | km_policy_expired(xp, dir, 1, 0); |
1da177e4 LT |
381 | xfrm_pol_put(xp); |
382 | } | |
383 | ||
1da177e4 LT |
384 | /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2 |
385 | * SPD calls. | |
386 | */ | |
387 | ||
0331b1f3 | 388 | struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp) |
1da177e4 LT |
389 | { |
390 | struct xfrm_policy *policy; | |
391 | ||
0da974f4 | 392 | policy = kzalloc(sizeof(struct xfrm_policy), gfp); |
1da177e4 LT |
393 | |
394 | if (policy) { | |
0331b1f3 | 395 | write_pnet(&policy->xp_net, net); |
12a169e7 | 396 | INIT_LIST_HEAD(&policy->walk.all); |
24969fac | 397 | INIT_HLIST_NODE(&policy->bydst_inexact_list); |
2518c7c2 DM |
398 | INIT_HLIST_NODE(&policy->bydst); |
399 | INIT_HLIST_NODE(&policy->byidx); | |
1da177e4 | 400 | rwlock_init(&policy->lock); |
850a6212 | 401 | refcount_set(&policy->refcnt, 1); |
a0073fe1 | 402 | skb_queue_head_init(&policy->polq.hold_queue); |
c3aed709 KC |
403 | timer_setup(&policy->timer, xfrm_policy_timer, 0); |
404 | timer_setup(&policy->polq.hold_timer, | |
405 | xfrm_policy_queue_process, 0); | |
1da177e4 LT |
406 | } |
407 | return policy; | |
408 | } | |
409 | EXPORT_SYMBOL(xfrm_policy_alloc); | |
410 | ||
56f04730 ED |
411 | static void xfrm_policy_destroy_rcu(struct rcu_head *head) |
412 | { | |
413 | struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu); | |
414 | ||
415 | security_xfrm_policy_free(policy->security); | |
416 | kfree(policy); | |
417 | } | |
418 | ||
1da177e4 LT |
419 | /* Destroy xfrm_policy: descendant resources must be released to this moment. */ |
420 | ||
64c31b3f | 421 | void xfrm_policy_destroy(struct xfrm_policy *policy) |
1da177e4 | 422 | { |
12a169e7 | 423 | BUG_ON(!policy->walk.dead); |
1da177e4 | 424 | |
0659eea9 | 425 | if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer)) |
1da177e4 LT |
426 | BUG(); |
427 | ||
919e43fa | 428 | xfrm_dev_policy_free(policy); |
56f04730 | 429 | call_rcu(&policy->rcu, xfrm_policy_destroy_rcu); |
1da177e4 | 430 | } |
64c31b3f | 431 | EXPORT_SYMBOL(xfrm_policy_destroy); |
1da177e4 | 432 | |
1365e547 | 433 | /* Rule must be locked. Release descendant resources, announce |
1da177e4 LT |
434 | * entry dead. The rule must be unlinked from lists to the moment. |
435 | */ | |
436 | ||
437 | static void xfrm_policy_kill(struct xfrm_policy *policy) | |
438 | { | |
4c59406e | 439 | write_lock_bh(&policy->lock); |
12a169e7 | 440 | policy->walk.dead = 1; |
4c59406e | 441 | write_unlock_bh(&policy->lock); |
1da177e4 | 442 | |
285ead17 | 443 | atomic_inc(&policy->genid); |
1da177e4 | 444 | |
e7d8f6cb SK |
445 | if (del_timer(&policy->polq.hold_timer)) |
446 | xfrm_pol_put(policy); | |
1ee5e667 | 447 | skb_queue_purge(&policy->polq.hold_queue); |
a0073fe1 | 448 | |
285ead17 TT |
449 | if (del_timer(&policy->timer)) |
450 | xfrm_pol_put(policy); | |
451 | ||
452 | xfrm_pol_put(policy); | |
1da177e4 LT |
453 | } |
454 | ||
2518c7c2 DM |
455 | static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024; |
456 | ||
e92303f8 | 457 | static inline unsigned int idx_hash(struct net *net, u32 index) |
2518c7c2 | 458 | { |
e92303f8 | 459 | return __idx_hash(index, net->xfrm.policy_idx_hmask); |
2518c7c2 DM |
460 | } |
461 | ||
b58555f1 CG |
462 | /* calculate policy hash thresholds */ |
463 | static void __get_hash_thresh(struct net *net, | |
464 | unsigned short family, int dir, | |
465 | u8 *dbits, u8 *sbits) | |
466 | { | |
467 | switch (family) { | |
468 | case AF_INET: | |
469 | *dbits = net->xfrm.policy_bydst[dir].dbits4; | |
470 | *sbits = net->xfrm.policy_bydst[dir].sbits4; | |
471 | break; | |
472 | ||
473 | case AF_INET6: | |
474 | *dbits = net->xfrm.policy_bydst[dir].dbits6; | |
475 | *sbits = net->xfrm.policy_bydst[dir].sbits6; | |
476 | break; | |
477 | ||
478 | default: | |
479 | *dbits = 0; | |
480 | *sbits = 0; | |
481 | } | |
482 | } | |
483 | ||
5f803b58 DM |
484 | static struct hlist_head *policy_hash_bysel(struct net *net, |
485 | const struct xfrm_selector *sel, | |
486 | unsigned short family, int dir) | |
2518c7c2 | 487 | { |
1121994c | 488 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
b58555f1 CG |
489 | unsigned int hash; |
490 | u8 dbits; | |
491 | u8 sbits; | |
492 | ||
493 | __get_hash_thresh(net, family, dir, &dbits, &sbits); | |
494 | hash = __sel_hash(sel, family, hmask, dbits, sbits); | |
2518c7c2 | 495 | |
e1e551bc | 496 | if (hash == hmask + 1) |
cc1bb845 | 497 | return NULL; |
e1e551bc FW |
498 | |
499 | return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, | |
500 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; | |
2518c7c2 DM |
501 | } |
502 | ||
5f803b58 DM |
503 | static struct hlist_head *policy_hash_direct(struct net *net, |
504 | const xfrm_address_t *daddr, | |
505 | const xfrm_address_t *saddr, | |
506 | unsigned short family, int dir) | |
2518c7c2 | 507 | { |
1121994c | 508 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
b58555f1 CG |
509 | unsigned int hash; |
510 | u8 dbits; | |
511 | u8 sbits; | |
512 | ||
513 | __get_hash_thresh(net, family, dir, &dbits, &sbits); | |
514 | hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits); | |
2518c7c2 | 515 | |
e1e551bc FW |
516 | return rcu_dereference_check(net->xfrm.policy_bydst[dir].table, |
517 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash; | |
2518c7c2 DM |
518 | } |
519 | ||
b58555f1 CG |
520 | static void xfrm_dst_hash_transfer(struct net *net, |
521 | struct hlist_head *list, | |
2518c7c2 | 522 | struct hlist_head *ndsttable, |
b58555f1 CG |
523 | unsigned int nhashmask, |
524 | int dir) | |
2518c7c2 | 525 | { |
b67bfe0d | 526 | struct hlist_node *tmp, *entry0 = NULL; |
2518c7c2 | 527 | struct xfrm_policy *pol; |
b791160b | 528 | unsigned int h0 = 0; |
b58555f1 CG |
529 | u8 dbits; |
530 | u8 sbits; | |
2518c7c2 | 531 | |
b791160b | 532 | redo: |
b67bfe0d | 533 | hlist_for_each_entry_safe(pol, tmp, list, bydst) { |
2518c7c2 DM |
534 | unsigned int h; |
535 | ||
b58555f1 | 536 | __get_hash_thresh(net, pol->family, dir, &dbits, &sbits); |
2518c7c2 | 537 | h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr, |
b58555f1 | 538 | pol->family, nhashmask, dbits, sbits); |
3c611d40 | 539 | if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) { |
a5eefc1d FW |
540 | hlist_del_rcu(&pol->bydst); |
541 | hlist_add_head_rcu(&pol->bydst, ndsttable + h); | |
b791160b YH |
542 | h0 = h; |
543 | } else { | |
544 | if (h != h0) | |
545 | continue; | |
a5eefc1d FW |
546 | hlist_del_rcu(&pol->bydst); |
547 | hlist_add_behind_rcu(&pol->bydst, entry0); | |
b791160b | 548 | } |
b67bfe0d | 549 | entry0 = &pol->bydst; |
b791160b YH |
550 | } |
551 | if (!hlist_empty(list)) { | |
552 | entry0 = NULL; | |
553 | goto redo; | |
2518c7c2 DM |
554 | } |
555 | } | |
556 | ||
557 | static void xfrm_idx_hash_transfer(struct hlist_head *list, | |
558 | struct hlist_head *nidxtable, | |
559 | unsigned int nhashmask) | |
560 | { | |
b67bfe0d | 561 | struct hlist_node *tmp; |
2518c7c2 DM |
562 | struct xfrm_policy *pol; |
563 | ||
b67bfe0d | 564 | hlist_for_each_entry_safe(pol, tmp, list, byidx) { |
2518c7c2 DM |
565 | unsigned int h; |
566 | ||
567 | h = __idx_hash(pol->index, nhashmask); | |
568 | hlist_add_head(&pol->byidx, nidxtable+h); | |
569 | } | |
570 | } | |
571 | ||
572 | static unsigned long xfrm_new_hash_mask(unsigned int old_hmask) | |
573 | { | |
574 | return ((old_hmask + 1) << 1) - 1; | |
575 | } | |
576 | ||
66caf628 | 577 | static void xfrm_bydst_resize(struct net *net, int dir) |
2518c7c2 | 578 | { |
66caf628 | 579 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; |
2518c7c2 DM |
580 | unsigned int nhashmask = xfrm_new_hash_mask(hmask); |
581 | unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); | |
44e36b42 | 582 | struct hlist_head *ndst = xfrm_hash_alloc(nsize); |
e1e551bc | 583 | struct hlist_head *odst; |
2518c7c2 DM |
584 | int i; |
585 | ||
586 | if (!ndst) | |
587 | return; | |
588 | ||
9d0380df | 589 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
2580d3f4 | 590 | write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
30846090 | 591 | |
e1e551bc FW |
592 | odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table, |
593 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)); | |
594 | ||
2518c7c2 | 595 | for (i = hmask; i >= 0; i--) |
b58555f1 | 596 | xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir); |
2518c7c2 | 597 | |
e1e551bc | 598 | rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst); |
66caf628 | 599 | net->xfrm.policy_bydst[dir].hmask = nhashmask; |
2518c7c2 | 600 | |
2580d3f4 | 601 | write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); |
9d0380df | 602 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
2518c7c2 | 603 | |
e1e551bc FW |
604 | synchronize_rcu(); |
605 | ||
44e36b42 | 606 | xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head)); |
2518c7c2 DM |
607 | } |
608 | ||
cc2bbbfd | 609 | static void xfrm_byidx_resize(struct net *net) |
2518c7c2 | 610 | { |
66caf628 | 611 | unsigned int hmask = net->xfrm.policy_idx_hmask; |
2518c7c2 DM |
612 | unsigned int nhashmask = xfrm_new_hash_mask(hmask); |
613 | unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head); | |
66caf628 | 614 | struct hlist_head *oidx = net->xfrm.policy_byidx; |
44e36b42 | 615 | struct hlist_head *nidx = xfrm_hash_alloc(nsize); |
2518c7c2 DM |
616 | int i; |
617 | ||
618 | if (!nidx) | |
619 | return; | |
620 | ||
9d0380df | 621 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
2518c7c2 DM |
622 | |
623 | for (i = hmask; i >= 0; i--) | |
624 | xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask); | |
625 | ||
66caf628 AD |
626 | net->xfrm.policy_byidx = nidx; |
627 | net->xfrm.policy_idx_hmask = nhashmask; | |
2518c7c2 | 628 | |
9d0380df | 629 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
2518c7c2 | 630 | |
44e36b42 | 631 | xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head)); |
2518c7c2 DM |
632 | } |
633 | ||
66caf628 | 634 | static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total) |
2518c7c2 | 635 | { |
66caf628 AD |
636 | unsigned int cnt = net->xfrm.policy_count[dir]; |
637 | unsigned int hmask = net->xfrm.policy_bydst[dir].hmask; | |
2518c7c2 DM |
638 | |
639 | if (total) | |
640 | *total += cnt; | |
641 | ||
642 | if ((hmask + 1) < xfrm_policy_hashmax && | |
643 | cnt > hmask) | |
644 | return 1; | |
645 | ||
646 | return 0; | |
647 | } | |
648 | ||
66caf628 | 649 | static inline int xfrm_byidx_should_resize(struct net *net, int total) |
2518c7c2 | 650 | { |
66caf628 | 651 | unsigned int hmask = net->xfrm.policy_idx_hmask; |
2518c7c2 DM |
652 | |
653 | if ((hmask + 1) < xfrm_policy_hashmax && | |
654 | total > hmask) | |
655 | return 1; | |
656 | ||
657 | return 0; | |
658 | } | |
659 | ||
e071041b | 660 | void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si) |
ecfd6b18 | 661 | { |
e071041b AD |
662 | si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN]; |
663 | si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT]; | |
664 | si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD]; | |
665 | si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX]; | |
666 | si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX]; | |
667 | si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX]; | |
668 | si->spdhcnt = net->xfrm.policy_idx_hmask; | |
ecfd6b18 | 669 | si->spdhmcnt = xfrm_policy_hashmax; |
ecfd6b18 JHS |
670 | } |
671 | EXPORT_SYMBOL(xfrm_spd_getinfo); | |
2518c7c2 | 672 | |
ecfd6b18 | 673 | static DEFINE_MUTEX(hash_resize_mutex); |
66caf628 | 674 | static void xfrm_hash_resize(struct work_struct *work) |
2518c7c2 | 675 | { |
66caf628 | 676 | struct net *net = container_of(work, struct net, xfrm.policy_hash_work); |
2518c7c2 DM |
677 | int dir, total; |
678 | ||
679 | mutex_lock(&hash_resize_mutex); | |
680 | ||
681 | total = 0; | |
53c2e285 | 682 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
66caf628 AD |
683 | if (xfrm_bydst_should_resize(net, dir, &total)) |
684 | xfrm_bydst_resize(net, dir); | |
2518c7c2 | 685 | } |
66caf628 | 686 | if (xfrm_byidx_should_resize(net, total)) |
cc2bbbfd | 687 | xfrm_byidx_resize(net); |
2518c7c2 DM |
688 | |
689 | mutex_unlock(&hash_resize_mutex); | |
690 | } | |
691 | ||
24969fac | 692 | /* Make sure *pol can be inserted into fastbin. |
aa8ef1b9 | 693 | * Useful to check that later insert requests will be successful |
24969fac FW |
694 | * (provided xfrm_policy_lock is held throughout). |
695 | */ | |
696 | static struct xfrm_pol_inexact_bin * | |
697 | xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir) | |
698 | { | |
699 | struct xfrm_pol_inexact_bin *bin, *prev; | |
700 | struct xfrm_pol_inexact_key k = { | |
701 | .family = pol->family, | |
702 | .type = pol->type, | |
703 | .dir = dir, | |
b5fe22e2 | 704 | .if_id = pol->if_id, |
24969fac FW |
705 | }; |
706 | struct net *net = xp_net(pol); | |
707 | ||
708 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); | |
709 | ||
710 | write_pnet(&k.net, net); | |
711 | bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k, | |
712 | xfrm_pol_inexact_params); | |
713 | if (bin) | |
714 | return bin; | |
715 | ||
716 | bin = kzalloc(sizeof(*bin), GFP_ATOMIC); | |
717 | if (!bin) | |
718 | return NULL; | |
719 | ||
720 | bin->k = k; | |
721 | INIT_HLIST_HEAD(&bin->hhead); | |
9cf545eb | 722 | bin->root_d = RB_ROOT; |
64a09a7b | 723 | bin->root_s = RB_ROOT; |
77cc278f | 724 | seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock); |
24969fac FW |
725 | |
726 | prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table, | |
727 | &bin->k, &bin->head, | |
728 | xfrm_pol_inexact_params); | |
729 | if (!prev) { | |
730 | list_add(&bin->inexact_bins, &net->xfrm.inexact_bins); | |
731 | return bin; | |
732 | } | |
733 | ||
734 | kfree(bin); | |
735 | ||
736 | return IS_ERR(prev) ? NULL : prev; | |
737 | } | |
738 | ||
6be3b0db FW |
739 | static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr, |
740 | int family, u8 prefixlen) | |
24969fac | 741 | { |
6be3b0db FW |
742 | if (xfrm_addr_any(addr, family)) |
743 | return true; | |
744 | ||
745 | if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6) | |
746 | return true; | |
747 | ||
748 | if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4) | |
749 | return true; | |
750 | ||
751 | return false; | |
752 | } | |
753 | ||
754 | static bool | |
755 | xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy) | |
756 | { | |
757 | const xfrm_address_t *addr; | |
758 | bool saddr_any, daddr_any; | |
759 | u8 prefixlen; | |
760 | ||
761 | addr = &policy->selector.saddr; | |
762 | prefixlen = policy->selector.prefixlen_s; | |
24969fac | 763 | |
6be3b0db FW |
764 | saddr_any = xfrm_pol_inexact_addr_use_any_list(addr, |
765 | policy->family, | |
766 | prefixlen); | |
767 | addr = &policy->selector.daddr; | |
768 | prefixlen = policy->selector.prefixlen_d; | |
769 | daddr_any = xfrm_pol_inexact_addr_use_any_list(addr, | |
770 | policy->family, | |
771 | prefixlen); | |
772 | return saddr_any && daddr_any; | |
773 | } | |
774 | ||
9cf545eb FW |
775 | static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node, |
776 | const xfrm_address_t *addr, u8 prefixlen) | |
777 | { | |
778 | node->addr = *addr; | |
779 | node->prefixlen = prefixlen; | |
780 | } | |
781 | ||
782 | static struct xfrm_pol_inexact_node * | |
783 | xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen) | |
784 | { | |
785 | struct xfrm_pol_inexact_node *node; | |
786 | ||
787 | node = kzalloc(sizeof(*node), GFP_ATOMIC); | |
788 | if (node) | |
789 | xfrm_pol_inexact_node_init(node, addr, prefixlen); | |
790 | ||
791 | return node; | |
792 | } | |
793 | ||
794 | static int xfrm_policy_addr_delta(const xfrm_address_t *a, | |
795 | const xfrm_address_t *b, | |
796 | u8 prefixlen, u16 family) | |
797 | { | |
da64ae2d | 798 | u32 ma, mb, mask; |
9cf545eb FW |
799 | unsigned int pdw, pbi; |
800 | int delta = 0; | |
801 | ||
802 | switch (family) { | |
803 | case AF_INET: | |
da64ae2d VH |
804 | if (prefixlen == 0) |
805 | return 0; | |
806 | mask = ~0U << (32 - prefixlen); | |
807 | ma = ntohl(a->a4) & mask; | |
808 | mb = ntohl(b->a4) & mask; | |
809 | if (ma < mb) | |
810 | delta = -1; | |
811 | else if (ma > mb) | |
812 | delta = 1; | |
813 | break; | |
9cf545eb FW |
814 | case AF_INET6: |
815 | pdw = prefixlen >> 5; | |
816 | pbi = prefixlen & 0x1f; | |
817 | ||
818 | if (pdw) { | |
819 | delta = memcmp(a->a6, b->a6, pdw << 2); | |
820 | if (delta) | |
821 | return delta; | |
822 | } | |
823 | if (pbi) { | |
da64ae2d VH |
824 | mask = ~0U << (32 - pbi); |
825 | ma = ntohl(a->a6[pdw]) & mask; | |
826 | mb = ntohl(b->a6[pdw]) & mask; | |
827 | if (ma < mb) | |
828 | delta = -1; | |
829 | else if (ma > mb) | |
830 | delta = 1; | |
9cf545eb FW |
831 | } |
832 | break; | |
833 | default: | |
834 | break; | |
835 | } | |
836 | ||
837 | return delta; | |
838 | } | |
839 | ||
840 | static void xfrm_policy_inexact_list_reinsert(struct net *net, | |
841 | struct xfrm_pol_inexact_node *n, | |
842 | u16 family) | |
843 | { | |
e901cbc2 | 844 | unsigned int matched_s, matched_d; |
9cf545eb FW |
845 | struct xfrm_policy *policy, *p; |
846 | ||
e901cbc2 FW |
847 | matched_s = 0; |
848 | matched_d = 0; | |
849 | ||
9cf545eb | 850 | list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { |
1d38900c | 851 | struct hlist_node *newpos = NULL; |
e901cbc2 FW |
852 | bool matches_s, matches_d; |
853 | ||
6d41d4fe | 854 | if (policy->walk.dead || !policy->bydst_reinsert) |
9cf545eb FW |
855 | continue; |
856 | ||
857 | WARN_ON_ONCE(policy->family != family); | |
858 | ||
859 | policy->bydst_reinsert = false; | |
860 | hlist_for_each_entry(p, &n->hhead, bydst) { | |
1d38900c FW |
861 | if (policy->priority > p->priority) |
862 | newpos = &p->bydst; | |
863 | else if (policy->priority == p->priority && | |
864 | policy->pos > p->pos) | |
9cf545eb FW |
865 | newpos = &p->bydst; |
866 | else | |
867 | break; | |
868 | } | |
869 | ||
3c611d40 | 870 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
355b00d1 | 871 | hlist_add_behind_rcu(&policy->bydst, newpos); |
9cf545eb | 872 | else |
355b00d1 | 873 | hlist_add_head_rcu(&policy->bydst, &n->hhead); |
e901cbc2 FW |
874 | |
875 | /* paranoia checks follow. | |
876 | * Check that the reinserted policy matches at least | |
877 | * saddr or daddr for current node prefix. | |
878 | * | |
879 | * Matching both is fine, matching saddr in one policy | |
880 | * (but not daddr) and then matching only daddr in another | |
881 | * is a bug. | |
882 | */ | |
883 | matches_s = xfrm_policy_addr_delta(&policy->selector.saddr, | |
884 | &n->addr, | |
885 | n->prefixlen, | |
886 | family) == 0; | |
887 | matches_d = xfrm_policy_addr_delta(&policy->selector.daddr, | |
888 | &n->addr, | |
889 | n->prefixlen, | |
890 | family) == 0; | |
891 | if (matches_s && matches_d) | |
892 | continue; | |
893 | ||
894 | WARN_ON_ONCE(!matches_s && !matches_d); | |
895 | if (matches_s) | |
896 | matched_s++; | |
897 | if (matches_d) | |
898 | matched_d++; | |
899 | WARN_ON_ONCE(matched_s && matched_d); | |
9cf545eb FW |
900 | } |
901 | } | |
902 | ||
6ac098b2 FW |
903 | static void xfrm_policy_inexact_node_reinsert(struct net *net, |
904 | struct xfrm_pol_inexact_node *n, | |
905 | struct rb_root *new, | |
906 | u16 family) | |
907 | { | |
6ac098b2 | 908 | struct xfrm_pol_inexact_node *node; |
12750aba | 909 | struct rb_node **p, *parent; |
6ac098b2 FW |
910 | |
911 | /* we should not have another subtree here */ | |
912 | WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root)); | |
12750aba FW |
913 | restart: |
914 | parent = NULL; | |
6ac098b2 FW |
915 | p = &new->rb_node; |
916 | while (*p) { | |
917 | u8 prefixlen; | |
918 | int delta; | |
919 | ||
920 | parent = *p; | |
921 | node = rb_entry(*p, struct xfrm_pol_inexact_node, node); | |
922 | ||
923 | prefixlen = min(node->prefixlen, n->prefixlen); | |
924 | ||
925 | delta = xfrm_policy_addr_delta(&n->addr, &node->addr, | |
926 | prefixlen, family); | |
927 | if (delta < 0) { | |
928 | p = &parent->rb_left; | |
929 | } else if (delta > 0) { | |
930 | p = &parent->rb_right; | |
931 | } else { | |
769a807d | 932 | bool same_prefixlen = node->prefixlen == n->prefixlen; |
6ac098b2 FW |
933 | struct xfrm_policy *tmp; |
934 | ||
12750aba | 935 | hlist_for_each_entry(tmp, &n->hhead, bydst) { |
6ac098b2 | 936 | tmp->bydst_reinsert = true; |
12750aba FW |
937 | hlist_del_rcu(&tmp->bydst); |
938 | } | |
6ac098b2 | 939 | |
769a807d FW |
940 | node->prefixlen = prefixlen; |
941 | ||
6ac098b2 FW |
942 | xfrm_policy_inexact_list_reinsert(net, node, family); |
943 | ||
769a807d | 944 | if (same_prefixlen) { |
6ac098b2 FW |
945 | kfree_rcu(n, rcu); |
946 | return; | |
947 | } | |
948 | ||
949 | rb_erase(*p, new); | |
950 | kfree_rcu(n, rcu); | |
951 | n = node; | |
12750aba | 952 | goto restart; |
6ac098b2 FW |
953 | } |
954 | } | |
955 | ||
956 | rb_link_node_rcu(&n->node, parent, p); | |
957 | rb_insert_color(&n->node, new); | |
958 | } | |
959 | ||
9cf545eb FW |
960 | /* merge nodes v and n */ |
961 | static void xfrm_policy_inexact_node_merge(struct net *net, | |
962 | struct xfrm_pol_inexact_node *v, | |
963 | struct xfrm_pol_inexact_node *n, | |
964 | u16 family) | |
965 | { | |
6ac098b2 | 966 | struct xfrm_pol_inexact_node *node; |
9cf545eb | 967 | struct xfrm_policy *tmp; |
6ac098b2 FW |
968 | struct rb_node *rnode; |
969 | ||
970 | /* To-be-merged node v has a subtree. | |
971 | * | |
972 | * Dismantle it and insert its nodes to n->root. | |
973 | */ | |
974 | while ((rnode = rb_first(&v->root)) != NULL) { | |
975 | node = rb_entry(rnode, struct xfrm_pol_inexact_node, node); | |
976 | rb_erase(&node->node, &v->root); | |
977 | xfrm_policy_inexact_node_reinsert(net, node, &n->root, | |
978 | family); | |
979 | } | |
9cf545eb | 980 | |
1d38900c | 981 | hlist_for_each_entry(tmp, &v->hhead, bydst) { |
9cf545eb | 982 | tmp->bydst_reinsert = true; |
1d38900c FW |
983 | hlist_del_rcu(&tmp->bydst); |
984 | } | |
9cf545eb | 985 | |
9cf545eb FW |
986 | xfrm_policy_inexact_list_reinsert(net, n, family); |
987 | } | |
988 | ||
989 | static struct xfrm_pol_inexact_node * | |
990 | xfrm_policy_inexact_insert_node(struct net *net, | |
991 | struct rb_root *root, | |
992 | xfrm_address_t *addr, | |
993 | u16 family, u8 prefixlen, u8 dir) | |
994 | { | |
995 | struct xfrm_pol_inexact_node *cached = NULL; | |
996 | struct rb_node **p, *parent = NULL; | |
997 | struct xfrm_pol_inexact_node *node; | |
998 | ||
999 | p = &root->rb_node; | |
1000 | while (*p) { | |
1001 | int delta; | |
1002 | ||
1003 | parent = *p; | |
1004 | node = rb_entry(*p, struct xfrm_pol_inexact_node, node); | |
1005 | ||
1006 | delta = xfrm_policy_addr_delta(addr, &node->addr, | |
1007 | node->prefixlen, | |
1008 | family); | |
1009 | if (delta == 0 && prefixlen >= node->prefixlen) { | |
1010 | WARN_ON_ONCE(cached); /* ipsec policies got lost */ | |
1011 | return node; | |
1012 | } | |
1013 | ||
1014 | if (delta < 0) | |
1015 | p = &parent->rb_left; | |
1016 | else | |
1017 | p = &parent->rb_right; | |
1018 | ||
1019 | if (prefixlen < node->prefixlen) { | |
1020 | delta = xfrm_policy_addr_delta(addr, &node->addr, | |
1021 | prefixlen, | |
1022 | family); | |
1023 | if (delta) | |
1024 | continue; | |
1025 | ||
1026 | /* This node is a subnet of the new prefix. It needs | |
1027 | * to be removed and re-inserted with the smaller | |
1028 | * prefix and all nodes that are now also covered | |
1029 | * by the reduced prefixlen. | |
1030 | */ | |
1031 | rb_erase(&node->node, root); | |
1032 | ||
1033 | if (!cached) { | |
1034 | xfrm_pol_inexact_node_init(node, addr, | |
1035 | prefixlen); | |
1036 | cached = node; | |
1037 | } else { | |
1038 | /* This node also falls within the new | |
1039 | * prefixlen. Merge the to-be-reinserted | |
1040 | * node and this one. | |
1041 | */ | |
1042 | xfrm_policy_inexact_node_merge(net, node, | |
1043 | cached, family); | |
1044 | kfree_rcu(node, rcu); | |
1045 | } | |
1046 | ||
1047 | /* restart */ | |
1048 | p = &root->rb_node; | |
1049 | parent = NULL; | |
1050 | } | |
1051 | } | |
1052 | ||
1053 | node = cached; | |
1054 | if (!node) { | |
1055 | node = xfrm_pol_inexact_node_alloc(addr, prefixlen); | |
1056 | if (!node) | |
1057 | return NULL; | |
1058 | } | |
1059 | ||
1060 | rb_link_node_rcu(&node->node, parent, p); | |
1061 | rb_insert_color(&node->node, root); | |
1062 | ||
1063 | return node; | |
1064 | } | |
1065 | ||
1066 | static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm) | |
1067 | { | |
1068 | struct xfrm_pol_inexact_node *node; | |
1069 | struct rb_node *rn = rb_first(r); | |
1070 | ||
1071 | while (rn) { | |
1072 | node = rb_entry(rn, struct xfrm_pol_inexact_node, node); | |
1073 | ||
6ac098b2 | 1074 | xfrm_policy_inexact_gc_tree(&node->root, rm); |
9cf545eb FW |
1075 | rn = rb_next(rn); |
1076 | ||
6ac098b2 | 1077 | if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) { |
9cf545eb FW |
1078 | WARN_ON_ONCE(rm); |
1079 | continue; | |
1080 | } | |
1081 | ||
1082 | rb_erase(&node->node, r); | |
1083 | kfree_rcu(node, rcu); | |
1084 | } | |
1085 | } | |
1086 | ||
6be3b0db FW |
1087 | static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit) |
1088 | { | |
9cf545eb FW |
1089 | write_seqcount_begin(&b->count); |
1090 | xfrm_policy_inexact_gc_tree(&b->root_d, net_exit); | |
64a09a7b | 1091 | xfrm_policy_inexact_gc_tree(&b->root_s, net_exit); |
9cf545eb FW |
1092 | write_seqcount_end(&b->count); |
1093 | ||
64a09a7b | 1094 | if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) || |
9cf545eb | 1095 | !hlist_empty(&b->hhead)) { |
6be3b0db | 1096 | WARN_ON_ONCE(net_exit); |
24969fac | 1097 | return; |
6be3b0db | 1098 | } |
24969fac FW |
1099 | |
1100 | if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head, | |
1101 | xfrm_pol_inexact_params) == 0) { | |
1102 | list_del(&b->inexact_bins); | |
1103 | kfree_rcu(b, rcu); | |
1104 | } | |
1105 | } | |
1106 | ||
6be3b0db FW |
1107 | static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b) |
1108 | { | |
1109 | struct net *net = read_pnet(&b->k.net); | |
1110 | ||
1111 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); | |
1112 | __xfrm_policy_inexact_prune_bin(b, false); | |
1113 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1114 | } | |
1115 | ||
24969fac FW |
1116 | static void __xfrm_policy_inexact_flush(struct net *net) |
1117 | { | |
6be3b0db | 1118 | struct xfrm_pol_inexact_bin *bin, *t; |
24969fac FW |
1119 | |
1120 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); | |
1121 | ||
6be3b0db FW |
1122 | list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins) |
1123 | __xfrm_policy_inexact_prune_bin(bin, false); | |
24969fac FW |
1124 | } |
1125 | ||
9cf545eb FW |
1126 | static struct hlist_head * |
1127 | xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin, | |
1128 | struct xfrm_policy *policy, u8 dir) | |
1129 | { | |
1130 | struct xfrm_pol_inexact_node *n; | |
1131 | struct net *net; | |
1132 | ||
1133 | net = xp_net(policy); | |
1134 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); | |
1135 | ||
1136 | if (xfrm_policy_inexact_insert_use_any_list(policy)) | |
1137 | return &bin->hhead; | |
1138 | ||
64a09a7b FW |
1139 | if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr, |
1140 | policy->family, | |
1141 | policy->selector.prefixlen_d)) { | |
1142 | write_seqcount_begin(&bin->count); | |
1143 | n = xfrm_policy_inexact_insert_node(net, | |
1144 | &bin->root_s, | |
1145 | &policy->selector.saddr, | |
1146 | policy->family, | |
1147 | policy->selector.prefixlen_s, | |
1148 | dir); | |
1149 | write_seqcount_end(&bin->count); | |
1150 | if (!n) | |
1151 | return NULL; | |
1152 | ||
1153 | return &n->hhead; | |
1154 | } | |
1155 | ||
9cf545eb FW |
1156 | /* daddr is fixed */ |
1157 | write_seqcount_begin(&bin->count); | |
1158 | n = xfrm_policy_inexact_insert_node(net, | |
1159 | &bin->root_d, | |
1160 | &policy->selector.daddr, | |
1161 | policy->family, | |
1162 | policy->selector.prefixlen_d, dir); | |
1163 | write_seqcount_end(&bin->count); | |
1164 | if (!n) | |
1165 | return NULL; | |
6ac098b2 FW |
1166 | |
1167 | /* saddr is wildcard */ | |
1168 | if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr, | |
1169 | policy->family, | |
1170 | policy->selector.prefixlen_s)) | |
1171 | return &n->hhead; | |
1172 | ||
1173 | write_seqcount_begin(&bin->count); | |
1174 | n = xfrm_policy_inexact_insert_node(net, | |
1175 | &n->root, | |
1176 | &policy->selector.saddr, | |
1177 | policy->family, | |
1178 | policy->selector.prefixlen_s, dir); | |
1179 | write_seqcount_end(&bin->count); | |
1180 | if (!n) | |
1181 | return NULL; | |
1182 | ||
9cf545eb FW |
1183 | return &n->hhead; |
1184 | } | |
1185 | ||
24969fac FW |
1186 | static struct xfrm_policy * |
1187 | xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl) | |
1188 | { | |
1189 | struct xfrm_pol_inexact_bin *bin; | |
1190 | struct xfrm_policy *delpol; | |
1191 | struct hlist_head *chain; | |
1192 | struct net *net; | |
1193 | ||
1194 | bin = xfrm_policy_inexact_alloc_bin(policy, dir); | |
1195 | if (!bin) | |
1196 | return ERR_PTR(-ENOMEM); | |
1197 | ||
6be3b0db FW |
1198 | net = xp_net(policy); |
1199 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); | |
1200 | ||
9cf545eb FW |
1201 | chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir); |
1202 | if (!chain) { | |
1203 | __xfrm_policy_inexact_prune_bin(bin, false); | |
1204 | return ERR_PTR(-ENOMEM); | |
6be3b0db FW |
1205 | } |
1206 | ||
6be3b0db FW |
1207 | delpol = xfrm_policy_insert_list(chain, policy, excl); |
1208 | if (delpol && excl) { | |
1209 | __xfrm_policy_inexact_prune_bin(bin, false); | |
24969fac | 1210 | return ERR_PTR(-EEXIST); |
6be3b0db | 1211 | } |
24969fac | 1212 | |
24969fac FW |
1213 | chain = &net->xfrm.policy_inexact[dir]; |
1214 | xfrm_policy_insert_inexact_list(chain, policy); | |
1215 | ||
6be3b0db FW |
1216 | if (delpol) |
1217 | __xfrm_policy_inexact_prune_bin(bin, false); | |
1218 | ||
24969fac FW |
1219 | return delpol; |
1220 | } | |
1221 | ||
880a6fab CG |
1222 | static void xfrm_hash_rebuild(struct work_struct *work) |
1223 | { | |
1224 | struct net *net = container_of(work, struct net, | |
1225 | xfrm.policy_hthresh.work); | |
1226 | unsigned int hmask; | |
1227 | struct xfrm_policy *pol; | |
1228 | struct xfrm_policy *policy; | |
1229 | struct hlist_head *chain; | |
1230 | struct hlist_head *odst; | |
1231 | struct hlist_node *newpos; | |
1232 | int i; | |
1233 | int dir; | |
1234 | unsigned seq; | |
1235 | u8 lbits4, rbits4, lbits6, rbits6; | |
1236 | ||
1237 | mutex_lock(&hash_resize_mutex); | |
1238 | ||
1239 | /* read selector prefixlen thresholds */ | |
1240 | do { | |
1241 | seq = read_seqbegin(&net->xfrm.policy_hthresh.lock); | |
1242 | ||
1243 | lbits4 = net->xfrm.policy_hthresh.lbits4; | |
1244 | rbits4 = net->xfrm.policy_hthresh.rbits4; | |
1245 | lbits6 = net->xfrm.policy_hthresh.lbits6; | |
1246 | rbits6 = net->xfrm.policy_hthresh.rbits6; | |
1247 | } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq)); | |
1248 | ||
9d0380df | 1249 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
2580d3f4 | 1250 | write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
880a6fab | 1251 | |
24969fac FW |
1252 | /* make sure that we can insert the indirect policies again before |
1253 | * we start with destructive action. | |
1254 | */ | |
1255 | list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) { | |
6be3b0db | 1256 | struct xfrm_pol_inexact_bin *bin; |
24969fac FW |
1257 | u8 dbits, sbits; |
1258 | ||
6d41d4fe DC |
1259 | if (policy->walk.dead) |
1260 | continue; | |
1261 | ||
24969fac | 1262 | dir = xfrm_policy_id2dir(policy->index); |
6d41d4fe | 1263 | if (dir >= XFRM_POLICY_MAX) |
24969fac FW |
1264 | continue; |
1265 | ||
1266 | if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) { | |
1267 | if (policy->family == AF_INET) { | |
1268 | dbits = rbits4; | |
1269 | sbits = lbits4; | |
1270 | } else { | |
1271 | dbits = rbits6; | |
1272 | sbits = lbits6; | |
1273 | } | |
1274 | } else { | |
1275 | if (policy->family == AF_INET) { | |
1276 | dbits = lbits4; | |
1277 | sbits = rbits4; | |
1278 | } else { | |
1279 | dbits = lbits6; | |
1280 | sbits = rbits6; | |
1281 | } | |
1282 | } | |
1283 | ||
1284 | if (policy->selector.prefixlen_d < dbits || | |
1285 | policy->selector.prefixlen_s < sbits) | |
1286 | continue; | |
1287 | ||
6be3b0db FW |
1288 | bin = xfrm_policy_inexact_alloc_bin(policy, dir); |
1289 | if (!bin) | |
24969fac | 1290 | goto out_unlock; |
9cf545eb FW |
1291 | |
1292 | if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir)) | |
1293 | goto out_unlock; | |
24969fac FW |
1294 | } |
1295 | ||
880a6fab | 1296 | /* reset the bydst and inexact table in all directions */ |
53c2e285 | 1297 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
1548bc4e FW |
1298 | struct hlist_node *n; |
1299 | ||
1300 | hlist_for_each_entry_safe(policy, n, | |
1301 | &net->xfrm.policy_inexact[dir], | |
fd709721 FW |
1302 | bydst_inexact_list) { |
1303 | hlist_del_rcu(&policy->bydst); | |
1548bc4e | 1304 | hlist_del_init(&policy->bydst_inexact_list); |
fd709721 | 1305 | } |
1548bc4e | 1306 | |
880a6fab CG |
1307 | hmask = net->xfrm.policy_bydst[dir].hmask; |
1308 | odst = net->xfrm.policy_bydst[dir].table; | |
fd709721 FW |
1309 | for (i = hmask; i >= 0; i--) { |
1310 | hlist_for_each_entry_safe(policy, n, odst + i, bydst) | |
1311 | hlist_del_rcu(&policy->bydst); | |
1312 | } | |
880a6fab CG |
1313 | if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) { |
1314 | /* dir out => dst = remote, src = local */ | |
1315 | net->xfrm.policy_bydst[dir].dbits4 = rbits4; | |
1316 | net->xfrm.policy_bydst[dir].sbits4 = lbits4; | |
1317 | net->xfrm.policy_bydst[dir].dbits6 = rbits6; | |
1318 | net->xfrm.policy_bydst[dir].sbits6 = lbits6; | |
1319 | } else { | |
1320 | /* dir in/fwd => dst = local, src = remote */ | |
1321 | net->xfrm.policy_bydst[dir].dbits4 = lbits4; | |
1322 | net->xfrm.policy_bydst[dir].sbits4 = rbits4; | |
1323 | net->xfrm.policy_bydst[dir].dbits6 = lbits6; | |
1324 | net->xfrm.policy_bydst[dir].sbits6 = rbits6; | |
1325 | } | |
1326 | } | |
1327 | ||
1328 | /* re-insert all policies by order of creation */ | |
1329 | list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) { | |
88584c30 FW |
1330 | if (policy->walk.dead) |
1331 | continue; | |
1332 | dir = xfrm_policy_id2dir(policy->index); | |
1333 | if (dir >= XFRM_POLICY_MAX) { | |
6916fb3b TB |
1334 | /* skip socket policies */ |
1335 | continue; | |
1336 | } | |
880a6fab CG |
1337 | newpos = NULL; |
1338 | chain = policy_hash_bysel(net, &policy->selector, | |
88584c30 | 1339 | policy->family, dir); |
1548bc4e | 1340 | |
24969fac FW |
1341 | if (!chain) { |
1342 | void *p = xfrm_policy_inexact_insert(policy, dir, 0); | |
1343 | ||
1344 | WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p)); | |
1345 | continue; | |
1346 | } | |
1347 | ||
880a6fab CG |
1348 | hlist_for_each_entry(pol, chain, bydst) { |
1349 | if (policy->priority >= pol->priority) | |
1350 | newpos = &pol->bydst; | |
1351 | else | |
1352 | break; | |
1353 | } | |
3c611d40 | 1354 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
9dffff20 | 1355 | hlist_add_behind_rcu(&policy->bydst, newpos); |
880a6fab | 1356 | else |
9dffff20 | 1357 | hlist_add_head_rcu(&policy->bydst, chain); |
880a6fab CG |
1358 | } |
1359 | ||
24969fac | 1360 | out_unlock: |
6be3b0db | 1361 | __xfrm_policy_inexact_flush(net); |
2580d3f4 | 1362 | write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation); |
9d0380df | 1363 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
880a6fab CG |
1364 | |
1365 | mutex_unlock(&hash_resize_mutex); | |
1366 | } | |
1367 | ||
1368 | void xfrm_policy_hash_rebuild(struct net *net) | |
1369 | { | |
1370 | schedule_work(&net->xfrm.policy_hthresh.work); | |
1371 | } | |
1372 | EXPORT_SYMBOL(xfrm_policy_hash_rebuild); | |
1373 | ||
1da177e4 LT |
1374 | /* Generate new index... KAME seems to generate them ordered by cost |
1375 | * of an absolute inpredictability of ordering of rules. This will not pass. */ | |
e682adf0 | 1376 | static u32 xfrm_gen_index(struct net *net, int dir, u32 index) |
1da177e4 | 1377 | { |
1da177e4 | 1378 | for (;;) { |
2518c7c2 DM |
1379 | struct hlist_head *list; |
1380 | struct xfrm_policy *p; | |
1381 | u32 idx; | |
1382 | int found; | |
1383 | ||
e682adf0 | 1384 | if (!index) { |
3e4bc239 ED |
1385 | idx = (net->xfrm.idx_generator | dir); |
1386 | net->xfrm.idx_generator += 8; | |
e682adf0 FD |
1387 | } else { |
1388 | idx = index; | |
1389 | index = 0; | |
1390 | } | |
1391 | ||
1da177e4 LT |
1392 | if (idx == 0) |
1393 | idx = 8; | |
1121994c | 1394 | list = net->xfrm.policy_byidx + idx_hash(net, idx); |
2518c7c2 | 1395 | found = 0; |
b67bfe0d | 1396 | hlist_for_each_entry(p, list, byidx) { |
2518c7c2 DM |
1397 | if (p->index == idx) { |
1398 | found = 1; | |
1da177e4 | 1399 | break; |
2518c7c2 | 1400 | } |
1da177e4 | 1401 | } |
2518c7c2 | 1402 | if (!found) |
1da177e4 LT |
1403 | return idx; |
1404 | } | |
1405 | } | |
1406 | ||
2518c7c2 DM |
1407 | static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2) |
1408 | { | |
1409 | u32 *p1 = (u32 *) s1; | |
1410 | u32 *p2 = (u32 *) s2; | |
1411 | int len = sizeof(struct xfrm_selector) / sizeof(u32); | |
1412 | int i; | |
1413 | ||
1414 | for (i = 0; i < len; i++) { | |
1415 | if (p1[i] != p2[i]) | |
1416 | return 1; | |
1417 | } | |
1418 | ||
1419 | return 0; | |
1420 | } | |
1421 | ||
a0073fe1 SK |
1422 | static void xfrm_policy_requeue(struct xfrm_policy *old, |
1423 | struct xfrm_policy *new) | |
1424 | { | |
1425 | struct xfrm_policy_queue *pq = &old->polq; | |
1426 | struct sk_buff_head list; | |
1427 | ||
de2ad486 LR |
1428 | if (skb_queue_empty(&pq->hold_queue)) |
1429 | return; | |
1430 | ||
a0073fe1 SK |
1431 | __skb_queue_head_init(&list); |
1432 | ||
1433 | spin_lock_bh(&pq->hold_queue.lock); | |
1434 | skb_queue_splice_init(&pq->hold_queue, &list); | |
e7d8f6cb SK |
1435 | if (del_timer(&pq->hold_timer)) |
1436 | xfrm_pol_put(old); | |
a0073fe1 SK |
1437 | spin_unlock_bh(&pq->hold_queue.lock); |
1438 | ||
a0073fe1 SK |
1439 | pq = &new->polq; |
1440 | ||
1441 | spin_lock_bh(&pq->hold_queue.lock); | |
1442 | skb_queue_splice(&list, &pq->hold_queue); | |
1443 | pq->timeout = XFRM_QUEUE_TMO_MIN; | |
e7d8f6cb SK |
1444 | if (!mod_timer(&pq->hold_timer, jiffies)) |
1445 | xfrm_pol_hold(new); | |
a0073fe1 SK |
1446 | spin_unlock_bh(&pq->hold_queue.lock); |
1447 | } | |
1448 | ||
4f47e8ab XL |
1449 | static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark, |
1450 | struct xfrm_policy *pol) | |
7cb8a939 | 1451 | { |
4f47e8ab | 1452 | return mark->v == pol->mark.v && mark->m == pol->mark.m; |
7cb8a939 SK |
1453 | } |
1454 | ||
24969fac FW |
1455 | static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed) |
1456 | { | |
1457 | const struct xfrm_pol_inexact_key *k = data; | |
1458 | u32 a = k->type << 24 | k->dir << 16 | k->family; | |
1459 | ||
b5fe22e2 FW |
1460 | return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)), |
1461 | seed); | |
24969fac FW |
1462 | } |
1463 | ||
1464 | static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed) | |
1465 | { | |
1466 | const struct xfrm_pol_inexact_bin *b = data; | |
1467 | ||
1468 | return xfrm_pol_bin_key(&b->k, 0, seed); | |
1469 | } | |
1470 | ||
1471 | static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg, | |
1472 | const void *ptr) | |
1473 | { | |
1474 | const struct xfrm_pol_inexact_key *key = arg->key; | |
1475 | const struct xfrm_pol_inexact_bin *b = ptr; | |
1476 | int ret; | |
1477 | ||
1478 | if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net))) | |
1479 | return -1; | |
1480 | ||
1481 | ret = b->k.dir ^ key->dir; | |
1482 | if (ret) | |
1483 | return ret; | |
1484 | ||
1485 | ret = b->k.type ^ key->type; | |
1486 | if (ret) | |
1487 | return ret; | |
1488 | ||
1489 | ret = b->k.family ^ key->family; | |
1490 | if (ret) | |
1491 | return ret; | |
1492 | ||
b5fe22e2 | 1493 | return b->k.if_id ^ key->if_id; |
24969fac FW |
1494 | } |
1495 | ||
1496 | static const struct rhashtable_params xfrm_pol_inexact_params = { | |
1497 | .head_offset = offsetof(struct xfrm_pol_inexact_bin, head), | |
1498 | .hashfn = xfrm_pol_bin_key, | |
1499 | .obj_hashfn = xfrm_pol_bin_obj, | |
1500 | .obj_cmpfn = xfrm_pol_bin_cmp, | |
1501 | .automatic_shrinking = true, | |
1502 | }; | |
1503 | ||
1504 | static void xfrm_policy_insert_inexact_list(struct hlist_head *chain, | |
1505 | struct xfrm_policy *policy) | |
1506 | { | |
1507 | struct xfrm_policy *pol, *delpol = NULL; | |
1508 | struct hlist_node *newpos = NULL; | |
6be3b0db | 1509 | int i = 0; |
24969fac FW |
1510 | |
1511 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { | |
1512 | if (pol->type == policy->type && | |
1513 | pol->if_id == policy->if_id && | |
1514 | !selector_cmp(&pol->selector, &policy->selector) && | |
4f47e8ab | 1515 | xfrm_policy_mark_match(&policy->mark, pol) && |
24969fac FW |
1516 | xfrm_sec_ctx_match(pol->security, policy->security) && |
1517 | !WARN_ON(delpol)) { | |
1518 | delpol = pol; | |
1519 | if (policy->priority > pol->priority) | |
1520 | continue; | |
1521 | } else if (policy->priority >= pol->priority) { | |
1522 | newpos = &pol->bydst_inexact_list; | |
1523 | continue; | |
1524 | } | |
1525 | if (delpol) | |
1526 | break; | |
1527 | } | |
1528 | ||
3c611d40 | 1529 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
24969fac FW |
1530 | hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos); |
1531 | else | |
1532 | hlist_add_head_rcu(&policy->bydst_inexact_list, chain); | |
6be3b0db FW |
1533 | |
1534 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { | |
1535 | pol->pos = i; | |
1536 | i++; | |
1537 | } | |
24969fac FW |
1538 | } |
1539 | ||
a927d6af FW |
1540 | static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain, |
1541 | struct xfrm_policy *policy, | |
1542 | bool excl) | |
1da177e4 | 1543 | { |
a927d6af | 1544 | struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL; |
1da177e4 | 1545 | |
b67bfe0d | 1546 | hlist_for_each_entry(pol, chain, bydst) { |
a6c7ab55 | 1547 | if (pol->type == policy->type && |
7e652640 | 1548 | pol->if_id == policy->if_id && |
2518c7c2 | 1549 | !selector_cmp(&pol->selector, &policy->selector) && |
4f47e8ab | 1550 | xfrm_policy_mark_match(&policy->mark, pol) && |
a6c7ab55 HX |
1551 | xfrm_sec_ctx_match(pol->security, policy->security) && |
1552 | !WARN_ON(delpol)) { | |
a927d6af FW |
1553 | if (excl) |
1554 | return ERR_PTR(-EEXIST); | |
1da177e4 LT |
1555 | delpol = pol; |
1556 | if (policy->priority > pol->priority) | |
1557 | continue; | |
1558 | } else if (policy->priority >= pol->priority) { | |
a927d6af | 1559 | newpos = pol; |
1da177e4 LT |
1560 | continue; |
1561 | } | |
1da177e4 LT |
1562 | if (delpol) |
1563 | break; | |
1da177e4 | 1564 | } |
24969fac | 1565 | |
3c611d40 | 1566 | if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET) |
a927d6af | 1567 | hlist_add_behind_rcu(&policy->bydst, &newpos->bydst); |
2518c7c2 | 1568 | else |
3c611d40 LR |
1569 | /* Packet offload policies enter to the head |
1570 | * to speed-up lookups. | |
1571 | */ | |
9dffff20 | 1572 | hlist_add_head_rcu(&policy->bydst, chain); |
a927d6af FW |
1573 | |
1574 | return delpol; | |
1575 | } | |
1576 | ||
1577 | int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) | |
1578 | { | |
1579 | struct net *net = xp_net(policy); | |
1580 | struct xfrm_policy *delpol; | |
1581 | struct hlist_head *chain; | |
1582 | ||
1583 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); | |
1584 | chain = policy_hash_bysel(net, &policy->selector, policy->family, dir); | |
24969fac | 1585 | if (chain) |
cc1bb845 | 1586 | delpol = xfrm_policy_insert_list(chain, policy, excl); |
24969fac FW |
1587 | else |
1588 | delpol = xfrm_policy_inexact_insert(policy, dir, excl); | |
a927d6af FW |
1589 | |
1590 | if (IS_ERR(delpol)) { | |
1591 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1592 | return PTR_ERR(delpol); | |
1593 | } | |
1594 | ||
12bfa8bd | 1595 | __xfrm_policy_link(policy, dir); |
ca4c3fc2 | 1596 | |
1597 | /* After previous checking, family can either be AF_INET or AF_INET6 */ | |
1598 | if (policy->family == AF_INET) | |
1599 | rt_genid_bump_ipv4(net); | |
1600 | else | |
1601 | rt_genid_bump_ipv6(net); | |
1602 | ||
a0073fe1 SK |
1603 | if (delpol) { |
1604 | xfrm_policy_requeue(delpol, policy); | |
29fa0b30 | 1605 | __xfrm_policy_unlink(delpol, dir); |
a0073fe1 | 1606 | } |
e682adf0 | 1607 | policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index); |
1121994c | 1608 | hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index)); |
386c5680 | 1609 | policy->curlft.add_time = ktime_get_real_seconds(); |
1da177e4 LT |
1610 | policy->curlft.use_time = 0; |
1611 | if (!mod_timer(&policy->timer, jiffies + HZ)) | |
1612 | xfrm_pol_hold(policy); | |
9d0380df | 1613 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 | 1614 | |
9b78a82c | 1615 | if (delpol) |
1da177e4 | 1616 | xfrm_policy_kill(delpol); |
1121994c AD |
1617 | else if (xfrm_bydst_should_resize(net, dir, NULL)) |
1618 | schedule_work(&net->xfrm.policy_hash_work); | |
9b78a82c | 1619 | |
1da177e4 LT |
1620 | return 0; |
1621 | } | |
1622 | EXPORT_SYMBOL(xfrm_policy_insert); | |
1623 | ||
6be3b0db | 1624 | static struct xfrm_policy * |
4f47e8ab XL |
1625 | __xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark, |
1626 | u32 if_id, u8 type, int dir, struct xfrm_selector *sel, | |
6be3b0db FW |
1627 | struct xfrm_sec_ctx *ctx) |
1628 | { | |
1629 | struct xfrm_policy *pol; | |
1630 | ||
1631 | if (!chain) | |
1632 | return NULL; | |
1633 | ||
1634 | hlist_for_each_entry(pol, chain, bydst) { | |
1635 | if (pol->type == type && | |
1636 | pol->if_id == if_id && | |
4f47e8ab | 1637 | xfrm_policy_mark_match(mark, pol) && |
6be3b0db FW |
1638 | !selector_cmp(sel, &pol->selector) && |
1639 | xfrm_sec_ctx_match(ctx, pol->security)) | |
1640 | return pol; | |
1641 | } | |
1642 | ||
1643 | return NULL; | |
1644 | } | |
1645 | ||
4f47e8ab XL |
1646 | struct xfrm_policy * |
1647 | xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id, | |
1648 | u8 type, int dir, struct xfrm_selector *sel, | |
1649 | struct xfrm_sec_ctx *ctx, int delete, int *err) | |
1da177e4 | 1650 | { |
24969fac FW |
1651 | struct xfrm_pol_inexact_bin *bin = NULL; |
1652 | struct xfrm_policy *pol, *ret = NULL; | |
2518c7c2 | 1653 | struct hlist_head *chain; |
1da177e4 | 1654 | |
ef41aaa0 | 1655 | *err = 0; |
9d0380df | 1656 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
8d1211a6 | 1657 | chain = policy_hash_bysel(net, sel, sel->family, dir); |
24969fac | 1658 | if (!chain) { |
6be3b0db FW |
1659 | struct xfrm_pol_inexact_candidates cand; |
1660 | int i; | |
1661 | ||
24969fac | 1662 | bin = xfrm_policy_inexact_lookup(net, type, |
b5fe22e2 | 1663 | sel->family, dir, if_id); |
24969fac FW |
1664 | if (!bin) { |
1665 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1666 | return NULL; | |
1667 | } | |
1668 | ||
6be3b0db FW |
1669 | if (!xfrm_policy_find_inexact_candidates(&cand, bin, |
1670 | &sel->saddr, | |
1671 | &sel->daddr)) { | |
1672 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1673 | return NULL; | |
1674 | } | |
1675 | ||
1676 | pol = NULL; | |
1677 | for (i = 0; i < ARRAY_SIZE(cand.res); i++) { | |
1678 | struct xfrm_policy *tmp; | |
1679 | ||
1680 | tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark, | |
1681 | if_id, type, dir, | |
1682 | sel, ctx); | |
39aa6928 FW |
1683 | if (!tmp) |
1684 | continue; | |
1685 | ||
1686 | if (!pol || tmp->pos < pol->pos) | |
6be3b0db FW |
1687 | pol = tmp; |
1688 | } | |
1689 | } else { | |
1690 | pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir, | |
1691 | sel, ctx); | |
24969fac FW |
1692 | } |
1693 | ||
6be3b0db FW |
1694 | if (pol) { |
1695 | xfrm_pol_hold(pol); | |
1696 | if (delete) { | |
1697 | *err = security_xfrm_policy_delete(pol->security); | |
1698 | if (*err) { | |
1699 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1700 | return pol; | |
2518c7c2 | 1701 | } |
6be3b0db | 1702 | __xfrm_policy_unlink(pol, dir); |
1da177e4 | 1703 | } |
6be3b0db | 1704 | ret = pol; |
1da177e4 | 1705 | } |
9d0380df | 1706 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 | 1707 | |
fe1a5f03 | 1708 | if (ret && delete) |
2518c7c2 | 1709 | xfrm_policy_kill(ret); |
6be3b0db FW |
1710 | if (bin && delete) |
1711 | xfrm_policy_inexact_prune_bin(bin); | |
2518c7c2 | 1712 | return ret; |
1da177e4 | 1713 | } |
df71837d | 1714 | EXPORT_SYMBOL(xfrm_policy_bysel_ctx); |
1da177e4 | 1715 | |
4f47e8ab XL |
1716 | struct xfrm_policy * |
1717 | xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id, | |
1718 | u8 type, int dir, u32 id, int delete, int *err) | |
1da177e4 | 1719 | { |
2518c7c2 DM |
1720 | struct xfrm_policy *pol, *ret; |
1721 | struct hlist_head *chain; | |
1da177e4 | 1722 | |
b5505c6e HX |
1723 | *err = -ENOENT; |
1724 | if (xfrm_policy_id2dir(id) != dir) | |
1725 | return NULL; | |
1726 | ||
ef41aaa0 | 1727 | *err = 0; |
9d0380df | 1728 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
8d1211a6 | 1729 | chain = net->xfrm.policy_byidx + idx_hash(net, id); |
2518c7c2 | 1730 | ret = NULL; |
b67bfe0d | 1731 | hlist_for_each_entry(pol, chain, byidx) { |
34f8d884 | 1732 | if (pol->type == type && pol->index == id && |
4f47e8ab | 1733 | pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) { |
1da177e4 | 1734 | xfrm_pol_hold(pol); |
2518c7c2 | 1735 | if (delete) { |
03e1ad7b PM |
1736 | *err = security_xfrm_policy_delete( |
1737 | pol->security); | |
ef41aaa0 | 1738 | if (*err) { |
9d0380df | 1739 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
ef41aaa0 EP |
1740 | return pol; |
1741 | } | |
29fa0b30 | 1742 | __xfrm_policy_unlink(pol, dir); |
2518c7c2 DM |
1743 | } |
1744 | ret = pol; | |
1da177e4 LT |
1745 | break; |
1746 | } | |
1747 | } | |
9d0380df | 1748 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 | 1749 | |
fe1a5f03 | 1750 | if (ret && delete) |
2518c7c2 | 1751 | xfrm_policy_kill(ret); |
2518c7c2 | 1752 | return ret; |
1da177e4 LT |
1753 | } |
1754 | EXPORT_SYMBOL(xfrm_policy_byid); | |
1755 | ||
4aa2e62c JL |
1756 | #ifdef CONFIG_SECURITY_NETWORK_XFRM |
1757 | static inline int | |
2e71029e | 1758 | xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) |
1da177e4 | 1759 | { |
ceb159e3 FW |
1760 | struct xfrm_policy *pol; |
1761 | int err = 0; | |
4aa2e62c | 1762 | |
ceb159e3 FW |
1763 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { |
1764 | if (pol->walk.dead || | |
1765 | xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || | |
1766 | pol->type != type) | |
1767 | continue; | |
4aa2e62c | 1768 | |
ceb159e3 FW |
1769 | err = security_xfrm_policy_delete(pol->security); |
1770 | if (err) { | |
1771 | xfrm_audit_policy_delete(pol, 0, task_valid); | |
1772 | return err; | |
4aa2e62c JL |
1773 | } |
1774 | } | |
1775 | return err; | |
1776 | } | |
919e43fa LR |
1777 | |
1778 | static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, | |
1779 | struct net_device *dev, | |
1780 | bool task_valid) | |
1781 | { | |
1782 | struct xfrm_policy *pol; | |
1783 | int err = 0; | |
1784 | ||
1785 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { | |
1786 | if (pol->walk.dead || | |
1787 | xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX || | |
1788 | pol->xdo.dev != dev) | |
1789 | continue; | |
1790 | ||
1791 | err = security_xfrm_policy_delete(pol->security); | |
1792 | if (err) { | |
1793 | xfrm_audit_policy_delete(pol, 0, task_valid); | |
1794 | return err; | |
1795 | } | |
1796 | } | |
1797 | return err; | |
1798 | } | |
4aa2e62c JL |
1799 | #else |
1800 | static inline int | |
2e71029e | 1801 | xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid) |
4aa2e62c JL |
1802 | { |
1803 | return 0; | |
1804 | } | |
919e43fa LR |
1805 | |
1806 | static inline int xfrm_dev_policy_flush_secctx_check(struct net *net, | |
1807 | struct net_device *dev, | |
1808 | bool task_valid) | |
1809 | { | |
1810 | return 0; | |
1811 | } | |
4aa2e62c JL |
1812 | #endif |
1813 | ||
2e71029e | 1814 | int xfrm_policy_flush(struct net *net, u8 type, bool task_valid) |
4aa2e62c | 1815 | { |
2f1eb65f | 1816 | int dir, err = 0, cnt = 0; |
ceb159e3 | 1817 | struct xfrm_policy *pol; |
1da177e4 | 1818 | |
9d0380df | 1819 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
4aa2e62c | 1820 | |
2e71029e | 1821 | err = xfrm_policy_flush_secctx_check(net, type, task_valid); |
4aa2e62c JL |
1822 | if (err) |
1823 | goto out; | |
1824 | ||
ceb159e3 FW |
1825 | again: |
1826 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { | |
6d41d4fe DC |
1827 | if (pol->walk.dead) |
1828 | continue; | |
1829 | ||
ceb159e3 | 1830 | dir = xfrm_policy_id2dir(pol->index); |
6d41d4fe | 1831 | if (dir >= XFRM_POLICY_MAX || |
ceb159e3 FW |
1832 | pol->type != type) |
1833 | continue; | |
2518c7c2 | 1834 | |
ceb159e3 FW |
1835 | __xfrm_policy_unlink(pol, dir); |
1836 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
bf06fcf4 | 1837 | xfrm_dev_policy_delete(pol); |
ceb159e3 FW |
1838 | cnt++; |
1839 | xfrm_audit_policy_delete(pol, 1, task_valid); | |
1840 | xfrm_policy_kill(pol); | |
1841 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); | |
1842 | goto again; | |
1da177e4 | 1843 | } |
24969fac FW |
1844 | if (cnt) |
1845 | __xfrm_policy_inexact_flush(net); | |
1846 | else | |
2f1eb65f | 1847 | err = -ESRCH; |
4aa2e62c | 1848 | out: |
9d0380df | 1849 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
4aa2e62c | 1850 | return err; |
1da177e4 LT |
1851 | } |
1852 | EXPORT_SYMBOL(xfrm_policy_flush); | |
1853 | ||
919e43fa LR |
1854 | int xfrm_dev_policy_flush(struct net *net, struct net_device *dev, |
1855 | bool task_valid) | |
1856 | { | |
1857 | int dir, err = 0, cnt = 0; | |
1858 | struct xfrm_policy *pol; | |
1859 | ||
1860 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); | |
1861 | ||
1862 | err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid); | |
1863 | if (err) | |
1864 | goto out; | |
1865 | ||
1866 | again: | |
1867 | list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) { | |
6d41d4fe DC |
1868 | if (pol->walk.dead) |
1869 | continue; | |
1870 | ||
919e43fa | 1871 | dir = xfrm_policy_id2dir(pol->index); |
6d41d4fe | 1872 | if (dir >= XFRM_POLICY_MAX || |
919e43fa LR |
1873 | pol->xdo.dev != dev) |
1874 | continue; | |
1875 | ||
1876 | __xfrm_policy_unlink(pol, dir); | |
1877 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
bf06fcf4 | 1878 | xfrm_dev_policy_delete(pol); |
919e43fa LR |
1879 | cnt++; |
1880 | xfrm_audit_policy_delete(pol, 1, task_valid); | |
1881 | xfrm_policy_kill(pol); | |
1882 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); | |
1883 | goto again; | |
1884 | } | |
1885 | if (cnt) | |
1886 | __xfrm_policy_inexact_flush(net); | |
1887 | else | |
1888 | err = -ESRCH; | |
1889 | out: | |
1890 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1891 | return err; | |
1892 | } | |
1893 | EXPORT_SYMBOL(xfrm_dev_policy_flush); | |
1894 | ||
cdcbca7c | 1895 | int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk, |
4c563f76 | 1896 | int (*func)(struct xfrm_policy *, int, int, void*), |
1da177e4 LT |
1897 | void *data) |
1898 | { | |
12a169e7 HX |
1899 | struct xfrm_policy *pol; |
1900 | struct xfrm_policy_walk_entry *x; | |
4c563f76 TT |
1901 | int error = 0; |
1902 | ||
1903 | if (walk->type >= XFRM_POLICY_TYPE_MAX && | |
1904 | walk->type != XFRM_POLICY_TYPE_ANY) | |
1905 | return -EINVAL; | |
1da177e4 | 1906 | |
12a169e7 | 1907 | if (list_empty(&walk->walk.all) && walk->seq != 0) |
4c563f76 TT |
1908 | return 0; |
1909 | ||
9d0380df | 1910 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
12a169e7 | 1911 | if (list_empty(&walk->walk.all)) |
cdcbca7c | 1912 | x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all); |
12a169e7 | 1913 | else |
80077702 LR |
1914 | x = list_first_entry(&walk->walk.all, |
1915 | struct xfrm_policy_walk_entry, all); | |
1916 | ||
cdcbca7c | 1917 | list_for_each_entry_from(x, &net->xfrm.policy_all, all) { |
12a169e7 | 1918 | if (x->dead) |
4c563f76 | 1919 | continue; |
12a169e7 HX |
1920 | pol = container_of(x, struct xfrm_policy, walk); |
1921 | if (walk->type != XFRM_POLICY_TYPE_ANY && | |
1922 | walk->type != pol->type) | |
1923 | continue; | |
1924 | error = func(pol, xfrm_policy_id2dir(pol->index), | |
1925 | walk->seq, data); | |
1926 | if (error) { | |
1927 | list_move_tail(&walk->walk.all, &x->all); | |
1928 | goto out; | |
2518c7c2 | 1929 | } |
12a169e7 | 1930 | walk->seq++; |
1da177e4 | 1931 | } |
12a169e7 | 1932 | if (walk->seq == 0) { |
baf5d743 JHS |
1933 | error = -ENOENT; |
1934 | goto out; | |
1935 | } | |
12a169e7 | 1936 | list_del_init(&walk->walk.all); |
1da177e4 | 1937 | out: |
9d0380df | 1938 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 LT |
1939 | return error; |
1940 | } | |
1941 | EXPORT_SYMBOL(xfrm_policy_walk); | |
1942 | ||
12a169e7 HX |
1943 | void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type) |
1944 | { | |
1945 | INIT_LIST_HEAD(&walk->walk.all); | |
1946 | walk->walk.dead = 1; | |
1947 | walk->type = type; | |
1948 | walk->seq = 0; | |
1949 | } | |
1950 | EXPORT_SYMBOL(xfrm_policy_walk_init); | |
1951 | ||
283bc9f3 | 1952 | void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net) |
12a169e7 HX |
1953 | { |
1954 | if (list_empty(&walk->walk.all)) | |
1955 | return; | |
1956 | ||
9d0380df | 1957 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */ |
12a169e7 | 1958 | list_del(&walk->walk.all); |
9d0380df | 1959 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
12a169e7 HX |
1960 | } |
1961 | EXPORT_SYMBOL(xfrm_policy_walk_done); | |
1962 | ||
134b0fc5 JM |
1963 | /* |
1964 | * Find policy to apply to this flow. | |
1965 | * | |
1966 | * Returns 0 if policy found, else an -errno. | |
1967 | */ | |
f299d557 DM |
1968 | static int xfrm_policy_match(const struct xfrm_policy *pol, |
1969 | const struct flowi *fl, | |
0de19788 | 1970 | u8 type, u16 family, u32 if_id) |
1da177e4 | 1971 | { |
f299d557 | 1972 | const struct xfrm_selector *sel = &pol->selector; |
bc9b35ad DM |
1973 | int ret = -ESRCH; |
1974 | bool match; | |
1da177e4 | 1975 | |
2518c7c2 | 1976 | if (pol->family != family || |
bc56b334 | 1977 | pol->if_id != if_id || |
1d28f42c | 1978 | (fl->flowi_mark & pol->mark.m) != pol->mark.v || |
2518c7c2 | 1979 | pol->type != type) |
134b0fc5 | 1980 | return ret; |
1da177e4 | 1981 | |
2518c7c2 | 1982 | match = xfrm_selector_match(sel, fl, family); |
134b0fc5 | 1983 | if (match) |
8a922805 | 1984 | ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid); |
134b0fc5 | 1985 | return ret; |
2518c7c2 | 1986 | } |
1da177e4 | 1987 | |
9cf545eb FW |
1988 | static struct xfrm_pol_inexact_node * |
1989 | xfrm_policy_lookup_inexact_addr(const struct rb_root *r, | |
77cc278f | 1990 | seqcount_spinlock_t *count, |
9cf545eb FW |
1991 | const xfrm_address_t *addr, u16 family) |
1992 | { | |
1993 | const struct rb_node *parent; | |
1994 | int seq; | |
1995 | ||
1996 | again: | |
1997 | seq = read_seqcount_begin(count); | |
1998 | ||
1999 | parent = rcu_dereference_raw(r->rb_node); | |
2000 | while (parent) { | |
2001 | struct xfrm_pol_inexact_node *node; | |
2002 | int delta; | |
2003 | ||
2004 | node = rb_entry(parent, struct xfrm_pol_inexact_node, node); | |
2005 | ||
2006 | delta = xfrm_policy_addr_delta(addr, &node->addr, | |
2007 | node->prefixlen, family); | |
2008 | if (delta < 0) { | |
2009 | parent = rcu_dereference_raw(parent->rb_left); | |
2010 | continue; | |
2011 | } else if (delta > 0) { | |
2012 | parent = rcu_dereference_raw(parent->rb_right); | |
2013 | continue; | |
2014 | } | |
2015 | ||
2016 | return node; | |
2017 | } | |
2018 | ||
2019 | if (read_seqcount_retry(count, seq)) | |
2020 | goto again; | |
2021 | ||
2022 | return NULL; | |
2023 | } | |
2024 | ||
6be3b0db FW |
2025 | static bool |
2026 | xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand, | |
2027 | struct xfrm_pol_inexact_bin *b, | |
2028 | const xfrm_address_t *saddr, | |
2029 | const xfrm_address_t *daddr) | |
2030 | { | |
9cf545eb FW |
2031 | struct xfrm_pol_inexact_node *n; |
2032 | u16 family; | |
2033 | ||
6be3b0db FW |
2034 | if (!b) |
2035 | return false; | |
2036 | ||
9cf545eb | 2037 | family = b->k.family; |
6be3b0db FW |
2038 | memset(cand, 0, sizeof(*cand)); |
2039 | cand->res[XFRM_POL_CAND_ANY] = &b->hhead; | |
9cf545eb FW |
2040 | |
2041 | n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr, | |
2042 | family); | |
6ac098b2 | 2043 | if (n) { |
9cf545eb | 2044 | cand->res[XFRM_POL_CAND_DADDR] = &n->hhead; |
6ac098b2 FW |
2045 | n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr, |
2046 | family); | |
2047 | if (n) | |
2048 | cand->res[XFRM_POL_CAND_BOTH] = &n->hhead; | |
2049 | } | |
9cf545eb | 2050 | |
64a09a7b FW |
2051 | n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr, |
2052 | family); | |
2053 | if (n) | |
2054 | cand->res[XFRM_POL_CAND_SADDR] = &n->hhead; | |
2055 | ||
6be3b0db FW |
2056 | return true; |
2057 | } | |
2058 | ||
24969fac | 2059 | static struct xfrm_pol_inexact_bin * |
b5fe22e2 FW |
2060 | xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family, |
2061 | u8 dir, u32 if_id) | |
24969fac FW |
2062 | { |
2063 | struct xfrm_pol_inexact_key k = { | |
2064 | .family = family, | |
2065 | .type = type, | |
2066 | .dir = dir, | |
b5fe22e2 | 2067 | .if_id = if_id, |
24969fac FW |
2068 | }; |
2069 | ||
2070 | write_pnet(&k.net, net); | |
2071 | ||
2072 | return rhashtable_lookup(&xfrm_policy_inexact_table, &k, | |
2073 | xfrm_pol_inexact_params); | |
2074 | } | |
2075 | ||
2076 | static struct xfrm_pol_inexact_bin * | |
b5fe22e2 FW |
2077 | xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, |
2078 | u8 dir, u32 if_id) | |
24969fac FW |
2079 | { |
2080 | struct xfrm_pol_inexact_bin *bin; | |
2081 | ||
2082 | lockdep_assert_held(&net->xfrm.xfrm_policy_lock); | |
2083 | ||
2084 | rcu_read_lock(); | |
b5fe22e2 | 2085 | bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); |
24969fac FW |
2086 | rcu_read_unlock(); |
2087 | ||
2088 | return bin; | |
2089 | } | |
2090 | ||
6be3b0db FW |
2091 | static struct xfrm_policy * |
2092 | __xfrm_policy_eval_candidates(struct hlist_head *chain, | |
2093 | struct xfrm_policy *prefer, | |
2094 | const struct flowi *fl, | |
0de19788 | 2095 | u8 type, u16 family, u32 if_id) |
6be3b0db FW |
2096 | { |
2097 | u32 priority = prefer ? prefer->priority : ~0u; | |
2098 | struct xfrm_policy *pol; | |
2099 | ||
2100 | if (!chain) | |
2101 | return NULL; | |
2102 | ||
2103 | hlist_for_each_entry_rcu(pol, chain, bydst) { | |
2104 | int err; | |
2105 | ||
2106 | if (pol->priority > priority) | |
2107 | break; | |
2108 | ||
0de19788 | 2109 | err = xfrm_policy_match(pol, fl, type, family, if_id); |
6be3b0db FW |
2110 | if (err) { |
2111 | if (err != -ESRCH) | |
2112 | return ERR_PTR(err); | |
2113 | ||
2114 | continue; | |
2115 | } | |
2116 | ||
2117 | if (prefer) { | |
2118 | /* matches. Is it older than *prefer? */ | |
2119 | if (pol->priority == priority && | |
2120 | prefer->pos < pol->pos) | |
2121 | return prefer; | |
2122 | } | |
2123 | ||
2124 | return pol; | |
2125 | } | |
2126 | ||
2127 | return NULL; | |
2128 | } | |
2129 | ||
2130 | static struct xfrm_policy * | |
2131 | xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand, | |
2132 | struct xfrm_policy *prefer, | |
2133 | const struct flowi *fl, | |
0de19788 | 2134 | u8 type, u16 family, u32 if_id) |
6be3b0db FW |
2135 | { |
2136 | struct xfrm_policy *tmp; | |
2137 | int i; | |
2138 | ||
2139 | for (i = 0; i < ARRAY_SIZE(cand->res); i++) { | |
2140 | tmp = __xfrm_policy_eval_candidates(cand->res[i], | |
2141 | prefer, | |
0de19788 | 2142 | fl, type, family, if_id); |
6be3b0db FW |
2143 | if (!tmp) |
2144 | continue; | |
2145 | ||
2146 | if (IS_ERR(tmp)) | |
2147 | return tmp; | |
2148 | prefer = tmp; | |
2149 | } | |
2150 | ||
2151 | return prefer; | |
2152 | } | |
2153 | ||
52479b62 | 2154 | static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, |
062cdb43 | 2155 | const struct flowi *fl, |
bc56b334 BW |
2156 | u16 family, u8 dir, |
2157 | u32 if_id) | |
2518c7c2 | 2158 | { |
6be3b0db | 2159 | struct xfrm_pol_inexact_candidates cand; |
0b597e7e | 2160 | const xfrm_address_t *daddr, *saddr; |
24969fac FW |
2161 | struct xfrm_pol_inexact_bin *bin; |
2162 | struct xfrm_policy *pol, *ret; | |
2518c7c2 | 2163 | struct hlist_head *chain; |
30846090 | 2164 | unsigned int sequence; |
24969fac | 2165 | int err; |
df71837d | 2166 | |
2518c7c2 DM |
2167 | daddr = xfrm_flowi_daddr(fl, family); |
2168 | saddr = xfrm_flowi_saddr(fl, family); | |
2169 | if (unlikely(!daddr || !saddr)) | |
2170 | return NULL; | |
2171 | ||
d7b04089 | 2172 | rcu_read_lock(); |
eaf22826 SK |
2173 | retry: |
2174 | do { | |
2580d3f4 | 2175 | sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation); |
eaf22826 | 2176 | chain = policy_hash_direct(net, daddr, saddr, family, dir); |
2580d3f4 | 2177 | } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)); |
30846090 | 2178 | |
2518c7c2 | 2179 | ret = NULL; |
a5eefc1d | 2180 | hlist_for_each_entry_rcu(pol, chain, bydst) { |
0de19788 | 2181 | err = xfrm_policy_match(pol, fl, type, family, if_id); |
134b0fc5 JM |
2182 | if (err) { |
2183 | if (err == -ESRCH) | |
2184 | continue; | |
2185 | else { | |
2186 | ret = ERR_PTR(err); | |
2187 | goto fail; | |
2188 | } | |
2189 | } else { | |
2518c7c2 DM |
2190 | ret = pol; |
2191 | break; | |
2192 | } | |
2193 | } | |
3c611d40 LR |
2194 | if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET) |
2195 | goto skip_inexact; | |
2196 | ||
b5fe22e2 | 2197 | bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id); |
6be3b0db FW |
2198 | if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr, |
2199 | daddr)) | |
24969fac | 2200 | goto skip_inexact; |
8faf491e | 2201 | |
6be3b0db | 2202 | pol = xfrm_policy_eval_candidates(&cand, ret, fl, type, |
0de19788 | 2203 | family, if_id); |
6be3b0db FW |
2204 | if (pol) { |
2205 | ret = pol; | |
2206 | if (IS_ERR(pol)) | |
2207 | goto fail; | |
1da177e4 | 2208 | } |
586f2eb4 | 2209 | |
24969fac | 2210 | skip_inexact: |
2580d3f4 | 2211 | if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence)) |
30846090 FW |
2212 | goto retry; |
2213 | ||
eaf22826 | 2214 | if (ret && !xfrm_pol_hold_rcu(ret)) |
e37cc8ad | 2215 | goto retry; |
134b0fc5 | 2216 | fail: |
a7c44247 | 2217 | rcu_read_unlock(); |
4e81bb83 | 2218 | |
2518c7c2 | 2219 | return ret; |
4e81bb83 MN |
2220 | } |
2221 | ||
bc56b334 BW |
2222 | static struct xfrm_policy *xfrm_policy_lookup(struct net *net, |
2223 | const struct flowi *fl, | |
2224 | u16 family, u8 dir, u32 if_id) | |
80c802f3 TT |
2225 | { |
2226 | #ifdef CONFIG_XFRM_SUB_POLICY | |
2227 | struct xfrm_policy *pol; | |
2228 | ||
bc56b334 BW |
2229 | pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, |
2230 | dir, if_id); | |
80c802f3 TT |
2231 | if (pol != NULL) |
2232 | return pol; | |
2233 | #endif | |
bc56b334 BW |
2234 | return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, |
2235 | dir, if_id); | |
80c802f3 TT |
2236 | } |
2237 | ||
6f9c9615 | 2238 | static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir, |
bc56b334 BW |
2239 | const struct flowi *fl, |
2240 | u16 family, u32 if_id) | |
1da177e4 LT |
2241 | { |
2242 | struct xfrm_policy *pol; | |
2243 | ||
d188ba86 | 2244 | rcu_read_lock(); |
ae33786f | 2245 | again: |
d188ba86 ED |
2246 | pol = rcu_dereference(sk->sk_policy[dir]); |
2247 | if (pol != NULL) { | |
ddc47e44 | 2248 | bool match; |
a716c119 | 2249 | int err = 0; |
df71837d | 2250 | |
ddc47e44 SK |
2251 | if (pol->family != family) { |
2252 | pol = NULL; | |
2253 | goto out; | |
2254 | } | |
2255 | ||
2256 | match = xfrm_selector_match(&pol->selector, fl, family); | |
3bccfbc7 | 2257 | if (match) { |
3c5b4d69 | 2258 | if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v || |
bc56b334 | 2259 | pol->if_id != if_id) { |
34f8d884 JHS |
2260 | pol = NULL; |
2261 | goto out; | |
2262 | } | |
03e1ad7b | 2263 | err = security_xfrm_policy_lookup(pol->security, |
8a922805 | 2264 | fl->flowi_secid); |
330e832a FW |
2265 | if (!err) { |
2266 | if (!xfrm_pol_hold_rcu(pol)) | |
2267 | goto again; | |
2268 | } else if (err == -ESRCH) { | |
3bccfbc7 | 2269 | pol = NULL; |
330e832a | 2270 | } else { |
3bccfbc7 | 2271 | pol = ERR_PTR(err); |
330e832a | 2272 | } |
3bccfbc7 | 2273 | } else |
1da177e4 LT |
2274 | pol = NULL; |
2275 | } | |
34f8d884 | 2276 | out: |
d188ba86 | 2277 | rcu_read_unlock(); |
1da177e4 LT |
2278 | return pol; |
2279 | } | |
2280 | ||
2281 | static void __xfrm_policy_link(struct xfrm_policy *pol, int dir) | |
2282 | { | |
98806f75 | 2283 | struct net *net = xp_net(pol); |
4e81bb83 | 2284 | |
98806f75 | 2285 | list_add(&pol->walk.all, &net->xfrm.policy_all); |
98806f75 | 2286 | net->xfrm.policy_count[dir]++; |
1da177e4 LT |
2287 | xfrm_pol_hold(pol); |
2288 | } | |
2289 | ||
2290 | static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol, | |
2291 | int dir) | |
2292 | { | |
98806f75 AD |
2293 | struct net *net = xp_net(pol); |
2294 | ||
53c2e285 | 2295 | if (list_empty(&pol->walk.all)) |
2518c7c2 | 2296 | return NULL; |
1da177e4 | 2297 | |
53c2e285 HX |
2298 | /* Socket policies are not hashed. */ |
2299 | if (!hlist_unhashed(&pol->bydst)) { | |
a5eefc1d | 2300 | hlist_del_rcu(&pol->bydst); |
24969fac | 2301 | hlist_del_init(&pol->bydst_inexact_list); |
53c2e285 HX |
2302 | hlist_del(&pol->byidx); |
2303 | } | |
2304 | ||
2305 | list_del_init(&pol->walk.all); | |
98806f75 | 2306 | net->xfrm.policy_count[dir]--; |
2518c7c2 DM |
2307 | |
2308 | return pol; | |
1da177e4 LT |
2309 | } |
2310 | ||
53c2e285 HX |
2311 | static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir) |
2312 | { | |
2313 | __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir); | |
2314 | } | |
2315 | ||
2316 | static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir) | |
2317 | { | |
2318 | __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir); | |
2319 | } | |
2320 | ||
4666faab | 2321 | int xfrm_policy_delete(struct xfrm_policy *pol, int dir) |
1da177e4 | 2322 | { |
283bc9f3 FD |
2323 | struct net *net = xp_net(pol); |
2324 | ||
9d0380df | 2325 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 | 2326 | pol = __xfrm_policy_unlink(pol, dir); |
9d0380df | 2327 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 | 2328 | if (pol) { |
919e43fa | 2329 | xfrm_dev_policy_delete(pol); |
1da177e4 | 2330 | xfrm_policy_kill(pol); |
4666faab | 2331 | return 0; |
1da177e4 | 2332 | } |
4666faab | 2333 | return -ENOENT; |
1da177e4 | 2334 | } |
a70fcb0b | 2335 | EXPORT_SYMBOL(xfrm_policy_delete); |
1da177e4 LT |
2336 | |
2337 | int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol) | |
2338 | { | |
be8f8284 | 2339 | struct net *net = sock_net(sk); |
1da177e4 LT |
2340 | struct xfrm_policy *old_pol; |
2341 | ||
4e81bb83 MN |
2342 | #ifdef CONFIG_XFRM_SUB_POLICY |
2343 | if (pol && pol->type != XFRM_POLICY_TYPE_MAIN) | |
2344 | return -EINVAL; | |
2345 | #endif | |
2346 | ||
9d0380df | 2347 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
d188ba86 ED |
2348 | old_pol = rcu_dereference_protected(sk->sk_policy[dir], |
2349 | lockdep_is_held(&net->xfrm.xfrm_policy_lock)); | |
1da177e4 | 2350 | if (pol) { |
386c5680 | 2351 | pol->curlft.add_time = ktime_get_real_seconds(); |
e682adf0 | 2352 | pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0); |
53c2e285 | 2353 | xfrm_sk_policy_link(pol, dir); |
1da177e4 | 2354 | } |
d188ba86 | 2355 | rcu_assign_pointer(sk->sk_policy[dir], pol); |
a0073fe1 SK |
2356 | if (old_pol) { |
2357 | if (pol) | |
2358 | xfrm_policy_requeue(old_pol, pol); | |
2359 | ||
ea2dea9d TT |
2360 | /* Unlinking succeeds always. This is the only function |
2361 | * allowed to delete or replace socket policy. | |
2362 | */ | |
53c2e285 | 2363 | xfrm_sk_policy_unlink(old_pol, dir); |
a0073fe1 | 2364 | } |
9d0380df | 2365 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 LT |
2366 | |
2367 | if (old_pol) { | |
2368 | xfrm_policy_kill(old_pol); | |
2369 | } | |
2370 | return 0; | |
2371 | } | |
2372 | ||
d3e40a9f | 2373 | static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir) |
1da177e4 | 2374 | { |
0331b1f3 | 2375 | struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC); |
283bc9f3 | 2376 | struct net *net = xp_net(old); |
1da177e4 LT |
2377 | |
2378 | if (newp) { | |
2379 | newp->selector = old->selector; | |
03e1ad7b PM |
2380 | if (security_xfrm_policy_clone(old->security, |
2381 | &newp->security)) { | |
df71837d TJ |
2382 | kfree(newp); |
2383 | return NULL; /* ENOMEM */ | |
2384 | } | |
1da177e4 LT |
2385 | newp->lft = old->lft; |
2386 | newp->curlft = old->curlft; | |
fb977e2c | 2387 | newp->mark = old->mark; |
7e652640 | 2388 | newp->if_id = old->if_id; |
1da177e4 LT |
2389 | newp->action = old->action; |
2390 | newp->flags = old->flags; | |
2391 | newp->xfrm_nr = old->xfrm_nr; | |
2392 | newp->index = old->index; | |
4e81bb83 | 2393 | newp->type = old->type; |
0e74aa1d | 2394 | newp->family = old->family; |
1da177e4 LT |
2395 | memcpy(newp->xfrm_vec, old->xfrm_vec, |
2396 | newp->xfrm_nr*sizeof(struct xfrm_tmpl)); | |
9d0380df | 2397 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
53c2e285 | 2398 | xfrm_sk_policy_link(newp, dir); |
9d0380df | 2399 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
1da177e4 LT |
2400 | xfrm_pol_put(newp); |
2401 | } | |
2402 | return newp; | |
2403 | } | |
2404 | ||
d188ba86 | 2405 | int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk) |
1da177e4 | 2406 | { |
d188ba86 ED |
2407 | const struct xfrm_policy *p; |
2408 | struct xfrm_policy *np; | |
2409 | int i, ret = 0; | |
2410 | ||
2411 | rcu_read_lock(); | |
2412 | for (i = 0; i < 2; i++) { | |
2413 | p = rcu_dereference(osk->sk_policy[i]); | |
2414 | if (p) { | |
2415 | np = clone_policy(p, i); | |
2416 | if (unlikely(!np)) { | |
2417 | ret = -ENOMEM; | |
2418 | break; | |
2419 | } | |
2420 | rcu_assign_pointer(sk->sk_policy[i], np); | |
2421 | } | |
2422 | } | |
2423 | rcu_read_unlock(); | |
2424 | return ret; | |
1da177e4 LT |
2425 | } |
2426 | ||
a1e59abf | 2427 | static int |
42a7b32b | 2428 | xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local, |
077fbac4 | 2429 | xfrm_address_t *remote, unsigned short family, u32 mark) |
a1e59abf PM |
2430 | { |
2431 | int err; | |
37b10383 | 2432 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
a1e59abf PM |
2433 | |
2434 | if (unlikely(afinfo == NULL)) | |
2435 | return -EINVAL; | |
077fbac4 | 2436 | err = afinfo->get_saddr(net, oif, local, remote, mark); |
bdba9fe0 | 2437 | rcu_read_unlock(); |
a1e59abf PM |
2438 | return err; |
2439 | } | |
2440 | ||
1da177e4 LT |
2441 | /* Resolve list of templates for the flow, given policy. */ |
2442 | ||
2443 | static int | |
a6c2e611 DM |
2444 | xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl, |
2445 | struct xfrm_state **xfrm, unsigned short family) | |
1da177e4 | 2446 | { |
fbda33b2 | 2447 | struct net *net = xp_net(policy); |
1da177e4 LT |
2448 | int nx; |
2449 | int i, error; | |
94802151 SK |
2450 | xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family); |
2451 | xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family); | |
a1e59abf | 2452 | xfrm_address_t tmp; |
1da177e4 | 2453 | |
9b7a787d | 2454 | for (nx = 0, i = 0; i < policy->xfrm_nr; i++) { |
1da177e4 | 2455 | struct xfrm_state *x; |
94802151 SK |
2456 | xfrm_address_t *remote = daddr; |
2457 | xfrm_address_t *local = saddr; | |
1da177e4 LT |
2458 | struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i]; |
2459 | ||
94802151 SK |
2460 | if (tmpl->mode == XFRM_MODE_TUNNEL || |
2461 | tmpl->mode == XFRM_MODE_BEET) { | |
2462 | remote = &tmpl->id.daddr; | |
2463 | local = &tmpl->saddr; | |
2464 | if (xfrm_addr_any(local, tmpl->encap_family)) { | |
2465 | error = xfrm_get_saddr(net, fl->flowi_oif, | |
2466 | &tmp, remote, | |
2467 | tmpl->encap_family, 0); | |
2468 | if (error) | |
2469 | goto fail; | |
2470 | local = &tmp; | |
2471 | } | |
1da177e4 LT |
2472 | } |
2473 | ||
bc56b334 BW |
2474 | x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, |
2475 | family, policy->if_id); | |
1da177e4 LT |
2476 | |
2477 | if (x && x->km.state == XFRM_STATE_VALID) { | |
2478 | xfrm[nx++] = x; | |
94802151 SK |
2479 | daddr = remote; |
2480 | saddr = local; | |
1da177e4 LT |
2481 | continue; |
2482 | } | |
2483 | if (x) { | |
2484 | error = (x->km.state == XFRM_STATE_ERROR ? | |
2485 | -EINVAL : -EAGAIN); | |
2486 | xfrm_state_put(x); | |
42054569 | 2487 | } else if (error == -ESRCH) { |
a4322266 | 2488 | error = -EAGAIN; |
42054569 | 2489 | } |
1da177e4 LT |
2490 | |
2491 | if (!tmpl->optional) | |
2492 | goto fail; | |
2493 | } | |
2494 | return nx; | |
2495 | ||
2496 | fail: | |
9b7a787d | 2497 | for (nx--; nx >= 0; nx--) |
1da177e4 LT |
2498 | xfrm_state_put(xfrm[nx]); |
2499 | return error; | |
2500 | } | |
2501 | ||
4e81bb83 | 2502 | static int |
a6c2e611 DM |
2503 | xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl, |
2504 | struct xfrm_state **xfrm, unsigned short family) | |
4e81bb83 | 2505 | { |
41a49cc3 MN |
2506 | struct xfrm_state *tp[XFRM_MAX_DEPTH]; |
2507 | struct xfrm_state **tpp = (npols > 1) ? tp : xfrm; | |
4e81bb83 MN |
2508 | int cnx = 0; |
2509 | int error; | |
2510 | int ret; | |
2511 | int i; | |
2512 | ||
2513 | for (i = 0; i < npols; i++) { | |
2514 | if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) { | |
2515 | error = -ENOBUFS; | |
2516 | goto fail; | |
2517 | } | |
41a49cc3 MN |
2518 | |
2519 | ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family); | |
4e81bb83 MN |
2520 | if (ret < 0) { |
2521 | error = ret; | |
2522 | goto fail; | |
2523 | } else | |
2524 | cnx += ret; | |
2525 | } | |
2526 | ||
41a49cc3 MN |
2527 | /* found states are sorted for outbound processing */ |
2528 | if (npols > 1) | |
2529 | xfrm_state_sort(xfrm, tpp, cnx, family); | |
2530 | ||
4e81bb83 MN |
2531 | return cnx; |
2532 | ||
2533 | fail: | |
9b7a787d | 2534 | for (cnx--; cnx >= 0; cnx--) |
41a49cc3 | 2535 | xfrm_state_put(tpp[cnx]); |
4e81bb83 MN |
2536 | return error; |
2537 | ||
2538 | } | |
2539 | ||
f5e2bb4f | 2540 | static int xfrm_get_tos(const struct flowi *fl, int family) |
25ee3286 | 2541 | { |
f24ea528 FW |
2542 | if (family == AF_INET) |
2543 | return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos; | |
143a4454 | 2544 | |
f24ea528 | 2545 | return 0; |
25ee3286 HX |
2546 | } |
2547 | ||
d7c7544c | 2548 | static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family) |
1da177e4 | 2549 | { |
37b10383 | 2550 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
d7c7544c | 2551 | struct dst_ops *dst_ops; |
25ee3286 HX |
2552 | struct xfrm_dst *xdst; |
2553 | ||
2554 | if (!afinfo) | |
2555 | return ERR_PTR(-EINVAL); | |
2556 | ||
d7c7544c AD |
2557 | switch (family) { |
2558 | case AF_INET: | |
2559 | dst_ops = &net->xfrm.xfrm4_dst_ops; | |
2560 | break; | |
dfd56b8b | 2561 | #if IS_ENABLED(CONFIG_IPV6) |
d7c7544c AD |
2562 | case AF_INET6: |
2563 | dst_ops = &net->xfrm.xfrm6_dst_ops; | |
2564 | break; | |
2565 | #endif | |
2566 | default: | |
2567 | BUG(); | |
2568 | } | |
b2a9c0ed | 2569 | xdst = dst_alloc(dst_ops, NULL, 1, DST_OBSOLETE_NONE, 0); |
25ee3286 | 2570 | |
d4cae562 | 2571 | if (likely(xdst)) { |
caf283d0 | 2572 | memset_after(xdst, 0, u.dst); |
d4cae562 | 2573 | } else |
0b150932 | 2574 | xdst = ERR_PTR(-ENOBUFS); |
80c802f3 | 2575 | |
bdba9fe0 | 2576 | rcu_read_unlock(); |
d4cae562 | 2577 | |
25ee3286 HX |
2578 | return xdst; |
2579 | } | |
2580 | ||
2e8b4aa8 FW |
2581 | static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst, |
2582 | int nfheader_len) | |
a1b05140 | 2583 | { |
2e8b4aa8 FW |
2584 | if (dst->ops->family == AF_INET6) { |
2585 | struct rt6_info *rt = (struct rt6_info *)dst; | |
2586 | path->path_cookie = rt6_get_cookie(rt); | |
2587 | path->u.rt6.rt6i_nfheader_len = nfheader_len; | |
2588 | } | |
a1b05140 MN |
2589 | } |
2590 | ||
87c1e12b | 2591 | static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, |
0c7b3eef | 2592 | const struct flowi *fl) |
25ee3286 | 2593 | { |
37b10383 | 2594 | const struct xfrm_policy_afinfo *afinfo = |
25ee3286 HX |
2595 | xfrm_policy_get_afinfo(xdst->u.dst.ops->family); |
2596 | int err; | |
2597 | ||
2598 | if (!afinfo) | |
1da177e4 | 2599 | return -EINVAL; |
25ee3286 | 2600 | |
87c1e12b | 2601 | err = afinfo->fill_dst(xdst, dev, fl); |
25ee3286 | 2602 | |
bdba9fe0 | 2603 | rcu_read_unlock(); |
25ee3286 | 2604 | |
1da177e4 LT |
2605 | return err; |
2606 | } | |
2607 | ||
80c802f3 | 2608 | |
25ee3286 HX |
2609 | /* Allocate chain of dst_entry's, attach known xfrm's, calculate |
2610 | * all the metrics... Shortly, bundle a bundle. | |
2611 | */ | |
2612 | ||
2613 | static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy, | |
5492093d DM |
2614 | struct xfrm_state **xfrm, |
2615 | struct xfrm_dst **bundle, | |
2616 | int nx, | |
98313ada | 2617 | const struct flowi *fl, |
25ee3286 HX |
2618 | struct dst_entry *dst) |
2619 | { | |
733a5fac | 2620 | const struct xfrm_state_afinfo *afinfo; |
4c145dce | 2621 | const struct xfrm_mode *inner_mode; |
d7c7544c | 2622 | struct net *net = xp_net(policy); |
25ee3286 HX |
2623 | unsigned long now = jiffies; |
2624 | struct net_device *dev; | |
45b018be DM |
2625 | struct xfrm_dst *xdst_prev = NULL; |
2626 | struct xfrm_dst *xdst0 = NULL; | |
25ee3286 HX |
2627 | int i = 0; |
2628 | int err; | |
2629 | int header_len = 0; | |
a1b05140 | 2630 | int nfheader_len = 0; |
25ee3286 HX |
2631 | int trailer_len = 0; |
2632 | int tos; | |
2633 | int family = policy->selector.family; | |
9bb182a7 YH |
2634 | xfrm_address_t saddr, daddr; |
2635 | ||
2636 | xfrm_flowi_addr_get(fl, &saddr, &daddr, family); | |
25ee3286 HX |
2637 | |
2638 | tos = xfrm_get_tos(fl, family); | |
25ee3286 HX |
2639 | |
2640 | dst_hold(dst); | |
2641 | ||
2642 | for (; i < nx; i++) { | |
d7c7544c | 2643 | struct xfrm_dst *xdst = xfrm_alloc_dst(net, family); |
25ee3286 HX |
2644 | struct dst_entry *dst1 = &xdst->u.dst; |
2645 | ||
2646 | err = PTR_ERR(xdst); | |
2647 | if (IS_ERR(xdst)) { | |
2648 | dst_release(dst); | |
2649 | goto put_states; | |
2650 | } | |
2651 | ||
5492093d | 2652 | bundle[i] = xdst; |
45b018be DM |
2653 | if (!xdst_prev) |
2654 | xdst0 = xdst; | |
10a7ef33 DM |
2655 | else |
2656 | /* Ref count is taken during xfrm_alloc_dst() | |
2657 | * No need to do dst_clone() on dst1 | |
2658 | */ | |
45b018be | 2659 | xfrm_dst_set_child(xdst_prev, &xdst->u.dst); |
10a7ef33 | 2660 | |
43a4dea4 SK |
2661 | if (xfrm[i]->sel.family == AF_UNSPEC) { |
2662 | inner_mode = xfrm_ip2inner_mode(xfrm[i], | |
2663 | xfrm_af2proto(family)); | |
2664 | if (!inner_mode) { | |
2665 | err = -EAFNOSUPPORT; | |
2666 | dst_release(dst); | |
2667 | goto put_states; | |
2668 | } | |
2669 | } else | |
c9500d7b | 2670 | inner_mode = &xfrm[i]->inner_mode; |
43a4dea4 | 2671 | |
25ee3286 | 2672 | xdst->route = dst; |
defb3519 | 2673 | dst_copy_metrics(dst1, dst); |
25ee3286 HX |
2674 | |
2675 | if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) { | |
e2612cd4 | 2676 | __u32 mark = 0; |
748b82c2 | 2677 | int oif; |
e2612cd4 BW |
2678 | |
2679 | if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m) | |
2680 | mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]); | |
9b42c1f1 | 2681 | |
25ee3286 | 2682 | family = xfrm[i]->props.family; |
748b82c2 DA |
2683 | oif = fl->flowi_oif ? : fl->flowi_l3mdev; |
2684 | dst = xfrm_dst_lookup(xfrm[i], tos, oif, | |
9b42c1f1 | 2685 | &saddr, &daddr, family, mark); |
25ee3286 HX |
2686 | err = PTR_ERR(dst); |
2687 | if (IS_ERR(dst)) | |
2688 | goto put_states; | |
2689 | } else | |
2690 | dst_hold(dst); | |
2691 | ||
2692 | dst1->xfrm = xfrm[i]; | |
80c802f3 | 2693 | xdst->xfrm_genid = xfrm[i]->genid; |
25ee3286 | 2694 | |
f5b0a874 | 2695 | dst1->obsolete = DST_OBSOLETE_FORCE_CHK; |
25ee3286 HX |
2696 | dst1->lastuse = now; |
2697 | ||
2698 | dst1->input = dst_discard; | |
733a5fac FW |
2699 | |
2700 | rcu_read_lock(); | |
2701 | afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family); | |
2702 | if (likely(afinfo)) | |
2703 | dst1->output = afinfo->output; | |
2704 | else | |
2705 | dst1->output = dst_discard_out; | |
2706 | rcu_read_unlock(); | |
25ee3286 | 2707 | |
45b018be | 2708 | xdst_prev = xdst; |
25ee3286 HX |
2709 | |
2710 | header_len += xfrm[i]->props.header_len; | |
a1b05140 MN |
2711 | if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT) |
2712 | nfheader_len += xfrm[i]->props.header_len; | |
25ee3286 HX |
2713 | trailer_len += xfrm[i]->props.trailer_len; |
2714 | } | |
2715 | ||
45b018be | 2716 | xfrm_dst_set_child(xdst_prev, dst); |
0f6c480f | 2717 | xdst0->path = dst; |
25ee3286 HX |
2718 | |
2719 | err = -ENODEV; | |
2720 | dev = dst->dev; | |
2721 | if (!dev) | |
2722 | goto free_dst; | |
2723 | ||
45b018be | 2724 | xfrm_init_path(xdst0, dst, nfheader_len); |
5492093d | 2725 | xfrm_init_pmtu(bundle, nx); |
25ee3286 | 2726 | |
45b018be DM |
2727 | for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst; |
2728 | xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) { | |
2729 | err = xfrm_fill_dst(xdst_prev, dev, fl); | |
25ee3286 HX |
2730 | if (err) |
2731 | goto free_dst; | |
2732 | ||
45b018be DM |
2733 | xdst_prev->u.dst.header_len = header_len; |
2734 | xdst_prev->u.dst.trailer_len = trailer_len; | |
2735 | header_len -= xdst_prev->u.dst.xfrm->props.header_len; | |
2736 | trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len; | |
25ee3286 HX |
2737 | } |
2738 | ||
45b018be | 2739 | return &xdst0->u.dst; |
25ee3286 HX |
2740 | |
2741 | put_states: | |
2742 | for (; i < nx; i++) | |
2743 | xfrm_state_put(xfrm[i]); | |
2744 | free_dst: | |
45b018be DM |
2745 | if (xdst0) |
2746 | dst_release_immediate(&xdst0->u.dst); | |
38369f54 SK |
2747 | |
2748 | return ERR_PTR(err); | |
25ee3286 HX |
2749 | } |
2750 | ||
73ff93cd | 2751 | static int xfrm_expand_policies(const struct flowi *fl, u16 family, |
80c802f3 TT |
2752 | struct xfrm_policy **pols, |
2753 | int *num_pols, int *num_xfrms) | |
2754 | { | |
2755 | int i; | |
2756 | ||
2757 | if (*num_pols == 0 || !pols[0]) { | |
2758 | *num_pols = 0; | |
2759 | *num_xfrms = 0; | |
2760 | return 0; | |
2761 | } | |
f85daf0e HH |
2762 | if (IS_ERR(pols[0])) { |
2763 | *num_pols = 0; | |
80c802f3 | 2764 | return PTR_ERR(pols[0]); |
f85daf0e | 2765 | } |
80c802f3 TT |
2766 | |
2767 | *num_xfrms = pols[0]->xfrm_nr; | |
2768 | ||
2769 | #ifdef CONFIG_XFRM_SUB_POLICY | |
ac1077e9 | 2770 | if (pols[0]->action == XFRM_POLICY_ALLOW && |
80c802f3 TT |
2771 | pols[0]->type != XFRM_POLICY_TYPE_MAIN) { |
2772 | pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]), | |
2773 | XFRM_POLICY_TYPE_MAIN, | |
2774 | fl, family, | |
bc56b334 BW |
2775 | XFRM_POLICY_OUT, |
2776 | pols[0]->if_id); | |
80c802f3 TT |
2777 | if (pols[1]) { |
2778 | if (IS_ERR(pols[1])) { | |
2779 | xfrm_pols_put(pols, *num_pols); | |
f85daf0e | 2780 | *num_pols = 0; |
80c802f3 TT |
2781 | return PTR_ERR(pols[1]); |
2782 | } | |
02d0892f | 2783 | (*num_pols)++; |
80c802f3 TT |
2784 | (*num_xfrms) += pols[1]->xfrm_nr; |
2785 | } | |
2786 | } | |
2787 | #endif | |
2788 | for (i = 0; i < *num_pols; i++) { | |
2789 | if (pols[i]->action != XFRM_POLICY_ALLOW) { | |
2790 | *num_xfrms = -1; | |
2791 | break; | |
2792 | } | |
2793 | } | |
2794 | ||
2795 | return 0; | |
2796 | ||
2797 | } | |
2798 | ||
2799 | static struct xfrm_dst * | |
2800 | xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols, | |
4ca2e685 | 2801 | const struct flowi *fl, u16 family, |
80c802f3 TT |
2802 | struct dst_entry *dst_orig) |
2803 | { | |
2804 | struct net *net = xp_net(pols[0]); | |
2805 | struct xfrm_state *xfrm[XFRM_MAX_DEPTH]; | |
5492093d | 2806 | struct xfrm_dst *bundle[XFRM_MAX_DEPTH]; |
e4db5b61 | 2807 | struct xfrm_dst *xdst; |
80c802f3 | 2808 | struct dst_entry *dst; |
80c802f3 TT |
2809 | int err; |
2810 | ||
cf379667 FW |
2811 | /* Try to instantiate a bundle */ |
2812 | err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family); | |
2813 | if (err <= 0) { | |
934ffce1 Y |
2814 | if (err == 0) |
2815 | return NULL; | |
2816 | ||
2817 | if (err != -EAGAIN) | |
cf379667 FW |
2818 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); |
2819 | return ERR_PTR(err); | |
2820 | } | |
2821 | ||
5492093d | 2822 | dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig); |
80c802f3 TT |
2823 | if (IS_ERR(dst)) { |
2824 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR); | |
2825 | return ERR_CAST(dst); | |
2826 | } | |
2827 | ||
2828 | xdst = (struct xfrm_dst *)dst; | |
2829 | xdst->num_xfrms = err; | |
80c802f3 | 2830 | xdst->num_pols = num_pols; |
3e94c2dc | 2831 | memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); |
80c802f3 TT |
2832 | xdst->policy_genid = atomic_read(&pols[0]->genid); |
2833 | ||
2834 | return xdst; | |
2835 | } | |
2836 | ||
c3aed709 | 2837 | static void xfrm_policy_queue_process(struct timer_list *t) |
a0073fe1 | 2838 | { |
a0073fe1 SK |
2839 | struct sk_buff *skb; |
2840 | struct sock *sk; | |
2841 | struct dst_entry *dst; | |
c3aed709 | 2842 | struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer); |
3f5312ae | 2843 | struct net *net = xp_net(pol); |
a0073fe1 SK |
2844 | struct xfrm_policy_queue *pq = &pol->polq; |
2845 | struct flowi fl; | |
2846 | struct sk_buff_head list; | |
b328ecc4 | 2847 | __u32 skb_mark; |
a0073fe1 SK |
2848 | |
2849 | spin_lock(&pq->hold_queue.lock); | |
2850 | skb = skb_peek(&pq->hold_queue); | |
2bb53e25 SK |
2851 | if (!skb) { |
2852 | spin_unlock(&pq->hold_queue.lock); | |
2853 | goto out; | |
2854 | } | |
a0073fe1 SK |
2855 | dst = skb_dst(skb); |
2856 | sk = skb->sk; | |
b328ecc4 SK |
2857 | |
2858 | /* Fixup the mark to support VTI. */ | |
2859 | skb_mark = skb->mark; | |
2860 | skb->mark = pol->mark.v; | |
a0073fe1 | 2861 | xfrm_decode_session(skb, &fl, dst->ops->family); |
b328ecc4 | 2862 | skb->mark = skb_mark; |
a0073fe1 SK |
2863 | spin_unlock(&pq->hold_queue.lock); |
2864 | ||
0f6c480f | 2865 | dst_hold(xfrm_dst_path(dst)); |
2471c981 | 2866 | dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE); |
a0073fe1 SK |
2867 | if (IS_ERR(dst)) |
2868 | goto purge_queue; | |
2869 | ||
2870 | if (dst->flags & DST_XFRM_QUEUE) { | |
2871 | dst_release(dst); | |
2872 | ||
2873 | if (pq->timeout >= XFRM_QUEUE_TMO_MAX) | |
2874 | goto purge_queue; | |
2875 | ||
2876 | pq->timeout = pq->timeout << 1; | |
e7d8f6cb SK |
2877 | if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout)) |
2878 | xfrm_pol_hold(pol); | |
7759d6a8 | 2879 | goto out; |
a0073fe1 SK |
2880 | } |
2881 | ||
2882 | dst_release(dst); | |
2883 | ||
2884 | __skb_queue_head_init(&list); | |
2885 | ||
2886 | spin_lock(&pq->hold_queue.lock); | |
2887 | pq->timeout = 0; | |
2888 | skb_queue_splice_init(&pq->hold_queue, &list); | |
2889 | spin_unlock(&pq->hold_queue.lock); | |
2890 | ||
2891 | while (!skb_queue_empty(&list)) { | |
2892 | skb = __skb_dequeue(&list); | |
2893 | ||
b328ecc4 SK |
2894 | /* Fixup the mark to support VTI. */ |
2895 | skb_mark = skb->mark; | |
2896 | skb->mark = pol->mark.v; | |
a0073fe1 | 2897 | xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family); |
b328ecc4 SK |
2898 | skb->mark = skb_mark; |
2899 | ||
0f6c480f DM |
2900 | dst_hold(xfrm_dst_path(skb_dst(skb))); |
2901 | dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0); | |
a0073fe1 | 2902 | if (IS_ERR(dst)) { |
a0073fe1 SK |
2903 | kfree_skb(skb); |
2904 | continue; | |
2905 | } | |
2906 | ||
895b5c9f | 2907 | nf_reset_ct(skb); |
a0073fe1 SK |
2908 | skb_dst_drop(skb); |
2909 | skb_dst_set(skb, dst); | |
2910 | ||
13206b6b | 2911 | dst_output(net, skb->sk, skb); |
a0073fe1 SK |
2912 | } |
2913 | ||
e7d8f6cb SK |
2914 | out: |
2915 | xfrm_pol_put(pol); | |
a0073fe1 SK |
2916 | return; |
2917 | ||
2918 | purge_queue: | |
2919 | pq->timeout = 0; | |
1ee5e667 | 2920 | skb_queue_purge(&pq->hold_queue); |
e7d8f6cb | 2921 | xfrm_pol_put(pol); |
a0073fe1 SK |
2922 | } |
2923 | ||
ede2059d | 2924 | static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb) |
a0073fe1 SK |
2925 | { |
2926 | unsigned long sched_next; | |
2927 | struct dst_entry *dst = skb_dst(skb); | |
2928 | struct xfrm_dst *xdst = (struct xfrm_dst *) dst; | |
e7d8f6cb SK |
2929 | struct xfrm_policy *pol = xdst->pols[0]; |
2930 | struct xfrm_policy_queue *pq = &pol->polq; | |
4d53eff4 | 2931 | |
39bb5e62 | 2932 | if (unlikely(skb_fclone_busy(sk, skb))) { |
4d53eff4 SK |
2933 | kfree_skb(skb); |
2934 | return 0; | |
2935 | } | |
a0073fe1 SK |
2936 | |
2937 | if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) { | |
2938 | kfree_skb(skb); | |
2939 | return -EAGAIN; | |
2940 | } | |
2941 | ||
2942 | skb_dst_force(skb); | |
a0073fe1 SK |
2943 | |
2944 | spin_lock_bh(&pq->hold_queue.lock); | |
2945 | ||
2946 | if (!pq->timeout) | |
2947 | pq->timeout = XFRM_QUEUE_TMO_MIN; | |
2948 | ||
2949 | sched_next = jiffies + pq->timeout; | |
2950 | ||
2951 | if (del_timer(&pq->hold_timer)) { | |
2952 | if (time_before(pq->hold_timer.expires, sched_next)) | |
2953 | sched_next = pq->hold_timer.expires; | |
e7d8f6cb | 2954 | xfrm_pol_put(pol); |
a0073fe1 SK |
2955 | } |
2956 | ||
2957 | __skb_queue_tail(&pq->hold_queue, skb); | |
e7d8f6cb SK |
2958 | if (!mod_timer(&pq->hold_timer, sched_next)) |
2959 | xfrm_pol_hold(pol); | |
a0073fe1 SK |
2960 | |
2961 | spin_unlock_bh(&pq->hold_queue.lock); | |
2962 | ||
2963 | return 0; | |
2964 | } | |
2965 | ||
2966 | static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net, | |
b8c203b2 | 2967 | struct xfrm_flo *xflo, |
a0073fe1 SK |
2968 | const struct flowi *fl, |
2969 | int num_xfrms, | |
2970 | u16 family) | |
2971 | { | |
2972 | int err; | |
2973 | struct net_device *dev; | |
b8c203b2 | 2974 | struct dst_entry *dst; |
a0073fe1 SK |
2975 | struct dst_entry *dst1; |
2976 | struct xfrm_dst *xdst; | |
2977 | ||
2978 | xdst = xfrm_alloc_dst(net, family); | |
2979 | if (IS_ERR(xdst)) | |
2980 | return xdst; | |
2981 | ||
b8c203b2 SK |
2982 | if (!(xflo->flags & XFRM_LOOKUP_QUEUE) || |
2983 | net->xfrm.sysctl_larval_drop || | |
2984 | num_xfrms <= 0) | |
a0073fe1 SK |
2985 | return xdst; |
2986 | ||
b8c203b2 | 2987 | dst = xflo->dst_orig; |
a0073fe1 SK |
2988 | dst1 = &xdst->u.dst; |
2989 | dst_hold(dst); | |
2990 | xdst->route = dst; | |
2991 | ||
2992 | dst_copy_metrics(dst1, dst); | |
2993 | ||
2994 | dst1->obsolete = DST_OBSOLETE_FORCE_CHK; | |
af13b3c3 | 2995 | dst1->flags |= DST_XFRM_QUEUE; |
a0073fe1 SK |
2996 | dst1->lastuse = jiffies; |
2997 | ||
2998 | dst1->input = dst_discard; | |
2999 | dst1->output = xdst_queue_output; | |
3000 | ||
3001 | dst_hold(dst); | |
45b018be | 3002 | xfrm_dst_set_child(xdst, dst); |
0f6c480f | 3003 | xdst->path = dst; |
a0073fe1 SK |
3004 | |
3005 | xfrm_init_path((struct xfrm_dst *)dst1, dst, 0); | |
3006 | ||
3007 | err = -ENODEV; | |
3008 | dev = dst->dev; | |
3009 | if (!dev) | |
3010 | goto free_dst; | |
3011 | ||
3012 | err = xfrm_fill_dst(xdst, dev, fl); | |
3013 | if (err) | |
3014 | goto free_dst; | |
3015 | ||
3016 | out: | |
3017 | return xdst; | |
3018 | ||
3019 | free_dst: | |
3020 | dst_release(dst1); | |
3021 | xdst = ERR_PTR(err); | |
3022 | goto out; | |
3023 | } | |
3024 | ||
bc56b334 BW |
3025 | static struct xfrm_dst *xfrm_bundle_lookup(struct net *net, |
3026 | const struct flowi *fl, | |
3027 | u16 family, u8 dir, | |
3028 | struct xfrm_flo *xflo, u32 if_id) | |
80c802f3 | 3029 | { |
80c802f3 | 3030 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
855dad99 | 3031 | int num_pols = 0, num_xfrms = 0, err; |
bd45c539 | 3032 | struct xfrm_dst *xdst; |
80c802f3 | 3033 | |
80c802f3 TT |
3034 | /* Resolve policies to use if we couldn't get them from |
3035 | * previous cache entry */ | |
855dad99 | 3036 | num_pols = 1; |
bc56b334 | 3037 | pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id); |
855dad99 | 3038 | err = xfrm_expand_policies(fl, family, pols, |
80c802f3 | 3039 | &num_pols, &num_xfrms); |
855dad99 FW |
3040 | if (err < 0) |
3041 | goto inc_error; | |
3042 | if (num_pols == 0) | |
3043 | return NULL; | |
3044 | if (num_xfrms <= 0) | |
3045 | goto make_dummy_bundle; | |
80c802f3 | 3046 | |
bd45c539 | 3047 | xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, |
76a42011 | 3048 | xflo->dst_orig); |
bd45c539 FW |
3049 | if (IS_ERR(xdst)) { |
3050 | err = PTR_ERR(xdst); | |
f203b76d SK |
3051 | if (err == -EREMOTE) { |
3052 | xfrm_pols_put(pols, num_pols); | |
3053 | return NULL; | |
3054 | } | |
3055 | ||
80c802f3 TT |
3056 | if (err != -EAGAIN) |
3057 | goto error; | |
855dad99 | 3058 | goto make_dummy_bundle; |
bd45c539 | 3059 | } else if (xdst == NULL) { |
d809ec89 | 3060 | num_xfrms = 0; |
855dad99 | 3061 | goto make_dummy_bundle; |
80c802f3 TT |
3062 | } |
3063 | ||
bd45c539 | 3064 | return xdst; |
80c802f3 TT |
3065 | |
3066 | make_dummy_bundle: | |
3067 | /* We found policies, but there's no bundles to instantiate: | |
3068 | * either because the policy blocks, has no transformations or | |
3069 | * we could not build template (no xfrm_states).*/ | |
b8c203b2 | 3070 | xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family); |
80c802f3 TT |
3071 | if (IS_ERR(xdst)) { |
3072 | xfrm_pols_put(pols, num_pols); | |
3073 | return ERR_CAST(xdst); | |
3074 | } | |
3075 | xdst->num_pols = num_pols; | |
3076 | xdst->num_xfrms = num_xfrms; | |
3e94c2dc | 3077 | memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols); |
80c802f3 | 3078 | |
bd45c539 | 3079 | return xdst; |
80c802f3 TT |
3080 | |
3081 | inc_error: | |
3082 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR); | |
3083 | error: | |
855dad99 | 3084 | xfrm_pols_put(pols, num_pols); |
80c802f3 TT |
3085 | return ERR_PTR(err); |
3086 | } | |
1da177e4 | 3087 | |
2774c131 DM |
3088 | static struct dst_entry *make_blackhole(struct net *net, u16 family, |
3089 | struct dst_entry *dst_orig) | |
3090 | { | |
37b10383 | 3091 | const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family); |
2774c131 DM |
3092 | struct dst_entry *ret; |
3093 | ||
3094 | if (!afinfo) { | |
3095 | dst_release(dst_orig); | |
433a1954 | 3096 | return ERR_PTR(-EINVAL); |
2774c131 DM |
3097 | } else { |
3098 | ret = afinfo->blackhole_route(net, dst_orig); | |
3099 | } | |
bdba9fe0 | 3100 | rcu_read_unlock(); |
2774c131 DM |
3101 | |
3102 | return ret; | |
3103 | } | |
3104 | ||
bc56b334 | 3105 | /* Finds/creates a bundle for given flow and if_id |
1da177e4 LT |
3106 | * |
3107 | * At the moment we eat a raw IP route. Mostly to speed up lookups | |
3108 | * on interfaces with disabled IPsec. | |
bc56b334 BW |
3109 | * |
3110 | * xfrm_lookup uses an if_id of 0 by default, and is provided for | |
3111 | * compatibility | |
1da177e4 | 3112 | */ |
bc56b334 BW |
3113 | struct dst_entry *xfrm_lookup_with_ifid(struct net *net, |
3114 | struct dst_entry *dst_orig, | |
3115 | const struct flowi *fl, | |
3116 | const struct sock *sk, | |
3117 | int flags, u32 if_id) | |
1da177e4 | 3118 | { |
4e81bb83 | 3119 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
80c802f3 | 3120 | struct xfrm_dst *xdst; |
452edd59 | 3121 | struct dst_entry *dst, *route; |
80c802f3 | 3122 | u16 family = dst_orig->ops->family; |
aff669bc | 3123 | u8 dir = XFRM_POLICY_OUT; |
4b021628 | 3124 | int i, err, num_pols, num_xfrms = 0, drop_pols = 0; |
e0d1caa7 | 3125 | |
80c802f3 TT |
3126 | dst = NULL; |
3127 | xdst = NULL; | |
3128 | route = NULL; | |
4e81bb83 | 3129 | |
bd5eb35f | 3130 | sk = sk_const_to_full_sk(sk); |
f7944fb1 | 3131 | if (sk && sk->sk_policy[XFRM_POLICY_OUT]) { |
80c802f3 | 3132 | num_pols = 1; |
bc56b334 BW |
3133 | pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family, |
3134 | if_id); | |
80c802f3 TT |
3135 | err = xfrm_expand_policies(fl, family, pols, |
3136 | &num_pols, &num_xfrms); | |
3137 | if (err < 0) | |
75b8c133 | 3138 | goto dropdst; |
80c802f3 TT |
3139 | |
3140 | if (num_pols) { | |
3141 | if (num_xfrms <= 0) { | |
3142 | drop_pols = num_pols; | |
3143 | goto no_transform; | |
3144 | } | |
3145 | ||
3146 | xdst = xfrm_resolve_and_create_bundle( | |
3147 | pols, num_pols, fl, | |
3148 | family, dst_orig); | |
76a42011 | 3149 | |
80c802f3 TT |
3150 | if (IS_ERR(xdst)) { |
3151 | xfrm_pols_put(pols, num_pols); | |
3152 | err = PTR_ERR(xdst); | |
f203b76d SK |
3153 | if (err == -EREMOTE) |
3154 | goto nopol; | |
3155 | ||
80c802f3 | 3156 | goto dropdst; |
d809ec89 TT |
3157 | } else if (xdst == NULL) { |
3158 | num_xfrms = 0; | |
3159 | drop_pols = num_pols; | |
3160 | goto no_transform; | |
80c802f3 TT |
3161 | } |
3162 | ||
80c802f3 | 3163 | route = xdst->route; |
0aa64774 | 3164 | } |
3bccfbc7 | 3165 | } |
1da177e4 | 3166 | |
80c802f3 | 3167 | if (xdst == NULL) { |
b8c203b2 SK |
3168 | struct xfrm_flo xflo; |
3169 | ||
3170 | xflo.dst_orig = dst_orig; | |
3171 | xflo.flags = flags; | |
3172 | ||
1da177e4 | 3173 | /* To accelerate a bit... */ |
9f8550e4 EB |
3174 | if (!if_id && ((dst_orig->flags & DST_NOXFRM) || |
3175 | !net->xfrm.policy_count[XFRM_POLICY_OUT])) | |
8b7817f3 | 3176 | goto nopol; |
1da177e4 | 3177 | |
bc56b334 | 3178 | xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id); |
bd45c539 | 3179 | if (xdst == NULL) |
80c802f3 | 3180 | goto nopol; |
bd45c539 FW |
3181 | if (IS_ERR(xdst)) { |
3182 | err = PTR_ERR(xdst); | |
75b8c133 | 3183 | goto dropdst; |
d66e37a9 | 3184 | } |
80c802f3 TT |
3185 | |
3186 | num_pols = xdst->num_pols; | |
3187 | num_xfrms = xdst->num_xfrms; | |
3e94c2dc | 3188 | memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols); |
80c802f3 TT |
3189 | route = xdst->route; |
3190 | } | |
3191 | ||
3192 | dst = &xdst->u.dst; | |
3193 | if (route == NULL && num_xfrms > 0) { | |
3194 | /* The only case when xfrm_bundle_lookup() returns a | |
3195 | * bundle with null route, is when the template could | |
3196 | * not be resolved. It means policies are there, but | |
3197 | * bundle could not be created, since we don't yet | |
3198 | * have the xfrm_state's. We need to wait for KM to | |
3199 | * negotiate new SA's or bail out with error.*/ | |
3200 | if (net->xfrm.sysctl_larval_drop) { | |
80c802f3 | 3201 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); |
ac37e251 W |
3202 | err = -EREMOTE; |
3203 | goto error; | |
80c802f3 | 3204 | } |
80c802f3 | 3205 | |
5b8ef341 | 3206 | err = -EAGAIN; |
80c802f3 TT |
3207 | |
3208 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES); | |
3209 | goto error; | |
1da177e4 LT |
3210 | } |
3211 | ||
80c802f3 TT |
3212 | no_transform: |
3213 | if (num_pols == 0) | |
8b7817f3 | 3214 | goto nopol; |
1da177e4 | 3215 | |
80c802f3 TT |
3216 | if ((flags & XFRM_LOOKUP_ICMP) && |
3217 | !(pols[0]->flags & XFRM_POLICY_ICMP)) { | |
3218 | err = -ENOENT; | |
8b7817f3 | 3219 | goto error; |
80c802f3 | 3220 | } |
8b7817f3 | 3221 | |
80c802f3 | 3222 | for (i = 0; i < num_pols; i++) |
de5724ca | 3223 | WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds()); |
8b7817f3 | 3224 | |
80c802f3 | 3225 | if (num_xfrms < 0) { |
1da177e4 | 3226 | /* Prohibit the flow */ |
59c9940e | 3227 | XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK); |
e104411b PM |
3228 | err = -EPERM; |
3229 | goto error; | |
80c802f3 TT |
3230 | } else if (num_xfrms > 0) { |
3231 | /* Flow transformed */ | |
80c802f3 TT |
3232 | dst_release(dst_orig); |
3233 | } else { | |
3234 | /* Flow passes untransformed */ | |
3235 | dst_release(dst); | |
452edd59 | 3236 | dst = dst_orig; |
1da177e4 | 3237 | } |
80c802f3 TT |
3238 | ok: |
3239 | xfrm_pols_put(pols, drop_pols); | |
0c183379 G |
3240 | if (dst && dst->xfrm && |
3241 | dst->xfrm->props.mode == XFRM_MODE_TUNNEL) | |
3242 | dst->flags |= DST_XFRM_TUNNEL; | |
452edd59 | 3243 | return dst; |
1da177e4 | 3244 | |
80c802f3 | 3245 | nopol: |
17ecd4a4 | 3246 | if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && |
b58b1f56 | 3247 | net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { |
2d151d39 SK |
3248 | err = -EPERM; |
3249 | goto error; | |
3250 | } | |
452edd59 DM |
3251 | if (!(flags & XFRM_LOOKUP_ICMP)) { |
3252 | dst = dst_orig; | |
80c802f3 | 3253 | goto ok; |
452edd59 | 3254 | } |
80c802f3 | 3255 | err = -ENOENT; |
1da177e4 | 3256 | error: |
80c802f3 | 3257 | dst_release(dst); |
75b8c133 | 3258 | dropdst: |
ac37e251 W |
3259 | if (!(flags & XFRM_LOOKUP_KEEP_DST_REF)) |
3260 | dst_release(dst_orig); | |
80c802f3 | 3261 | xfrm_pols_put(pols, drop_pols); |
452edd59 | 3262 | return ERR_PTR(err); |
1da177e4 | 3263 | } |
bc56b334 BW |
3264 | EXPORT_SYMBOL(xfrm_lookup_with_ifid); |
3265 | ||
3266 | /* Main function: finds/creates a bundle for given flow. | |
3267 | * | |
3268 | * At the moment we eat a raw IP route. Mostly to speed up lookups | |
3269 | * on interfaces with disabled IPsec. | |
3270 | */ | |
3271 | struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig, | |
3272 | const struct flowi *fl, const struct sock *sk, | |
3273 | int flags) | |
3274 | { | |
3275 | return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0); | |
3276 | } | |
1da177e4 LT |
3277 | EXPORT_SYMBOL(xfrm_lookup); |
3278 | ||
f92ee619 SK |
3279 | /* Callers of xfrm_lookup_route() must ensure a call to dst_output(). |
3280 | * Otherwise we may send out blackholed packets. | |
3281 | */ | |
3282 | struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig, | |
3283 | const struct flowi *fl, | |
6f9c9615 | 3284 | const struct sock *sk, int flags) |
f92ee619 | 3285 | { |
b8c203b2 | 3286 | struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk, |
ac37e251 W |
3287 | flags | XFRM_LOOKUP_QUEUE | |
3288 | XFRM_LOOKUP_KEEP_DST_REF); | |
f92ee619 | 3289 | |
45586c70 | 3290 | if (PTR_ERR(dst) == -EREMOTE) |
f92ee619 SK |
3291 | return make_blackhole(net, dst_orig->ops->family, dst_orig); |
3292 | ||
8cc88773 TR |
3293 | if (IS_ERR(dst)) |
3294 | dst_release(dst_orig); | |
3295 | ||
f92ee619 SK |
3296 | return dst; |
3297 | } | |
3298 | EXPORT_SYMBOL(xfrm_lookup_route); | |
3299 | ||
df0ba92a | 3300 | static inline int |
8f029de2 | 3301 | xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl) |
df0ba92a | 3302 | { |
2294be0f | 3303 | struct sec_path *sp = skb_sec_path(skb); |
df0ba92a | 3304 | struct xfrm_state *x; |
df0ba92a | 3305 | |
2294be0f | 3306 | if (!sp || idx < 0 || idx >= sp->len) |
df0ba92a | 3307 | return 0; |
2294be0f | 3308 | x = sp->xvec[idx]; |
df0ba92a MN |
3309 | if (!x->type->reject) |
3310 | return 0; | |
1ecafede | 3311 | return x->type->reject(x, skb, fl); |
df0ba92a MN |
3312 | } |
3313 | ||
1da177e4 LT |
3314 | /* When skb is transformed back to its "native" form, we have to |
3315 | * check policy restrictions. At the moment we make this in maximally | |
3316 | * stupid way. Shame on me. :-) Of course, connected sockets must | |
3317 | * have policy cached at them. | |
3318 | */ | |
3319 | ||
3320 | static inline int | |
7db454b9 | 3321 | xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x, |
8680407b | 3322 | unsigned short family, u32 if_id) |
1da177e4 LT |
3323 | { |
3324 | if (xfrm_state_kern(x)) | |
928ba416 | 3325 | return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family); |
1da177e4 LT |
3326 | return x->id.proto == tmpl->id.proto && |
3327 | (x->id.spi == tmpl->id.spi || !tmpl->id.spi) && | |
3328 | (x->props.reqid == tmpl->reqid || !tmpl->reqid) && | |
3329 | x->props.mode == tmpl->mode && | |
c5d18e98 | 3330 | (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) || |
f3bd4840 | 3331 | !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) && |
7e49e6de | 3332 | !(x->props.mode != XFRM_MODE_TRANSPORT && |
8680407b BW |
3333 | xfrm_state_addr_cmp(tmpl, x, family)) && |
3334 | (if_id == 0 || if_id == x->if_id); | |
1da177e4 LT |
3335 | } |
3336 | ||
df0ba92a MN |
3337 | /* |
3338 | * 0 or more than 0 is returned when validation is succeeded (either bypass | |
7a7ae1eb | 3339 | * because of optional transport mode, or next index of the matched secpath |
df0ba92a MN |
3340 | * state with the template. |
3341 | * -1 is returned when no matching template is found. | |
3342 | * Otherwise "-2 - errored_index" is returned. | |
3343 | */ | |
1da177e4 | 3344 | static inline int |
22cccb7e | 3345 | xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start, |
8680407b | 3346 | unsigned short family, u32 if_id) |
1da177e4 LT |
3347 | { |
3348 | int idx = start; | |
3349 | ||
3350 | if (tmpl->optional) { | |
7e49e6de | 3351 | if (tmpl->mode == XFRM_MODE_TRANSPORT) |
1da177e4 LT |
3352 | return start; |
3353 | } else | |
3354 | start = -1; | |
3355 | for (; idx < sp->len; idx++) { | |
8680407b | 3356 | if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id)) |
1da177e4 | 3357 | return ++idx; |
df0ba92a | 3358 | if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) { |
1f8b6df6 BW |
3359 | if (idx < sp->verified_cnt) { |
3360 | /* Secpath entry previously verified, consider optional and | |
3361 | * continue searching | |
3362 | */ | |
3363 | continue; | |
3364 | } | |
3365 | ||
df0ba92a MN |
3366 | if (start == -1) |
3367 | start = -2-idx; | |
1da177e4 | 3368 | break; |
df0ba92a | 3369 | } |
1da177e4 LT |
3370 | } |
3371 | return start; | |
3372 | } | |
3373 | ||
c53ac41e FW |
3374 | static void |
3375 | decode_session4(struct sk_buff *skb, struct flowi *fl, bool reverse) | |
3376 | { | |
3377 | const struct iphdr *iph = ip_hdr(skb); | |
858e5400 FW |
3378 | int ihl = iph->ihl; |
3379 | u8 *xprth = skb_network_header(skb) + ihl * 4; | |
c53ac41e FW |
3380 | struct flowi4 *fl4 = &fl->u.ip4; |
3381 | int oif = 0; | |
3382 | ||
c3b4c3a4 | 3383 | if (skb_dst(skb) && skb_dst(skb)->dev) |
c53ac41e FW |
3384 | oif = skb_dst(skb)->dev->ifindex; |
3385 | ||
3386 | memset(fl4, 0, sizeof(struct flowi4)); | |
3387 | fl4->flowi4_mark = skb->mark; | |
3388 | fl4->flowi4_oif = reverse ? skb->skb_iif : oif; | |
3389 | ||
858e5400 FW |
3390 | fl4->flowi4_proto = iph->protocol; |
3391 | fl4->daddr = reverse ? iph->saddr : iph->daddr; | |
3392 | fl4->saddr = reverse ? iph->daddr : iph->saddr; | |
23e7b1bf | 3393 | fl4->flowi4_tos = iph->tos & ~INET_ECN_MASK; |
858e5400 | 3394 | |
c53ac41e FW |
3395 | if (!ip_is_fragment(iph)) { |
3396 | switch (iph->protocol) { | |
3397 | case IPPROTO_UDP: | |
3398 | case IPPROTO_UDPLITE: | |
3399 | case IPPROTO_TCP: | |
3400 | case IPPROTO_SCTP: | |
3401 | case IPPROTO_DCCP: | |
3402 | if (xprth + 4 < skb->data || | |
3403 | pskb_may_pull(skb, xprth + 4 - skb->data)) { | |
3404 | __be16 *ports; | |
3405 | ||
858e5400 | 3406 | xprth = skb_network_header(skb) + ihl * 4; |
c53ac41e FW |
3407 | ports = (__be16 *)xprth; |
3408 | ||
3409 | fl4->fl4_sport = ports[!!reverse]; | |
3410 | fl4->fl4_dport = ports[!reverse]; | |
3411 | } | |
3412 | break; | |
3413 | case IPPROTO_ICMP: | |
3414 | if (xprth + 2 < skb->data || | |
3415 | pskb_may_pull(skb, xprth + 2 - skb->data)) { | |
3416 | u8 *icmp; | |
3417 | ||
858e5400 | 3418 | xprth = skb_network_header(skb) + ihl * 4; |
c53ac41e FW |
3419 | icmp = xprth; |
3420 | ||
3421 | fl4->fl4_icmp_type = icmp[0]; | |
3422 | fl4->fl4_icmp_code = icmp[1]; | |
3423 | } | |
3424 | break; | |
c53ac41e FW |
3425 | case IPPROTO_GRE: |
3426 | if (xprth + 12 < skb->data || | |
3427 | pskb_may_pull(skb, xprth + 12 - skb->data)) { | |
3428 | __be16 *greflags; | |
3429 | __be32 *gre_hdr; | |
3430 | ||
858e5400 | 3431 | xprth = skb_network_header(skb) + ihl * 4; |
c53ac41e FW |
3432 | greflags = (__be16 *)xprth; |
3433 | gre_hdr = (__be32 *)xprth; | |
3434 | ||
3435 | if (greflags[0] & GRE_KEY) { | |
3436 | if (greflags[0] & GRE_CSUM) | |
3437 | gre_hdr++; | |
3438 | fl4->fl4_gre_key = gre_hdr[1]; | |
3439 | } | |
3440 | } | |
3441 | break; | |
3442 | default: | |
c53ac41e FW |
3443 | break; |
3444 | } | |
3445 | } | |
c53ac41e FW |
3446 | } |
3447 | ||
3448 | #if IS_ENABLED(CONFIG_IPV6) | |
3449 | static void | |
3450 | decode_session6(struct sk_buff *skb, struct flowi *fl, bool reverse) | |
3451 | { | |
3452 | struct flowi6 *fl6 = &fl->u.ip6; | |
3453 | int onlyproto = 0; | |
3454 | const struct ipv6hdr *hdr = ipv6_hdr(skb); | |
3455 | u32 offset = sizeof(*hdr); | |
3456 | struct ipv6_opt_hdr *exthdr; | |
3457 | const unsigned char *nh = skb_network_header(skb); | |
3458 | u16 nhoff = IP6CB(skb)->nhoff; | |
3459 | int oif = 0; | |
3460 | u8 nexthdr; | |
3461 | ||
3462 | if (!nhoff) | |
3463 | nhoff = offsetof(struct ipv6hdr, nexthdr); | |
3464 | ||
3465 | nexthdr = nh[nhoff]; | |
3466 | ||
c3b4c3a4 | 3467 | if (skb_dst(skb) && skb_dst(skb)->dev) |
c53ac41e FW |
3468 | oif = skb_dst(skb)->dev->ifindex; |
3469 | ||
3470 | memset(fl6, 0, sizeof(struct flowi6)); | |
3471 | fl6->flowi6_mark = skb->mark; | |
3472 | fl6->flowi6_oif = reverse ? skb->skb_iif : oif; | |
3473 | ||
3474 | fl6->daddr = reverse ? hdr->saddr : hdr->daddr; | |
3475 | fl6->saddr = reverse ? hdr->daddr : hdr->saddr; | |
3476 | ||
3477 | while (nh + offset + sizeof(*exthdr) < skb->data || | |
3478 | pskb_may_pull(skb, nh + offset + sizeof(*exthdr) - skb->data)) { | |
3479 | nh = skb_network_header(skb); | |
3480 | exthdr = (struct ipv6_opt_hdr *)(nh + offset); | |
3481 | ||
3482 | switch (nexthdr) { | |
3483 | case NEXTHDR_FRAGMENT: | |
3484 | onlyproto = 1; | |
df561f66 | 3485 | fallthrough; |
c53ac41e FW |
3486 | case NEXTHDR_ROUTING: |
3487 | case NEXTHDR_HOP: | |
3488 | case NEXTHDR_DEST: | |
3489 | offset += ipv6_optlen(exthdr); | |
3490 | nexthdr = exthdr->nexthdr; | |
c53ac41e FW |
3491 | break; |
3492 | case IPPROTO_UDP: | |
3493 | case IPPROTO_UDPLITE: | |
3494 | case IPPROTO_TCP: | |
3495 | case IPPROTO_SCTP: | |
3496 | case IPPROTO_DCCP: | |
3497 | if (!onlyproto && (nh + offset + 4 < skb->data || | |
3498 | pskb_may_pull(skb, nh + offset + 4 - skb->data))) { | |
3499 | __be16 *ports; | |
3500 | ||
3501 | nh = skb_network_header(skb); | |
3502 | ports = (__be16 *)(nh + offset); | |
3503 | fl6->fl6_sport = ports[!!reverse]; | |
3504 | fl6->fl6_dport = ports[!reverse]; | |
3505 | } | |
3506 | fl6->flowi6_proto = nexthdr; | |
3507 | return; | |
3508 | case IPPROTO_ICMPV6: | |
3509 | if (!onlyproto && (nh + offset + 2 < skb->data || | |
3510 | pskb_may_pull(skb, nh + offset + 2 - skb->data))) { | |
3511 | u8 *icmp; | |
3512 | ||
3513 | nh = skb_network_header(skb); | |
3514 | icmp = (u8 *)(nh + offset); | |
3515 | fl6->fl6_icmp_type = icmp[0]; | |
3516 | fl6->fl6_icmp_code = icmp[1]; | |
3517 | } | |
3518 | fl6->flowi6_proto = nexthdr; | |
3519 | return; | |
bcf141b2 GB |
3520 | case IPPROTO_GRE: |
3521 | if (!onlyproto && | |
3522 | (nh + offset + 12 < skb->data || | |
3523 | pskb_may_pull(skb, nh + offset + 12 - skb->data))) { | |
3524 | struct gre_base_hdr *gre_hdr; | |
3525 | __be32 *gre_key; | |
3526 | ||
3527 | nh = skb_network_header(skb); | |
3528 | gre_hdr = (struct gre_base_hdr *)(nh + offset); | |
3529 | gre_key = (__be32 *)(gre_hdr + 1); | |
3530 | ||
3531 | if (gre_hdr->flags & GRE_KEY) { | |
3532 | if (gre_hdr->flags & GRE_CSUM) | |
3533 | gre_key++; | |
3534 | fl6->fl6_gre_key = *gre_key; | |
3535 | } | |
3536 | } | |
3537 | fl6->flowi6_proto = nexthdr; | |
3538 | return; | |
3539 | ||
c53ac41e FW |
3540 | #if IS_ENABLED(CONFIG_IPV6_MIP6) |
3541 | case IPPROTO_MH: | |
3542 | offset += ipv6_optlen(exthdr); | |
3543 | if (!onlyproto && (nh + offset + 3 < skb->data || | |
3544 | pskb_may_pull(skb, nh + offset + 3 - skb->data))) { | |
3545 | struct ip6_mh *mh; | |
3546 | ||
3547 | nh = skb_network_header(skb); | |
3548 | mh = (struct ip6_mh *)(nh + offset); | |
3549 | fl6->fl6_mh_type = mh->ip6mh_type; | |
3550 | } | |
3551 | fl6->flowi6_proto = nexthdr; | |
3552 | return; | |
3553 | #endif | |
c53ac41e | 3554 | default: |
c53ac41e FW |
3555 | fl6->flowi6_proto = nexthdr; |
3556 | return; | |
3557 | } | |
3558 | } | |
3559 | } | |
3560 | #endif | |
3561 | ||
d5422efe HX |
3562 | int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl, |
3563 | unsigned int family, int reverse) | |
1da177e4 | 3564 | { |
c53ac41e FW |
3565 | switch (family) { |
3566 | case AF_INET: | |
3567 | decode_session4(skb, fl, reverse); | |
3568 | break; | |
3569 | #if IS_ENABLED(CONFIG_IPV6) | |
3570 | case AF_INET6: | |
3571 | decode_session6(skb, fl, reverse); | |
3572 | break; | |
3573 | #endif | |
3574 | default: | |
1da177e4 | 3575 | return -EAFNOSUPPORT; |
c53ac41e | 3576 | } |
1da177e4 | 3577 | |
c53ac41e | 3578 | return security_xfrm_decode_session(skb, &fl->flowi_secid); |
1da177e4 | 3579 | } |
d5422efe | 3580 | EXPORT_SYMBOL(__xfrm_decode_session); |
1da177e4 | 3581 | |
9a7386ec | 3582 | static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp) |
1da177e4 LT |
3583 | { |
3584 | for (; k < sp->len; k++) { | |
df0ba92a | 3585 | if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) { |
d1d9facf | 3586 | *idxp = k; |
1da177e4 | 3587 | return 1; |
df0ba92a | 3588 | } |
1da177e4 LT |
3589 | } |
3590 | ||
3591 | return 0; | |
3592 | } | |
3593 | ||
a716c119 | 3594 | int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, |
1da177e4 LT |
3595 | unsigned short family) |
3596 | { | |
f6e1e25d | 3597 | struct net *net = dev_net(skb->dev); |
1da177e4 | 3598 | struct xfrm_policy *pol; |
4e81bb83 MN |
3599 | struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX]; |
3600 | int npols = 0; | |
3601 | int xfrm_nr; | |
3602 | int pi; | |
d5422efe | 3603 | int reverse; |
1da177e4 | 3604 | struct flowi fl; |
df0ba92a | 3605 | int xerr_idx = -1; |
bc56b334 | 3606 | const struct xfrm_if_cb *ifcb; |
2294be0f | 3607 | struct sec_path *sp; |
bc56b334 BW |
3608 | u32 if_id = 0; |
3609 | ||
3610 | rcu_read_lock(); | |
3611 | ifcb = xfrm_if_get_cb(); | |
3612 | ||
3613 | if (ifcb) { | |
abc340b3 EB |
3614 | struct xfrm_if_decode_session_result r; |
3615 | ||
3616 | if (ifcb->decode_session(skb, family, &r)) { | |
3617 | if_id = r.if_id; | |
3618 | net = r.net; | |
660899dd | 3619 | } |
bc56b334 BW |
3620 | } |
3621 | rcu_read_unlock(); | |
1da177e4 | 3622 | |
d5422efe HX |
3623 | reverse = dir & ~XFRM_POLICY_MASK; |
3624 | dir &= XFRM_POLICY_MASK; | |
d5422efe | 3625 | |
0aa64774 | 3626 | if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) { |
59c9940e | 3627 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR); |
1da177e4 | 3628 | return 0; |
0aa64774 MN |
3629 | } |
3630 | ||
eb9c7ebe | 3631 | nf_nat_decode_session(skb, &fl, family); |
1da177e4 LT |
3632 | |
3633 | /* First, check used SA against their selectors. */ | |
2294be0f FW |
3634 | sp = skb_sec_path(skb); |
3635 | if (sp) { | |
1da177e4 LT |
3636 | int i; |
3637 | ||
2294be0f FW |
3638 | for (i = sp->len - 1; i >= 0; i--) { |
3639 | struct xfrm_state *x = sp->xvec[i]; | |
0aa64774 | 3640 | if (!xfrm_selector_match(&x->sel, &fl, family)) { |
59c9940e | 3641 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH); |
1da177e4 | 3642 | return 0; |
0aa64774 | 3643 | } |
1da177e4 LT |
3644 | } |
3645 | } | |
3646 | ||
3647 | pol = NULL; | |
bd5eb35f | 3648 | sk = sk_to_full_sk(sk); |
3bccfbc7 | 3649 | if (sk && sk->sk_policy[dir]) { |
bc56b334 | 3650 | pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id); |
0aa64774 | 3651 | if (IS_ERR(pol)) { |
59c9940e | 3652 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
3bccfbc7 | 3653 | return 0; |
0aa64774 | 3654 | } |
3bccfbc7 | 3655 | } |
1da177e4 | 3656 | |
86dc8ee0 | 3657 | if (!pol) |
bc56b334 | 3658 | pol = xfrm_policy_lookup(net, &fl, family, dir, if_id); |
1da177e4 | 3659 | |
0aa64774 | 3660 | if (IS_ERR(pol)) { |
59c9940e | 3661 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
134b0fc5 | 3662 | return 0; |
0aa64774 | 3663 | } |
134b0fc5 | 3664 | |
df0ba92a | 3665 | if (!pol) { |
b58b1f56 | 3666 | if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { |
2d151d39 SK |
3667 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); |
3668 | return 0; | |
3669 | } | |
3670 | ||
2294be0f | 3671 | if (sp && secpath_has_nontransport(sp, 0, &xerr_idx)) { |
df0ba92a | 3672 | xfrm_secpath_reject(xerr_idx, skb, &fl); |
59c9940e | 3673 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS); |
df0ba92a MN |
3674 | return 0; |
3675 | } | |
3676 | return 1; | |
3677 | } | |
1da177e4 | 3678 | |
0a9e5794 ED |
3679 | /* This lockless write can happen from different cpus. */ |
3680 | WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds()); | |
1da177e4 | 3681 | |
4e81bb83 | 3682 | pols[0] = pol; |
02d0892f | 3683 | npols++; |
4e81bb83 MN |
3684 | #ifdef CONFIG_XFRM_SUB_POLICY |
3685 | if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) { | |
f6e1e25d | 3686 | pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, |
4e81bb83 | 3687 | &fl, family, |
bc56b334 | 3688 | XFRM_POLICY_IN, if_id); |
4e81bb83 | 3689 | if (pols[1]) { |
0aa64774 | 3690 | if (IS_ERR(pols[1])) { |
59c9940e | 3691 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); |
9c9cb23e | 3692 | xfrm_pol_put(pols[0]); |
134b0fc5 | 3693 | return 0; |
0aa64774 | 3694 | } |
0a9e5794 ED |
3695 | /* This write can happen from different cpus. */ |
3696 | WRITE_ONCE(pols[1]->curlft.use_time, | |
3697 | ktime_get_real_seconds()); | |
02d0892f | 3698 | npols++; |
4e81bb83 MN |
3699 | } |
3700 | } | |
3701 | #endif | |
3702 | ||
1da177e4 | 3703 | if (pol->action == XFRM_POLICY_ALLOW) { |
1da177e4 | 3704 | static struct sec_path dummy; |
4e81bb83 | 3705 | struct xfrm_tmpl *tp[XFRM_MAX_DEPTH]; |
41a49cc3 | 3706 | struct xfrm_tmpl *stp[XFRM_MAX_DEPTH]; |
4e81bb83 MN |
3707 | struct xfrm_tmpl **tpp = tp; |
3708 | int ti = 0; | |
1da177e4 LT |
3709 | int i, k; |
3710 | ||
2294be0f FW |
3711 | sp = skb_sec_path(skb); |
3712 | if (!sp) | |
1da177e4 LT |
3713 | sp = &dummy; |
3714 | ||
4e81bb83 MN |
3715 | for (pi = 0; pi < npols; pi++) { |
3716 | if (pols[pi] != pol && | |
0aa64774 | 3717 | pols[pi]->action != XFRM_POLICY_ALLOW) { |
59c9940e | 3718 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); |
4e81bb83 | 3719 | goto reject; |
0aa64774 MN |
3720 | } |
3721 | if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) { | |
59c9940e | 3722 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); |
4e81bb83 | 3723 | goto reject_error; |
0aa64774 | 3724 | } |
4e81bb83 MN |
3725 | for (i = 0; i < pols[pi]->xfrm_nr; i++) |
3726 | tpp[ti++] = &pols[pi]->xfrm_vec[i]; | |
3727 | } | |
3728 | xfrm_nr = ti; | |
2d151d39 | 3729 | |
41a49cc3 | 3730 | if (npols > 1) { |
3aaf3915 | 3731 | xfrm_tmpl_sort(stp, tpp, xfrm_nr, family); |
41a49cc3 MN |
3732 | tpp = stp; |
3733 | } | |
4e81bb83 | 3734 | |
1da177e4 LT |
3735 | /* For each tunnel xfrm, find the first matching tmpl. |
3736 | * For each tmpl before that, find corresponding xfrm. | |
3737 | * Order is _important_. Later we will implement | |
3738 | * some barriers, but at the moment barriers | |
3739 | * are implied between each two transformations. | |
1f8b6df6 BW |
3740 | * Upon success, marks secpath entries as having been |
3741 | * verified to allow them to be skipped in future policy | |
3742 | * checks (e.g. nested tunnels). | |
1da177e4 | 3743 | */ |
4e81bb83 | 3744 | for (i = xfrm_nr-1, k = 0; i >= 0; i--) { |
8680407b | 3745 | k = xfrm_policy_ok(tpp[i], sp, k, family, if_id); |
df0ba92a | 3746 | if (k < 0) { |
d1d9facf JM |
3747 | if (k < -1) |
3748 | /* "-2 - errored_index" returned */ | |
3749 | xerr_idx = -(2+k); | |
59c9940e | 3750 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); |
1da177e4 | 3751 | goto reject; |
df0ba92a | 3752 | } |
1da177e4 LT |
3753 | } |
3754 | ||
0aa64774 | 3755 | if (secpath_has_nontransport(sp, k, &xerr_idx)) { |
59c9940e | 3756 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH); |
1da177e4 | 3757 | goto reject; |
0aa64774 | 3758 | } |
1da177e4 | 3759 | |
4e81bb83 | 3760 | xfrm_pols_put(pols, npols); |
1f8b6df6 BW |
3761 | sp->verified_cnt = k; |
3762 | ||
1da177e4 LT |
3763 | return 1; |
3764 | } | |
59c9940e | 3765 | XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK); |
1da177e4 LT |
3766 | |
3767 | reject: | |
df0ba92a | 3768 | xfrm_secpath_reject(xerr_idx, skb, &fl); |
4e81bb83 MN |
3769 | reject_error: |
3770 | xfrm_pols_put(pols, npols); | |
1da177e4 LT |
3771 | return 0; |
3772 | } | |
3773 | EXPORT_SYMBOL(__xfrm_policy_check); | |
3774 | ||
3775 | int __xfrm_route_forward(struct sk_buff *skb, unsigned short family) | |
3776 | { | |
99a66657 | 3777 | struct net *net = dev_net(skb->dev); |
1da177e4 | 3778 | struct flowi fl; |
adf30907 | 3779 | struct dst_entry *dst; |
73137147 | 3780 | int res = 1; |
1da177e4 | 3781 | |
0aa64774 | 3782 | if (xfrm_decode_session(skb, &fl, family) < 0) { |
72032fdb | 3783 | XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR); |
1da177e4 | 3784 | return 0; |
0aa64774 | 3785 | } |
1da177e4 | 3786 | |
fafeeb6c | 3787 | skb_dst_force(skb); |
9e143793 SK |
3788 | if (!skb_dst(skb)) { |
3789 | XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR); | |
3790 | return 0; | |
3791 | } | |
adf30907 | 3792 | |
b8c203b2 | 3793 | dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE); |
452edd59 | 3794 | if (IS_ERR(dst)) { |
73137147 | 3795 | res = 0; |
452edd59 DM |
3796 | dst = NULL; |
3797 | } | |
adf30907 ED |
3798 | skb_dst_set(skb, dst); |
3799 | return res; | |
1da177e4 LT |
3800 | } |
3801 | EXPORT_SYMBOL(__xfrm_route_forward); | |
3802 | ||
d49c73c7 DM |
3803 | /* Optimize later using cookies and generation ids. */ |
3804 | ||
1da177e4 LT |
3805 | static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie) |
3806 | { | |
d49c73c7 | 3807 | /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete |
f5b0a874 DM |
3808 | * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to |
3809 | * get validated by dst_ops->check on every use. We do this | |
3810 | * because when a normal route referenced by an XFRM dst is | |
3811 | * obsoleted we do not go looking around for all parent | |
3812 | * referencing XFRM dsts so that we can invalidate them. It | |
3813 | * is just too much work. Instead we make the checks here on | |
3814 | * every use. For example: | |
d49c73c7 DM |
3815 | * |
3816 | * XFRM dst A --> IPv4 dst X | |
3817 | * | |
3818 | * X is the "xdst->route" of A (X is also the "dst->path" of A | |
3819 | * in this example). If X is marked obsolete, "A" will not | |
3820 | * notice. That's what we are validating here via the | |
3821 | * stale_bundle() check. | |
3822 | * | |
52df157f WW |
3823 | * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will |
3824 | * be marked on it. | |
09c75704 | 3825 | * This will force stale_bundle() to fail on any xdst bundle with |
52df157f | 3826 | * this dst linked in it. |
399c180a | 3827 | */ |
d49c73c7 DM |
3828 | if (dst->obsolete < 0 && !stale_bundle(dst)) |
3829 | return dst; | |
3830 | ||
1da177e4 LT |
3831 | return NULL; |
3832 | } | |
3833 | ||
3834 | static int stale_bundle(struct dst_entry *dst) | |
3835 | { | |
12fdb4d3 | 3836 | return !xfrm_bundle_ok((struct xfrm_dst *)dst); |
1da177e4 LT |
3837 | } |
3838 | ||
aabc9761 | 3839 | void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev) |
1da177e4 | 3840 | { |
b92cf4aa | 3841 | while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) { |
4d33ab08 | 3842 | dst->dev = blackhole_netdev; |
de3cb747 | 3843 | dev_hold(dst->dev); |
1da177e4 LT |
3844 | dev_put(dev); |
3845 | } | |
3846 | } | |
aabc9761 | 3847 | EXPORT_SYMBOL(xfrm_dst_ifdown); |
1da177e4 LT |
3848 | |
3849 | static void xfrm_link_failure(struct sk_buff *skb) | |
3850 | { | |
3851 | /* Impossible. Such dst must be popped before reaches point of failure. */ | |
1da177e4 LT |
3852 | } |
3853 | ||
3854 | static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst) | |
3855 | { | |
3856 | if (dst) { | |
3857 | if (dst->obsolete) { | |
3858 | dst_release(dst); | |
3859 | dst = NULL; | |
3860 | } | |
3861 | } | |
3862 | return dst; | |
3863 | } | |
3864 | ||
5492093d | 3865 | static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr) |
1da177e4 | 3866 | { |
5492093d DM |
3867 | while (nr--) { |
3868 | struct xfrm_dst *xdst = bundle[nr]; | |
1da177e4 | 3869 | u32 pmtu, route_mtu_cached; |
5492093d | 3870 | struct dst_entry *dst; |
1da177e4 | 3871 | |
5492093d | 3872 | dst = &xdst->u.dst; |
b92cf4aa | 3873 | pmtu = dst_mtu(xfrm_dst_child(dst)); |
1da177e4 LT |
3874 | xdst->child_mtu_cached = pmtu; |
3875 | ||
3876 | pmtu = xfrm_state_mtu(dst->xfrm, pmtu); | |
3877 | ||
3878 | route_mtu_cached = dst_mtu(xdst->route); | |
3879 | xdst->route_mtu_cached = route_mtu_cached; | |
3880 | ||
3881 | if (pmtu > route_mtu_cached) | |
3882 | pmtu = route_mtu_cached; | |
3883 | ||
defb3519 | 3884 | dst_metric_set(dst, RTAX_MTU, pmtu); |
5492093d | 3885 | } |
1da177e4 LT |
3886 | } |
3887 | ||
1da177e4 LT |
3888 | /* Check that the bundle accepts the flow and its components are |
3889 | * still valid. | |
3890 | */ | |
3891 | ||
12fdb4d3 | 3892 | static int xfrm_bundle_ok(struct xfrm_dst *first) |
1da177e4 | 3893 | { |
5492093d | 3894 | struct xfrm_dst *bundle[XFRM_MAX_DEPTH]; |
1da177e4 | 3895 | struct dst_entry *dst = &first->u.dst; |
5492093d DM |
3896 | struct xfrm_dst *xdst; |
3897 | int start_from, nr; | |
1da177e4 LT |
3898 | u32 mtu; |
3899 | ||
0f6c480f | 3900 | if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) || |
1da177e4 LT |
3901 | (dst->dev && !netif_running(dst->dev))) |
3902 | return 0; | |
3903 | ||
a0073fe1 SK |
3904 | if (dst->flags & DST_XFRM_QUEUE) |
3905 | return 1; | |
3906 | ||
5492093d | 3907 | start_from = nr = 0; |
1da177e4 LT |
3908 | do { |
3909 | struct xfrm_dst *xdst = (struct xfrm_dst *)dst; | |
3910 | ||
1da177e4 LT |
3911 | if (dst->xfrm->km.state != XFRM_STATE_VALID) |
3912 | return 0; | |
80c802f3 TT |
3913 | if (xdst->xfrm_genid != dst->xfrm->genid) |
3914 | return 0; | |
b1312c89 TT |
3915 | if (xdst->num_pols > 0 && |
3916 | xdst->policy_genid != atomic_read(&xdst->pols[0]->genid)) | |
9d4a706d | 3917 | return 0; |
e53820de | 3918 | |
5492093d DM |
3919 | bundle[nr++] = xdst; |
3920 | ||
b92cf4aa | 3921 | mtu = dst_mtu(xfrm_dst_child(dst)); |
1da177e4 | 3922 | if (xdst->child_mtu_cached != mtu) { |
5492093d | 3923 | start_from = nr; |
1da177e4 LT |
3924 | xdst->child_mtu_cached = mtu; |
3925 | } | |
3926 | ||
92d63dec | 3927 | if (!dst_check(xdst->route, xdst->route_cookie)) |
1da177e4 LT |
3928 | return 0; |
3929 | mtu = dst_mtu(xdst->route); | |
3930 | if (xdst->route_mtu_cached != mtu) { | |
5492093d | 3931 | start_from = nr; |
1da177e4 LT |
3932 | xdst->route_mtu_cached = mtu; |
3933 | } | |
3934 | ||
b92cf4aa | 3935 | dst = xfrm_dst_child(dst); |
1da177e4 LT |
3936 | } while (dst->xfrm); |
3937 | ||
5492093d | 3938 | if (likely(!start_from)) |
1da177e4 LT |
3939 | return 1; |
3940 | ||
5492093d DM |
3941 | xdst = bundle[start_from - 1]; |
3942 | mtu = xdst->child_mtu_cached; | |
3943 | while (start_from--) { | |
3944 | dst = &xdst->u.dst; | |
1da177e4 LT |
3945 | |
3946 | mtu = xfrm_state_mtu(dst->xfrm, mtu); | |
5492093d DM |
3947 | if (mtu > xdst->route_mtu_cached) |
3948 | mtu = xdst->route_mtu_cached; | |
defb3519 | 3949 | dst_metric_set(dst, RTAX_MTU, mtu); |
5492093d | 3950 | if (!start_from) |
1da177e4 LT |
3951 | break; |
3952 | ||
5492093d DM |
3953 | xdst = bundle[start_from - 1]; |
3954 | xdst->child_mtu_cached = mtu; | |
1da177e4 LT |
3955 | } |
3956 | ||
3957 | return 1; | |
3958 | } | |
3959 | ||
0dbaee3b DM |
3960 | static unsigned int xfrm_default_advmss(const struct dst_entry *dst) |
3961 | { | |
0f6c480f | 3962 | return dst_metric_advmss(xfrm_dst_path(dst)); |
0dbaee3b DM |
3963 | } |
3964 | ||
ebb762f2 | 3965 | static unsigned int xfrm_mtu(const struct dst_entry *dst) |
d33e4553 | 3966 | { |
618f9bc7 SK |
3967 | unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); |
3968 | ||
0f6c480f | 3969 | return mtu ? : dst_mtu(xfrm_dst_path(dst)); |
d33e4553 DM |
3970 | } |
3971 | ||
1ecc9ad0 JA |
3972 | static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst, |
3973 | const void *daddr) | |
63fca65d | 3974 | { |
0f6c480f | 3975 | while (dst->xfrm) { |
63fca65d JA |
3976 | const struct xfrm_state *xfrm = dst->xfrm; |
3977 | ||
013cb81e SK |
3978 | dst = xfrm_dst_child(dst); |
3979 | ||
63fca65d JA |
3980 | if (xfrm->props.mode == XFRM_MODE_TRANSPORT) |
3981 | continue; | |
3982 | if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR) | |
3983 | daddr = xfrm->coaddr; | |
3984 | else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR)) | |
3985 | daddr = &xfrm->id.daddr; | |
3986 | } | |
1ecc9ad0 JA |
3987 | return daddr; |
3988 | } | |
3989 | ||
3990 | static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst, | |
3991 | struct sk_buff *skb, | |
3992 | const void *daddr) | |
3993 | { | |
0f6c480f | 3994 | const struct dst_entry *path = xfrm_dst_path(dst); |
1ecc9ad0 JA |
3995 | |
3996 | if (!skb) | |
3997 | daddr = xfrm_get_dst_nexthop(dst, daddr); | |
3998 | return path->ops->neigh_lookup(path, skb, daddr); | |
3999 | } | |
4000 | ||
4001 | static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr) | |
4002 | { | |
0f6c480f | 4003 | const struct dst_entry *path = xfrm_dst_path(dst); |
1ecc9ad0 JA |
4004 | |
4005 | daddr = xfrm_get_dst_nexthop(dst, daddr); | |
63fca65d JA |
4006 | path->ops->confirm_neigh(path, daddr); |
4007 | } | |
4008 | ||
a2817d8b | 4009 | int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family) |
1da177e4 LT |
4010 | { |
4011 | int err = 0; | |
a2817d8b FW |
4012 | |
4013 | if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo))) | |
1da177e4 | 4014 | return -EAFNOSUPPORT; |
a2817d8b | 4015 | |
ef8531b6 | 4016 | spin_lock(&xfrm_policy_afinfo_lock); |
a2817d8b | 4017 | if (unlikely(xfrm_policy_afinfo[family] != NULL)) |
f31e8d4f | 4018 | err = -EEXIST; |
1da177e4 LT |
4019 | else { |
4020 | struct dst_ops *dst_ops = afinfo->dst_ops; | |
4021 | if (likely(dst_ops->kmem_cachep == NULL)) | |
4022 | dst_ops->kmem_cachep = xfrm_dst_cache; | |
4023 | if (likely(dst_ops->check == NULL)) | |
4024 | dst_ops->check = xfrm_dst_check; | |
0dbaee3b DM |
4025 | if (likely(dst_ops->default_advmss == NULL)) |
4026 | dst_ops->default_advmss = xfrm_default_advmss; | |
ebb762f2 SK |
4027 | if (likely(dst_ops->mtu == NULL)) |
4028 | dst_ops->mtu = xfrm_mtu; | |
1da177e4 LT |
4029 | if (likely(dst_ops->negative_advice == NULL)) |
4030 | dst_ops->negative_advice = xfrm_negative_advice; | |
4031 | if (likely(dst_ops->link_failure == NULL)) | |
4032 | dst_ops->link_failure = xfrm_link_failure; | |
d3aaeb38 DM |
4033 | if (likely(dst_ops->neigh_lookup == NULL)) |
4034 | dst_ops->neigh_lookup = xfrm_neigh_lookup; | |
63fca65d JA |
4035 | if (likely(!dst_ops->confirm_neigh)) |
4036 | dst_ops->confirm_neigh = xfrm_confirm_neigh; | |
a2817d8b | 4037 | rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo); |
1da177e4 | 4038 | } |
ef8531b6 | 4039 | spin_unlock(&xfrm_policy_afinfo_lock); |
d7c7544c | 4040 | |
1da177e4 LT |
4041 | return err; |
4042 | } | |
4043 | EXPORT_SYMBOL(xfrm_policy_register_afinfo); | |
4044 | ||
a2817d8b | 4045 | void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo) |
1da177e4 | 4046 | { |
2b61997a | 4047 | struct dst_ops *dst_ops = afinfo->dst_ops; |
a2817d8b | 4048 | int i; |
2b61997a | 4049 | |
a2817d8b FW |
4050 | for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) { |
4051 | if (xfrm_policy_afinfo[i] != afinfo) | |
4052 | continue; | |
4053 | RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL); | |
4054 | break; | |
ef8531b6 | 4055 | } |
ef8531b6 | 4056 | |
2b61997a | 4057 | synchronize_rcu(); |
ef8531b6 | 4058 | |
2b61997a FW |
4059 | dst_ops->kmem_cachep = NULL; |
4060 | dst_ops->check = NULL; | |
4061 | dst_ops->negative_advice = NULL; | |
4062 | dst_ops->link_failure = NULL; | |
1da177e4 LT |
4063 | } |
4064 | EXPORT_SYMBOL(xfrm_policy_unregister_afinfo); | |
4065 | ||
f203b76d SK |
4066 | void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb) |
4067 | { | |
4068 | spin_lock(&xfrm_if_cb_lock); | |
4069 | rcu_assign_pointer(xfrm_if_cb, ifcb); | |
4070 | spin_unlock(&xfrm_if_cb_lock); | |
4071 | } | |
4072 | EXPORT_SYMBOL(xfrm_if_register_cb); | |
4073 | ||
4074 | void xfrm_if_unregister_cb(void) | |
4075 | { | |
4076 | RCU_INIT_POINTER(xfrm_if_cb, NULL); | |
4077 | synchronize_rcu(); | |
4078 | } | |
4079 | EXPORT_SYMBOL(xfrm_if_unregister_cb); | |
4080 | ||
558f82ef | 4081 | #ifdef CONFIG_XFRM_STATISTICS |
59c9940e | 4082 | static int __net_init xfrm_statistics_init(struct net *net) |
558f82ef | 4083 | { |
c68cd1a0 | 4084 | int rv; |
698365fa WC |
4085 | net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib); |
4086 | if (!net->mib.xfrm_statistics) | |
558f82ef | 4087 | return -ENOMEM; |
c68cd1a0 AD |
4088 | rv = xfrm_proc_init(net); |
4089 | if (rv < 0) | |
698365fa | 4090 | free_percpu(net->mib.xfrm_statistics); |
c68cd1a0 | 4091 | return rv; |
558f82ef | 4092 | } |
59c9940e AD |
4093 | |
4094 | static void xfrm_statistics_fini(struct net *net) | |
4095 | { | |
c68cd1a0 | 4096 | xfrm_proc_fini(net); |
698365fa | 4097 | free_percpu(net->mib.xfrm_statistics); |
59c9940e AD |
4098 | } |
4099 | #else | |
4100 | static int __net_init xfrm_statistics_init(struct net *net) | |
4101 | { | |
4102 | return 0; | |
4103 | } | |
4104 | ||
4105 | static void xfrm_statistics_fini(struct net *net) | |
4106 | { | |
4107 | } | |
558f82ef MN |
4108 | #endif |
4109 | ||
d62ddc21 | 4110 | static int __net_init xfrm_policy_init(struct net *net) |
1da177e4 | 4111 | { |
2518c7c2 | 4112 | unsigned int hmask, sz; |
24969fac | 4113 | int dir, err; |
2518c7c2 | 4114 | |
24969fac | 4115 | if (net_eq(net, &init_net)) { |
d62ddc21 | 4116 | xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache", |
1da177e4 | 4117 | sizeof(struct xfrm_dst), |
e5d679f3 | 4118 | 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, |
20c2df83 | 4119 | NULL); |
24969fac FW |
4120 | err = rhashtable_init(&xfrm_policy_inexact_table, |
4121 | &xfrm_pol_inexact_params); | |
4122 | BUG_ON(err); | |
4123 | } | |
1da177e4 | 4124 | |
2518c7c2 DM |
4125 | hmask = 8 - 1; |
4126 | sz = (hmask+1) * sizeof(struct hlist_head); | |
4127 | ||
93b851c1 AD |
4128 | net->xfrm.policy_byidx = xfrm_hash_alloc(sz); |
4129 | if (!net->xfrm.policy_byidx) | |
4130 | goto out_byidx; | |
8100bea7 | 4131 | net->xfrm.policy_idx_hmask = hmask; |
2518c7c2 | 4132 | |
53c2e285 | 4133 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
2518c7c2 DM |
4134 | struct xfrm_policy_hash *htab; |
4135 | ||
dc2caba7 | 4136 | net->xfrm.policy_count[dir] = 0; |
53c2e285 | 4137 | net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0; |
8b18f8ea | 4138 | INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]); |
2518c7c2 | 4139 | |
a35f6c5d | 4140 | htab = &net->xfrm.policy_bydst[dir]; |
44e36b42 | 4141 | htab->table = xfrm_hash_alloc(sz); |
2518c7c2 | 4142 | if (!htab->table) |
a35f6c5d AD |
4143 | goto out_bydst; |
4144 | htab->hmask = hmask; | |
b58555f1 CG |
4145 | htab->dbits4 = 32; |
4146 | htab->sbits4 = 32; | |
4147 | htab->dbits6 = 128; | |
4148 | htab->sbits6 = 128; | |
2518c7c2 | 4149 | } |
880a6fab CG |
4150 | net->xfrm.policy_hthresh.lbits4 = 32; |
4151 | net->xfrm.policy_hthresh.rbits4 = 32; | |
4152 | net->xfrm.policy_hthresh.lbits6 = 128; | |
4153 | net->xfrm.policy_hthresh.rbits6 = 128; | |
4154 | ||
4155 | seqlock_init(&net->xfrm.policy_hthresh.lock); | |
2518c7c2 | 4156 | |
adfcf0b2 | 4157 | INIT_LIST_HEAD(&net->xfrm.policy_all); |
24969fac | 4158 | INIT_LIST_HEAD(&net->xfrm.inexact_bins); |
66caf628 | 4159 | INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize); |
880a6fab | 4160 | INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild); |
d62ddc21 | 4161 | return 0; |
93b851c1 | 4162 | |
a35f6c5d AD |
4163 | out_bydst: |
4164 | for (dir--; dir >= 0; dir--) { | |
4165 | struct xfrm_policy_hash *htab; | |
4166 | ||
4167 | htab = &net->xfrm.policy_bydst[dir]; | |
4168 | xfrm_hash_free(htab->table, sz); | |
4169 | } | |
4170 | xfrm_hash_free(net->xfrm.policy_byidx, sz); | |
93b851c1 AD |
4171 | out_byidx: |
4172 | return -ENOMEM; | |
d62ddc21 AD |
4173 | } |
4174 | ||
4175 | static void xfrm_policy_fini(struct net *net) | |
4176 | { | |
6be3b0db | 4177 | struct xfrm_pol_inexact_bin *b, *t; |
93b851c1 | 4178 | unsigned int sz; |
8b18f8ea | 4179 | int dir; |
93b851c1 | 4180 | |
7c2776ee AD |
4181 | flush_work(&net->xfrm.policy_hash_work); |
4182 | #ifdef CONFIG_XFRM_SUB_POLICY | |
2e71029e | 4183 | xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false); |
7c2776ee | 4184 | #endif |
2e71029e | 4185 | xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false); |
7c2776ee | 4186 | |
adfcf0b2 | 4187 | WARN_ON(!list_empty(&net->xfrm.policy_all)); |
93b851c1 | 4188 | |
53c2e285 | 4189 | for (dir = 0; dir < XFRM_POLICY_MAX; dir++) { |
a35f6c5d AD |
4190 | struct xfrm_policy_hash *htab; |
4191 | ||
8b18f8ea | 4192 | WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir])); |
a35f6c5d AD |
4193 | |
4194 | htab = &net->xfrm.policy_bydst[dir]; | |
5b653b2a | 4195 | sz = (htab->hmask + 1) * sizeof(struct hlist_head); |
a35f6c5d AD |
4196 | WARN_ON(!hlist_empty(htab->table)); |
4197 | xfrm_hash_free(htab->table, sz); | |
8b18f8ea AD |
4198 | } |
4199 | ||
8100bea7 | 4200 | sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head); |
93b851c1 AD |
4201 | WARN_ON(!hlist_empty(net->xfrm.policy_byidx)); |
4202 | xfrm_hash_free(net->xfrm.policy_byidx, sz); | |
24969fac | 4203 | |
6be3b0db FW |
4204 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
4205 | list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins) | |
4206 | __xfrm_policy_inexact_prune_bin(b, true); | |
4207 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); | |
1da177e4 LT |
4208 | } |
4209 | ||
d62ddc21 AD |
4210 | static int __net_init xfrm_net_init(struct net *net) |
4211 | { | |
4212 | int rv; | |
4213 | ||
c282222a FW |
4214 | /* Initialize the per-net locks here */ |
4215 | spin_lock_init(&net->xfrm.xfrm_state_lock); | |
4216 | spin_lock_init(&net->xfrm.xfrm_policy_lock); | |
2580d3f4 | 4217 | seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock); |
c282222a | 4218 | mutex_init(&net->xfrm.xfrm_cfg_mutex); |
b58b1f56 ND |
4219 | net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT; |
4220 | net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT; | |
4221 | net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT; | |
c282222a | 4222 | |
59c9940e AD |
4223 | rv = xfrm_statistics_init(net); |
4224 | if (rv < 0) | |
4225 | goto out_statistics; | |
d62ddc21 AD |
4226 | rv = xfrm_state_init(net); |
4227 | if (rv < 0) | |
4228 | goto out_state; | |
4229 | rv = xfrm_policy_init(net); | |
4230 | if (rv < 0) | |
4231 | goto out_policy; | |
b27aeadb AD |
4232 | rv = xfrm_sysctl_init(net); |
4233 | if (rv < 0) | |
4234 | goto out_sysctl; | |
283bc9f3 | 4235 | |
d62ddc21 AD |
4236 | return 0; |
4237 | ||
b27aeadb AD |
4238 | out_sysctl: |
4239 | xfrm_policy_fini(net); | |
d62ddc21 AD |
4240 | out_policy: |
4241 | xfrm_state_fini(net); | |
4242 | out_state: | |
59c9940e AD |
4243 | xfrm_statistics_fini(net); |
4244 | out_statistics: | |
d62ddc21 AD |
4245 | return rv; |
4246 | } | |
4247 | ||
4248 | static void __net_exit xfrm_net_exit(struct net *net) | |
4249 | { | |
b27aeadb | 4250 | xfrm_sysctl_fini(net); |
d62ddc21 AD |
4251 | xfrm_policy_fini(net); |
4252 | xfrm_state_fini(net); | |
59c9940e | 4253 | xfrm_statistics_fini(net); |
d62ddc21 AD |
4254 | } |
4255 | ||
4256 | static struct pernet_operations __net_initdata xfrm_net_ops = { | |
4257 | .init = xfrm_net_init, | |
4258 | .exit = xfrm_net_exit, | |
4259 | }; | |
4260 | ||
1da177e4 LT |
4261 | void __init xfrm_init(void) |
4262 | { | |
d62ddc21 | 4263 | register_pernet_subsys(&xfrm_net_ops); |
e9a441b6 | 4264 | xfrm_dev_init(); |
1da177e4 | 4265 | xfrm_input_init(); |
f203b76d | 4266 | |
95a35b42 | 4267 | #ifdef CONFIG_XFRM_ESPINTCP |
e27cca96 SD |
4268 | espintcp_init(); |
4269 | #endif | |
1da177e4 LT |
4270 | } |
4271 | ||
ab5f5e8b | 4272 | #ifdef CONFIG_AUDITSYSCALL |
1486cbd7 IJ |
4273 | static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp, |
4274 | struct audit_buffer *audit_buf) | |
ab5f5e8b | 4275 | { |
875179fa PM |
4276 | struct xfrm_sec_ctx *ctx = xp->security; |
4277 | struct xfrm_selector *sel = &xp->selector; | |
4278 | ||
4279 | if (ctx) | |
ab5f5e8b | 4280 | audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s", |
875179fa | 4281 | ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str); |
ab5f5e8b | 4282 | |
9b7a787d | 4283 | switch (sel->family) { |
ab5f5e8b | 4284 | case AF_INET: |
21454aaa | 4285 | audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4); |
875179fa PM |
4286 | if (sel->prefixlen_s != 32) |
4287 | audit_log_format(audit_buf, " src_prefixlen=%d", | |
4288 | sel->prefixlen_s); | |
21454aaa | 4289 | audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4); |
875179fa PM |
4290 | if (sel->prefixlen_d != 32) |
4291 | audit_log_format(audit_buf, " dst_prefixlen=%d", | |
4292 | sel->prefixlen_d); | |
ab5f5e8b JL |
4293 | break; |
4294 | case AF_INET6: | |
5b095d98 | 4295 | audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6); |
875179fa PM |
4296 | if (sel->prefixlen_s != 128) |
4297 | audit_log_format(audit_buf, " src_prefixlen=%d", | |
4298 | sel->prefixlen_s); | |
5b095d98 | 4299 | audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6); |
875179fa PM |
4300 | if (sel->prefixlen_d != 128) |
4301 | audit_log_format(audit_buf, " dst_prefixlen=%d", | |
4302 | sel->prefixlen_d); | |
ab5f5e8b JL |
4303 | break; |
4304 | } | |
4305 | } | |
4306 | ||
2e71029e | 4307 | void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid) |
ab5f5e8b JL |
4308 | { |
4309 | struct audit_buffer *audit_buf; | |
ab5f5e8b | 4310 | |
afeb14b4 | 4311 | audit_buf = xfrm_audit_start("SPD-add"); |
ab5f5e8b JL |
4312 | if (audit_buf == NULL) |
4313 | return; | |
2e71029e | 4314 | xfrm_audit_helper_usrinfo(task_valid, audit_buf); |
afeb14b4 | 4315 | audit_log_format(audit_buf, " res=%u", result); |
ab5f5e8b JL |
4316 | xfrm_audit_common_policyinfo(xp, audit_buf); |
4317 | audit_log_end(audit_buf); | |
4318 | } | |
4319 | EXPORT_SYMBOL_GPL(xfrm_audit_policy_add); | |
4320 | ||
68277acc | 4321 | void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result, |
2e71029e | 4322 | bool task_valid) |
ab5f5e8b JL |
4323 | { |
4324 | struct audit_buffer *audit_buf; | |
ab5f5e8b | 4325 | |
afeb14b4 | 4326 | audit_buf = xfrm_audit_start("SPD-delete"); |
ab5f5e8b JL |
4327 | if (audit_buf == NULL) |
4328 | return; | |
2e71029e | 4329 | xfrm_audit_helper_usrinfo(task_valid, audit_buf); |
afeb14b4 | 4330 | audit_log_format(audit_buf, " res=%u", result); |
ab5f5e8b JL |
4331 | xfrm_audit_common_policyinfo(xp, audit_buf); |
4332 | audit_log_end(audit_buf); | |
4333 | } | |
4334 | EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete); | |
4335 | #endif | |
4336 | ||
80c9abaa | 4337 | #ifdef CONFIG_XFRM_MIGRATE |
bc9b35ad DM |
4338 | static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp, |
4339 | const struct xfrm_selector *sel_tgt) | |
80c9abaa SS |
4340 | { |
4341 | if (sel_cmp->proto == IPSEC_ULPROTO_ANY) { | |
4342 | if (sel_tgt->family == sel_cmp->family && | |
70e94e66 YH |
4343 | xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr, |
4344 | sel_cmp->family) && | |
4345 | xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr, | |
4346 | sel_cmp->family) && | |
80c9abaa SS |
4347 | sel_tgt->prefixlen_d == sel_cmp->prefixlen_d && |
4348 | sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) { | |
bc9b35ad | 4349 | return true; |
80c9abaa SS |
4350 | } |
4351 | } else { | |
4352 | if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) { | |
bc9b35ad | 4353 | return true; |
80c9abaa SS |
4354 | } |
4355 | } | |
bc9b35ad | 4356 | return false; |
80c9abaa SS |
4357 | } |
4358 | ||
3e94c2dc | 4359 | static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel, |
c1aca308 | 4360 | u8 dir, u8 type, struct net *net, u32 if_id) |
80c9abaa SS |
4361 | { |
4362 | struct xfrm_policy *pol, *ret = NULL; | |
80c9abaa SS |
4363 | struct hlist_head *chain; |
4364 | u32 priority = ~0U; | |
4365 | ||
9d0380df | 4366 | spin_lock_bh(&net->xfrm.xfrm_policy_lock); |
8d549c4f | 4367 | chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir); |
b67bfe0d | 4368 | hlist_for_each_entry(pol, chain, bydst) { |
c1aca308 YY |
4369 | if ((if_id == 0 || pol->if_id == if_id) && |
4370 | xfrm_migrate_selector_match(sel, &pol->selector) && | |
80c9abaa SS |
4371 | pol->type == type) { |
4372 | ret = pol; | |
4373 | priority = ret->priority; | |
4374 | break; | |
4375 | } | |
4376 | } | |
8d549c4f | 4377 | chain = &net->xfrm.policy_inexact[dir]; |
24969fac | 4378 | hlist_for_each_entry(pol, chain, bydst_inexact_list) { |
8faf491e LR |
4379 | if ((pol->priority >= priority) && ret) |
4380 | break; | |
4381 | ||
c1aca308 YY |
4382 | if ((if_id == 0 || pol->if_id == if_id) && |
4383 | xfrm_migrate_selector_match(sel, &pol->selector) && | |
8faf491e | 4384 | pol->type == type) { |
80c9abaa SS |
4385 | ret = pol; |
4386 | break; | |
4387 | } | |
4388 | } | |
4389 | ||
586f2eb4 | 4390 | xfrm_pol_hold(ret); |
80c9abaa | 4391 | |
9d0380df | 4392 | spin_unlock_bh(&net->xfrm.xfrm_policy_lock); |
80c9abaa SS |
4393 | |
4394 | return ret; | |
4395 | } | |
4396 | ||
dd701754 | 4397 | static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t) |
80c9abaa SS |
4398 | { |
4399 | int match = 0; | |
4400 | ||
4401 | if (t->mode == m->mode && t->id.proto == m->proto && | |
4402 | (m->reqid == 0 || t->reqid == m->reqid)) { | |
4403 | switch (t->mode) { | |
4404 | case XFRM_MODE_TUNNEL: | |
4405 | case XFRM_MODE_BEET: | |
70e94e66 YH |
4406 | if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr, |
4407 | m->old_family) && | |
4408 | xfrm_addr_equal(&t->saddr, &m->old_saddr, | |
4409 | m->old_family)) { | |
80c9abaa SS |
4410 | match = 1; |
4411 | } | |
4412 | break; | |
4413 | case XFRM_MODE_TRANSPORT: | |
4414 | /* in case of transport mode, template does not store | |
4415 | any IP addresses, hence we just compare mode and | |
4416 | protocol */ | |
4417 | match = 1; | |
4418 | break; | |
4419 | default: | |
4420 | break; | |
4421 | } | |
4422 | } | |
4423 | return match; | |
4424 | } | |
4425 | ||
4426 | /* update endpoint address(es) of template(s) */ | |
4427 | static int xfrm_policy_migrate(struct xfrm_policy *pol, | |
bd122403 SD |
4428 | struct xfrm_migrate *m, int num_migrate, |
4429 | struct netlink_ext_ack *extack) | |
80c9abaa SS |
4430 | { |
4431 | struct xfrm_migrate *mp; | |
80c9abaa SS |
4432 | int i, j, n = 0; |
4433 | ||
4434 | write_lock_bh(&pol->lock); | |
12a169e7 | 4435 | if (unlikely(pol->walk.dead)) { |
80c9abaa | 4436 | /* target policy has been deleted */ |
bd122403 | 4437 | NL_SET_ERR_MSG(extack, "Target policy not found"); |
80c9abaa SS |
4438 | write_unlock_bh(&pol->lock); |
4439 | return -ENOENT; | |
4440 | } | |
4441 | ||
4442 | for (i = 0; i < pol->xfrm_nr; i++) { | |
4443 | for (j = 0, mp = m; j < num_migrate; j++, mp++) { | |
4444 | if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i])) | |
4445 | continue; | |
4446 | n++; | |
1bfcb10f HX |
4447 | if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL && |
4448 | pol->xfrm_vec[i].mode != XFRM_MODE_BEET) | |
80c9abaa SS |
4449 | continue; |
4450 | /* update endpoints */ | |
4451 | memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr, | |
4452 | sizeof(pol->xfrm_vec[i].id.daddr)); | |
4453 | memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr, | |
4454 | sizeof(pol->xfrm_vec[i].saddr)); | |
4455 | pol->xfrm_vec[i].encap_family = mp->new_family; | |
4456 | /* flush bundles */ | |
80c802f3 | 4457 | atomic_inc(&pol->genid); |
80c9abaa SS |
4458 | } |
4459 | } | |
4460 | ||
4461 | write_unlock_bh(&pol->lock); | |
4462 | ||
4463 | if (!n) | |
4464 | return -ENODATA; | |
4465 | ||
4466 | return 0; | |
4467 | } | |
4468 | ||
bd122403 SD |
4469 | static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate, |
4470 | struct netlink_ext_ack *extack) | |
80c9abaa SS |
4471 | { |
4472 | int i, j; | |
4473 | ||
bd122403 SD |
4474 | if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) { |
4475 | NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)"); | |
80c9abaa | 4476 | return -EINVAL; |
bd122403 | 4477 | } |
80c9abaa SS |
4478 | |
4479 | for (i = 0; i < num_migrate; i++) { | |
80c9abaa | 4480 | if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) || |
bd122403 SD |
4481 | xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) { |
4482 | NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null"); | |
80c9abaa | 4483 | return -EINVAL; |
bd122403 | 4484 | } |
80c9abaa SS |
4485 | |
4486 | /* check if there is any duplicated entry */ | |
4487 | for (j = i + 1; j < num_migrate; j++) { | |
4488 | if (!memcmp(&m[i].old_daddr, &m[j].old_daddr, | |
4489 | sizeof(m[i].old_daddr)) && | |
4490 | !memcmp(&m[i].old_saddr, &m[j].old_saddr, | |
4491 | sizeof(m[i].old_saddr)) && | |
4492 | m[i].proto == m[j].proto && | |
4493 | m[i].mode == m[j].mode && | |
4494 | m[i].reqid == m[j].reqid && | |
bd122403 SD |
4495 | m[i].old_family == m[j].old_family) { |
4496 | NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique"); | |
80c9abaa | 4497 | return -EINVAL; |
bd122403 | 4498 | } |
80c9abaa SS |
4499 | } |
4500 | } | |
4501 | ||
4502 | return 0; | |
4503 | } | |
4504 | ||
b4b7c0b3 | 4505 | int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, |
13c1d189 | 4506 | struct xfrm_migrate *m, int num_migrate, |
4ab47d47 | 4507 | struct xfrm_kmaddress *k, struct net *net, |
bd122403 SD |
4508 | struct xfrm_encap_tmpl *encap, u32 if_id, |
4509 | struct netlink_ext_ack *extack) | |
80c9abaa SS |
4510 | { |
4511 | int i, err, nx_cur = 0, nx_new = 0; | |
4512 | struct xfrm_policy *pol = NULL; | |
4513 | struct xfrm_state *x, *xc; | |
4514 | struct xfrm_state *x_cur[XFRM_MAX_DEPTH]; | |
4515 | struct xfrm_state *x_new[XFRM_MAX_DEPTH]; | |
4516 | struct xfrm_migrate *mp; | |
4517 | ||
7bab0963 | 4518 | /* Stage 0 - sanity checks */ |
bd122403 | 4519 | err = xfrm_migrate_check(m, num_migrate, extack); |
f157c416 | 4520 | if (err < 0) |
80c9abaa SS |
4521 | goto out; |
4522 | ||
7bab0963 | 4523 | if (dir >= XFRM_POLICY_MAX) { |
bd122403 | 4524 | NL_SET_ERR_MSG(extack, "Invalid policy direction"); |
7bab0963 VD |
4525 | err = -EINVAL; |
4526 | goto out; | |
4527 | } | |
4528 | ||
80c9abaa | 4529 | /* Stage 1 - find policy */ |
f157c416 SD |
4530 | pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id); |
4531 | if (!pol) { | |
bd122403 | 4532 | NL_SET_ERR_MSG(extack, "Target policy not found"); |
80c9abaa SS |
4533 | err = -ENOENT; |
4534 | goto out; | |
4535 | } | |
4536 | ||
4537 | /* Stage 2 - find and update state(s) */ | |
4538 | for (i = 0, mp = m; i < num_migrate; i++, mp++) { | |
c1aca308 | 4539 | if ((x = xfrm_migrate_state_find(mp, net, if_id))) { |
80c9abaa SS |
4540 | x_cur[nx_cur] = x; |
4541 | nx_cur++; | |
4ab47d47 AA |
4542 | xc = xfrm_state_migrate(x, mp, encap); |
4543 | if (xc) { | |
80c9abaa SS |
4544 | x_new[nx_new] = xc; |
4545 | nx_new++; | |
4546 | } else { | |
4547 | err = -ENODATA; | |
4548 | goto restore_state; | |
4549 | } | |
4550 | } | |
4551 | } | |
4552 | ||
4553 | /* Stage 3 - update policy */ | |
bd122403 | 4554 | err = xfrm_policy_migrate(pol, m, num_migrate, extack); |
f157c416 | 4555 | if (err < 0) |
80c9abaa SS |
4556 | goto restore_state; |
4557 | ||
4558 | /* Stage 4 - delete old state(s) */ | |
4559 | if (nx_cur) { | |
4560 | xfrm_states_put(x_cur, nx_cur); | |
4561 | xfrm_states_delete(x_cur, nx_cur); | |
4562 | } | |
4563 | ||
4564 | /* Stage 5 - announce */ | |
8bafd730 | 4565 | km_migrate(sel, dir, type, m, num_migrate, k, encap); |
80c9abaa SS |
4566 | |
4567 | xfrm_pol_put(pol); | |
4568 | ||
4569 | return 0; | |
4570 | out: | |
4571 | return err; | |
4572 | ||
4573 | restore_state: | |
4574 | if (pol) | |
4575 | xfrm_pol_put(pol); | |
4576 | if (nx_cur) | |
4577 | xfrm_states_put(x_cur, nx_cur); | |
4578 | if (nx_new) | |
4579 | xfrm_states_delete(x_new, nx_new); | |
4580 | ||
4581 | return err; | |
4582 | } | |
e610e679 | 4583 | EXPORT_SYMBOL(xfrm_migrate); |
80c9abaa | 4584 | #endif |