projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xfrm: remove the xfrm_queue_purge definition
[linux-2.6-block.git] / net / xfrm / xfrm_policy.c
CommitLineData
a716c119 1/*
1da177e4
LT		 2 * xfrm_policy.c
3 *
4 * Changes:
5 * Mitsuru KANDA @USAGI
6 * Kazunori MIYAZAWA @USAGI
7 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8 * IPv6 support
9 * Kazunori MIYAZAWA @USAGI
10 * YOSHIFUJI Hideaki
11 * Split up af-specific portion
12 * Derek Atkins <derek@ihtfp.com> Add the post_input processor
df71837d 13 *
1da177e4
LT		 14 */
15
66cdb3ca 16#include <linux/err.h>
1da177e4
LT		 17#include <linux/slab.h>
18#include <linux/kmod.h>
19#include <linux/list.h>
20#include <linux/spinlock.h>
21#include <linux/workqueue.h>
22#include <linux/notifier.h>
23#include <linux/netdevice.h>
eb9c7ebe 24#include <linux/netfilter.h>
1da177e4 25#include <linux/module.h>
2518c7c2 26#include <linux/cache.h>
68277acc 27#include <linux/audit.h>
25ee3286 28#include <net/dst.h>
6ce74ec7 29#include <net/flow.h>
1da177e4
LT		 30#include <net/xfrm.h>
31#include <net/ip.h>
558f82ef
MN		 32#ifdef CONFIG_XFRM_STATISTICS
33#include <net/snmp.h>
34#endif
1da177e4 35
44e36b42
DM		 36#include "xfrm_hash.h"
37
a0073fe1
SK		 38#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
39#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
40#define XFRM_MAX_QUEUE_LEN 100
41
b8c203b2
SK		 42struct xfrm_flo {
43 struct dst_entry *dst_orig;
44 u8 flags;
45};
46
418a99ac
PJ		 47static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
48static struct xfrm_policy_afinfo __rcu *xfrm_policy_afinfo[NPROTO]
49 __read_mostly;
1da177e4 50
e18b890b 51static struct kmem_cache *xfrm_dst_cache __read_mostly;
1da177e4 52
25ee3286 53static void xfrm_init_pmtu(struct dst_entry *dst);
80c802f3 54static int stale_bundle(struct dst_entry *dst);
12fdb4d3 55static int xfrm_bundle_ok(struct xfrm_dst *xdst);
a0073fe1 56static void xfrm_policy_queue_process(unsigned long arg);
1da177e4 57
12bfa8bd 58static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
29fa0b30
WY		 59static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
60 int dir);
61
bc9b35ad 62static inline bool
200ce96e 63__xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
77681021 64{
7e1dc7b6
DM		 65 const struct flowi4 *fl4 = &fl->u.ip4;
66
26bff940
AD		 67 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
68 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
7e1dc7b6
DM		 69 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
70 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
71 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
72 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
77681021
AM		 73}
74
bc9b35ad 75static inline bool
200ce96e 76__xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
77681021 77{
7e1dc7b6
DM		 78 const struct flowi6 *fl6 = &fl->u.ip6;
79
80 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
81 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
82 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
83 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
84 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
85 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
77681021
AM		 86}
87
bc9b35ad
DM		 88bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
89 unsigned short family)
77681021
AM		 90{
91 switch (family) {
92 case AF_INET:
93 return __xfrm4_selector_match(sel, fl);
94 case AF_INET6:
95 return __xfrm6_selector_match(sel, fl);
96 }
bc9b35ad 97 return false;
77681021
AM		 98}
99
ef8531b6
ED		 100static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
101{
102 struct xfrm_policy_afinfo *afinfo;
103
104 if (unlikely(family >= NPROTO))
105 return NULL;
106 rcu_read_lock();
107 afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
108 if (unlikely(!afinfo))
109 rcu_read_unlock();
110 return afinfo;
111}
112
113static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
114{
115 rcu_read_unlock();
116}
117
c5b3cf46 118static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos,
6418c4e0
DM		 119 const xfrm_address_t *saddr,
120 const xfrm_address_t *daddr,
9bb182a7
YH		 121 int family)
122{
123 struct xfrm_policy_afinfo *afinfo;
124 struct dst_entry *dst;
125
126 afinfo = xfrm_policy_get_afinfo(family);
127 if (unlikely(afinfo == NULL))
128 return ERR_PTR(-EAFNOSUPPORT);
129
c5b3cf46 130 dst = afinfo->dst_lookup(net, tos, saddr, daddr);
9bb182a7
YH		 131
132 xfrm_policy_put_afinfo(afinfo);
133
134 return dst;
135}
136
25ee3286 137static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos,
9bb182a7
YH		 138 xfrm_address_t *prev_saddr,
139 xfrm_address_t *prev_daddr,
25ee3286 140 int family)
1da177e4 141{
c5b3cf46 142 struct net *net = xs_net(x);
66cdb3ca
HX		 143 xfrm_address_t *saddr = &x->props.saddr;
144 xfrm_address_t *daddr = &x->id.daddr;
66cdb3ca
HX		 145 struct dst_entry *dst;
146
9bb182a7 147 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
66cdb3ca 148 saddr = x->coaddr;
9bb182a7
YH		 149 daddr = prev_daddr;
150 }
151 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
152 saddr = prev_saddr;
66cdb3ca 153 daddr = x->coaddr;
9bb182a7 154 }
1da177e4 155
c5b3cf46 156 dst = __xfrm_dst_lookup(net, tos, saddr, daddr, family);
9bb182a7
YH		 157
158 if (!IS_ERR(dst)) {
159 if (prev_saddr != saddr)
160 memcpy(prev_saddr, saddr, sizeof(*prev_saddr));
161 if (prev_daddr != daddr)
162 memcpy(prev_daddr, daddr, sizeof(*prev_daddr));
163 }
1da177e4 164
66cdb3ca 165 return dst;
1da177e4 166}
1da177e4 167
1da177e4
LT		 168static inline unsigned long make_jiffies(long secs)
169{
170 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
171 return MAX_SCHEDULE_TIMEOUT-1;
172 else
a716c119 173 return secs*HZ;
1da177e4
LT		 174}
175
176static void xfrm_policy_timer(unsigned long data)
177{
3e94c2dc 178 struct xfrm_policy *xp = (struct xfrm_policy *)data;
9d729f72 179 unsigned long now = get_seconds();
1da177e4
LT		 180 long next = LONG_MAX;
181 int warn = 0;
182 int dir;
183
184 read_lock(&xp->lock);
185
ea2dea9d 186 if (unlikely(xp->walk.dead))
1da177e4
LT		 187 goto out;
188
77d8d7a6 189 dir = xfrm_policy_id2dir(xp->index);
1da177e4
LT		 190
191 if (xp->lft.hard_add_expires_seconds) {
192 long tmo = xp->lft.hard_add_expires_seconds +
193 xp->curlft.add_time - now;
194 if (tmo <= 0)
195 goto expired;
196 if (tmo < next)
197 next = tmo;
198 }
199 if (xp->lft.hard_use_expires_seconds) {
200 long tmo = xp->lft.hard_use_expires_seconds +
201 (xp->curlft.use_time ? : xp->curlft.add_time) - now;
202 if (tmo <= 0)
203 goto expired;
204 if (tmo < next)
205 next = tmo;
206 }
207 if (xp->lft.soft_add_expires_seconds) {
208 long tmo = xp->lft.soft_add_expires_seconds +
209 xp->curlft.add_time - now;
210 if (tmo <= 0) {
211 warn = 1;
212 tmo = XFRM_KM_TIMEOUT;
213 }
214 if (tmo < next)
215 next = tmo;
216 }
217 if (xp->lft.soft_use_expires_seconds) {
218 long tmo = xp->lft.soft_use_expires_seconds +
219 (xp->curlft.use_time ? : xp->curlft.add_time) - now;
220 if (tmo <= 0) {
221 warn = 1;
222 tmo = XFRM_KM_TIMEOUT;
223 }
224 if (tmo < next)
225 next = tmo;
226 }
227
228 if (warn)
6c5c8ca7 229 km_policy_expired(xp, dir, 0, 0);
1da177e4
LT		 230 if (next != LONG_MAX &&
231 !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
232 xfrm_pol_hold(xp);
233
234out:
235 read_unlock(&xp->lock);
236 xfrm_pol_put(xp);
237 return;
238
239expired:
240 read_unlock(&xp->lock);
4666faab 241 if (!xfrm_policy_delete(xp, dir))
6c5c8ca7 242 km_policy_expired(xp, dir, 1, 0);
1da177e4
LT		 243 xfrm_pol_put(xp);
244}
245
fe1a5f03
TT		 246static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
247{
248 struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
249
250 if (unlikely(pol->walk.dead))
251 flo = NULL;
252 else
253 xfrm_pol_hold(pol);
254
255 return flo;
256}
257
258static int xfrm_policy_flo_check(struct flow_cache_object *flo)
259{
260 struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
261
262 return !pol->walk.dead;
263}
264
265static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
266{
267 xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
268}
269
270static const struct flow_cache_ops xfrm_policy_fc_ops = {
271 .get = xfrm_policy_flo_get,
272 .check = xfrm_policy_flo_check,
273 .delete = xfrm_policy_flo_delete,
274};
1da177e4
LT		 275
276/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
277 * SPD calls.
278 */
279
0331b1f3 280struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
1da177e4
LT		 281{
282 struct xfrm_policy *policy;
283
0da974f4 284 policy = kzalloc(sizeof(struct xfrm_policy), gfp);
1da177e4
LT		 285
286 if (policy) {
0331b1f3 287 write_pnet(&policy->xp_net, net);
12a169e7 288 INIT_LIST_HEAD(&policy->walk.all);
2518c7c2
DM		 289 INIT_HLIST_NODE(&policy->bydst);
290 INIT_HLIST_NODE(&policy->byidx);
1da177e4 291 rwlock_init(&policy->lock);
2518c7c2 292 atomic_set(&policy->refcnt, 1);
a0073fe1 293 skb_queue_head_init(&policy->polq.hold_queue);
b24b8a24
PE		 294 setup_timer(&policy->timer, xfrm_policy_timer,
295 (unsigned long)policy);
a0073fe1
SK		 296 setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
297 (unsigned long)policy);
fe1a5f03 298 policy->flo.ops = &xfrm_policy_fc_ops;
1da177e4
LT		 299 }
300 return policy;
301}
302EXPORT_SYMBOL(xfrm_policy_alloc);
303
304/* Destroy xfrm_policy: descendant resources must be released to this moment. */
305
64c31b3f 306void xfrm_policy_destroy(struct xfrm_policy *policy)
1da177e4 307{
12a169e7 308 BUG_ON(!policy->walk.dead);
1da177e4 309
0659eea9 310 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
1da177e4
LT		 311 BUG();
312
03e1ad7b 313 security_xfrm_policy_free(policy->security);
1da177e4
LT		 314 kfree(policy);
315}
64c31b3f 316EXPORT_SYMBOL(xfrm_policy_destroy);
1da177e4 317
1da177e4
LT		 318/* Rule must be locked. Release descentant resources, announce
319 * entry dead. The rule must be unlinked from lists to the moment.
320 */
321
322static void xfrm_policy_kill(struct xfrm_policy *policy)
323{
12a169e7 324 policy->walk.dead = 1;
1da177e4 325
285ead17 326 atomic_inc(&policy->genid);
1da177e4 327
e7d8f6cb
SK		 328 if (del_timer(&policy->polq.hold_timer))
329 xfrm_pol_put(policy);
1ee5e667 330 skb_queue_purge(&policy->polq.hold_queue);
a0073fe1 331
285ead17
TT		 332 if (del_timer(&policy->timer))
333 xfrm_pol_put(policy);
334
335 xfrm_pol_put(policy);
1da177e4
LT		 336}
337
2518c7c2
DM		 338static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
339
e92303f8 340static inline unsigned int idx_hash(struct net *net, u32 index)
2518c7c2 341{
e92303f8 342 return __idx_hash(index, net->xfrm.policy_idx_hmask);
2518c7c2
DM		 343}
344
b58555f1
CG		 345/* calculate policy hash thresholds */
346static void __get_hash_thresh(struct net *net,
347 unsigned short family, int dir,
348 u8 *dbits, u8 *sbits)
349{
350 switch (family) {
351 case AF_INET:
352 *dbits = net->xfrm.policy_bydst[dir].dbits4;
353 *sbits = net->xfrm.policy_bydst[dir].sbits4;
354 break;
355
356 case AF_INET6:
357 *dbits = net->xfrm.policy_bydst[dir].dbits6;
358 *sbits = net->xfrm.policy_bydst[dir].sbits6;
359 break;
360
361 default:
362 *dbits = 0;
363 *sbits = 0;
364 }
365}
366
5f803b58
DM		 367static struct hlist_head *policy_hash_bysel(struct net *net,
368 const struct xfrm_selector *sel,
369 unsigned short family, int dir)
2518c7c2 370{
1121994c 371 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
b58555f1
CG		 372 unsigned int hash;
373 u8 dbits;
374 u8 sbits;
375
376 __get_hash_thresh(net, family, dir, &dbits, &sbits);
377 hash = __sel_hash(sel, family, hmask, dbits, sbits);
2518c7c2
DM		 378
379 return (hash == hmask + 1 ?
1121994c
AD		 380 &net->xfrm.policy_inexact[dir] :
381 net->xfrm.policy_bydst[dir].table + hash);
2518c7c2
DM		 382}
383
5f803b58
DM		 384static struct hlist_head *policy_hash_direct(struct net *net,
385 const xfrm_address_t *daddr,
386 const xfrm_address_t *saddr,
387 unsigned short family, int dir)
2518c7c2 388{
1121994c 389 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
b58555f1
CG		 390 unsigned int hash;
391 u8 dbits;
392 u8 sbits;
393
394 __get_hash_thresh(net, family, dir, &dbits, &sbits);
395 hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
2518c7c2 396
1121994c 397 return net->xfrm.policy_bydst[dir].table + hash;
2518c7c2
DM		 398}
399
b58555f1
CG		 400static void xfrm_dst_hash_transfer(struct net *net,
401 struct hlist_head *list,
2518c7c2 402 struct hlist_head *ndsttable,
b58555f1
CG		 403 unsigned int nhashmask,
404 int dir)
2518c7c2 405{
b67bfe0d 406 struct hlist_node *tmp, *entry0 = NULL;
2518c7c2 407 struct xfrm_policy *pol;
b791160b 408 unsigned int h0 = 0;
b58555f1
CG		 409 u8 dbits;
410 u8 sbits;
2518c7c2 411
b791160b 412redo:
b67bfe0d 413 hlist_for_each_entry_safe(pol, tmp, list, bydst) {
2518c7c2
DM		 414 unsigned int h;
415
b58555f1 416 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
2518c7c2 417 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
b58555f1 418 pol->family, nhashmask, dbits, sbits);
b791160b 419 if (!entry0) {
b67bfe0d 420 hlist_del(&pol->bydst);
b791160b
YH		 421 hlist_add_head(&pol->bydst, ndsttable+h);
422 h0 = h;
423 } else {
424 if (h != h0)
425 continue;
b67bfe0d 426 hlist_del(&pol->bydst);
1d023284 427 hlist_add_behind(&pol->bydst, entry0);
b791160b 428 }
b67bfe0d 429 entry0 = &pol->bydst;
b791160b
YH		 430 }
431 if (!hlist_empty(list)) {
432 entry0 = NULL;
433 goto redo;
2518c7c2
DM		 434 }
435}
436
437static void xfrm_idx_hash_transfer(struct hlist_head *list,
438 struct hlist_head *nidxtable,
439 unsigned int nhashmask)
440{
b67bfe0d 441 struct hlist_node *tmp;
2518c7c2
DM		 442 struct xfrm_policy *pol;
443
b67bfe0d 444 hlist_for_each_entry_safe(pol, tmp, list, byidx) {
2518c7c2
DM		 445 unsigned int h;
446
447 h = __idx_hash(pol->index, nhashmask);
448 hlist_add_head(&pol->byidx, nidxtable+h);
449 }
450}
451
452static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
453{
454 return ((old_hmask + 1) << 1) - 1;
455}
456
66caf628 457static void xfrm_bydst_resize(struct net *net, int dir)
2518c7c2 458{
66caf628 459 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
2518c7c2
DM		 460 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
461 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
66caf628 462 struct hlist_head *odst = net->xfrm.policy_bydst[dir].table;
44e36b42 463 struct hlist_head *ndst = xfrm_hash_alloc(nsize);
2518c7c2
DM		 464 int i;
465
466 if (!ndst)
467 return;
468
283bc9f3 469 write_lock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2
DM		 470
471 for (i = hmask; i >= 0; i--)
b58555f1 472 xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
2518c7c2 473
66caf628
AD		 474 net->xfrm.policy_bydst[dir].table = ndst;
475 net->xfrm.policy_bydst[dir].hmask = nhashmask;
2518c7c2 476
283bc9f3 477 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2 478
44e36b42 479 xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
2518c7c2
DM		 480}
481
66caf628 482static void xfrm_byidx_resize(struct net *net, int total)
2518c7c2 483{
66caf628 484 unsigned int hmask = net->xfrm.policy_idx_hmask;
2518c7c2
DM		 485 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
486 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
66caf628 487 struct hlist_head *oidx = net->xfrm.policy_byidx;
44e36b42 488 struct hlist_head *nidx = xfrm_hash_alloc(nsize);
2518c7c2
DM		 489 int i;
490
491 if (!nidx)
492 return;
493
283bc9f3 494 write_lock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2
DM		 495
496 for (i = hmask; i >= 0; i--)
497 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
498
66caf628
AD		 499 net->xfrm.policy_byidx = nidx;
500 net->xfrm.policy_idx_hmask = nhashmask;
2518c7c2 501
283bc9f3 502 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2 503
44e36b42 504 xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
2518c7c2
DM		 505}
506
66caf628 507static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
2518c7c2 508{
66caf628
AD		 509 unsigned int cnt = net->xfrm.policy_count[dir];
510 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
2518c7c2
DM		 511
512 if (total)
513 *total += cnt;
514
515 if ((hmask + 1) < xfrm_policy_hashmax &&
516 cnt > hmask)
517 return 1;
518
519 return 0;
520}
521
66caf628 522static inline int xfrm_byidx_should_resize(struct net *net, int total)
2518c7c2 523{
66caf628 524 unsigned int hmask = net->xfrm.policy_idx_hmask;
2518c7c2
DM		 525
526 if ((hmask + 1) < xfrm_policy_hashmax &&
527 total > hmask)
528 return 1;
529
530 return 0;
531}
532
e071041b 533void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
ecfd6b18 534{
283bc9f3 535 read_lock_bh(&net->xfrm.xfrm_policy_lock);
e071041b
AD		 536 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
537 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
538 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
539 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
540 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
541 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
542 si->spdhcnt = net->xfrm.policy_idx_hmask;
ecfd6b18 543 si->spdhmcnt = xfrm_policy_hashmax;
283bc9f3 544 read_unlock_bh(&net->xfrm.xfrm_policy_lock);
ecfd6b18
JHS		 545}
546EXPORT_SYMBOL(xfrm_spd_getinfo);
2518c7c2 547
ecfd6b18 548static DEFINE_MUTEX(hash_resize_mutex);
66caf628 549static void xfrm_hash_resize(struct work_struct *work)
2518c7c2 550{
66caf628 551 struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
2518c7c2
DM		 552 int dir, total;
553
554 mutex_lock(&hash_resize_mutex);
555
556 total = 0;
53c2e285 557 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
66caf628
AD		 558 if (xfrm_bydst_should_resize(net, dir, &total))
559 xfrm_bydst_resize(net, dir);
2518c7c2 560 }
66caf628
AD		 561 if (xfrm_byidx_should_resize(net, total))
562 xfrm_byidx_resize(net, total);
2518c7c2
DM		 563
564 mutex_unlock(&hash_resize_mutex);
565}
566
880a6fab
CG		 567static void xfrm_hash_rebuild(struct work_struct *work)
568{
569 struct net *net = container_of(work, struct net,
570 xfrm.policy_hthresh.work);
571 unsigned int hmask;
572 struct xfrm_policy *pol;
573 struct xfrm_policy *policy;
574 struct hlist_head *chain;
575 struct hlist_head *odst;
576 struct hlist_node *newpos;
577 int i;
578 int dir;
579 unsigned seq;
580 u8 lbits4, rbits4, lbits6, rbits6;
581
582 mutex_lock(&hash_resize_mutex);
583
584 /* read selector prefixlen thresholds */
585 do {
586 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
587
588 lbits4 = net->xfrm.policy_hthresh.lbits4;
589 rbits4 = net->xfrm.policy_hthresh.rbits4;
590 lbits6 = net->xfrm.policy_hthresh.lbits6;
591 rbits6 = net->xfrm.policy_hthresh.rbits6;
592 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
593
594 write_lock_bh(&net->xfrm.xfrm_policy_lock);
595
596 /* reset the bydst and inexact table in all directions */
53c2e285 597 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
880a6fab
CG		 598 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
599 hmask = net->xfrm.policy_bydst[dir].hmask;
600 odst = net->xfrm.policy_bydst[dir].table;
601 for (i = hmask; i >= 0; i--)
602 INIT_HLIST_HEAD(odst + i);
603 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
604 /* dir out => dst = remote, src = local */
605 net->xfrm.policy_bydst[dir].dbits4 = rbits4;
606 net->xfrm.policy_bydst[dir].sbits4 = lbits4;
607 net->xfrm.policy_bydst[dir].dbits6 = rbits6;
608 net->xfrm.policy_bydst[dir].sbits6 = lbits6;
609 } else {
610 /* dir in/fwd => dst = local, src = remote */
611 net->xfrm.policy_bydst[dir].dbits4 = lbits4;
612 net->xfrm.policy_bydst[dir].sbits4 = rbits4;
613 net->xfrm.policy_bydst[dir].dbits6 = lbits6;
614 net->xfrm.policy_bydst[dir].sbits6 = rbits6;
615 }
616 }
617
618 /* re-insert all policies by order of creation */
619 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
620 newpos = NULL;
621 chain = policy_hash_bysel(net, &policy->selector,
622 policy->family,
623 xfrm_policy_id2dir(policy->index));
624 hlist_for_each_entry(pol, chain, bydst) {
625 if (policy->priority >= pol->priority)
626 newpos = &pol->bydst;
627 else
628 break;
629 }
630 if (newpos)
631 hlist_add_behind(&policy->bydst, newpos);
632 else
633 hlist_add_head(&policy->bydst, chain);
634 }
635
636 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
637
638 mutex_unlock(&hash_resize_mutex);
639}
640
641void xfrm_policy_hash_rebuild(struct net *net)
642{
643 schedule_work(&net->xfrm.policy_hthresh.work);
644}
645EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
646
1da177e4
LT		 647/* Generate new index... KAME seems to generate them ordered by cost
648 * of an absolute inpredictability of ordering of rules. This will not pass. */
e682adf0 649static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
1da177e4 650{
1da177e4
LT		 651 static u32 idx_generator;
652
653 for (;;) {
2518c7c2
DM		 654 struct hlist_head *list;
655 struct xfrm_policy *p;
656 u32 idx;
657 int found;
658
e682adf0
FD		 659 if (!index) {
660 idx = (idx_generator | dir);
661 idx_generator += 8;
662 } else {
663 idx = index;
664 index = 0;
665 }
666
1da177e4
LT		 667 if (idx == 0)
668 idx = 8;
1121994c 669 list = net->xfrm.policy_byidx + idx_hash(net, idx);
2518c7c2 670 found = 0;
b67bfe0d 671 hlist_for_each_entry(p, list, byidx) {
2518c7c2
DM		 672 if (p->index == idx) {
673 found = 1;
1da177e4 674 break;
2518c7c2 675 }
1da177e4 676 }
2518c7c2 677 if (!found)
1da177e4
LT		 678 return idx;
679 }
680}
681
2518c7c2
DM		 682static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
683{
684 u32 *p1 = (u32 *) s1;
685 u32 *p2 = (u32 *) s2;
686 int len = sizeof(struct xfrm_selector) / sizeof(u32);
687 int i;
688
689 for (i = 0; i < len; i++) {
690 if (p1[i] != p2[i])
691 return 1;
692 }
693
694 return 0;
695}
696
a0073fe1
SK		 697static void xfrm_policy_requeue(struct xfrm_policy *old,
698 struct xfrm_policy *new)
699{
700 struct xfrm_policy_queue *pq = &old->polq;
701 struct sk_buff_head list;
702
703 __skb_queue_head_init(&list);
704
705 spin_lock_bh(&pq->hold_queue.lock);
706 skb_queue_splice_init(&pq->hold_queue, &list);
e7d8f6cb
SK		 707 if (del_timer(&pq->hold_timer))
708 xfrm_pol_put(old);
a0073fe1
SK		 709 spin_unlock_bh(&pq->hold_queue.lock);
710
711 if (skb_queue_empty(&list))
712 return;
713
714 pq = &new->polq;
715
716 spin_lock_bh(&pq->hold_queue.lock);
717 skb_queue_splice(&list, &pq->hold_queue);
718 pq->timeout = XFRM_QUEUE_TMO_MIN;
e7d8f6cb
SK		 719 if (!mod_timer(&pq->hold_timer, jiffies))
720 xfrm_pol_hold(new);
a0073fe1
SK		 721 spin_unlock_bh(&pq->hold_queue.lock);
722}
723
7cb8a939
SK		 724static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
725 struct xfrm_policy *pol)
726{
727 u32 mark = policy->mark.v & policy->mark.m;
728
729 if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
730 return true;
731
732 if ((mark & pol->mark.m) == pol->mark.v &&
733 policy->priority == pol->priority)
734 return true;
735
736 return false;
737}
738
1da177e4
LT		 739int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
740{
1121994c 741 struct net *net = xp_net(policy);
2518c7c2
DM		 742 struct xfrm_policy *pol;
743 struct xfrm_policy *delpol;
744 struct hlist_head *chain;
b67bfe0d 745 struct hlist_node *newpos;
1da177e4 746
283bc9f3 747 write_lock_bh(&net->xfrm.xfrm_policy_lock);
1121994c 748 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
2518c7c2
DM		 749 delpol = NULL;
750 newpos = NULL;
b67bfe0d 751 hlist_for_each_entry(pol, chain, bydst) {
a6c7ab55 752 if (pol->type == policy->type &&
2518c7c2 753 !selector_cmp(&pol->selector, &policy->selector) &&
7cb8a939 754 xfrm_policy_mark_match(policy, pol) &&
a6c7ab55
HX		 755 xfrm_sec_ctx_match(pol->security, policy->security) &&
756 !WARN_ON(delpol)) {
1da177e4 757 if (excl) {
283bc9f3 758 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 759 return -EEXIST;
760 }
1da177e4
LT		 761 delpol = pol;
762 if (policy->priority > pol->priority)
763 continue;
764 } else if (policy->priority >= pol->priority) {
a6c7ab55 765 newpos = &pol->bydst;
1da177e4
LT		 766 continue;
767 }
1da177e4
LT		 768 if (delpol)
769 break;
1da177e4
LT		 770 }
771 if (newpos)
1d023284 772 hlist_add_behind(&policy->bydst, newpos);
2518c7c2
DM		 773 else
774 hlist_add_head(&policy->bydst, chain);
12bfa8bd 775 __xfrm_policy_link(policy, dir);
ca925cf1 776 atomic_inc(&net->xfrm.flow_cache_genid);
ca4c3fc2 777
778 /* After previous checking, family can either be AF_INET or AF_INET6 */
779 if (policy->family == AF_INET)
780 rt_genid_bump_ipv4(net);
781 else
782 rt_genid_bump_ipv6(net);
783
a0073fe1
SK		 784 if (delpol) {
785 xfrm_policy_requeue(delpol, policy);
29fa0b30 786 __xfrm_policy_unlink(delpol, dir);
a0073fe1 787 }
e682adf0 788 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1121994c 789 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
9d729f72 790 policy->curlft.add_time = get_seconds();
1da177e4
LT		 791 policy->curlft.use_time = 0;
792 if (!mod_timer(&policy->timer, jiffies + HZ))
793 xfrm_pol_hold(policy);
283bc9f3 794 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 795
9b78a82c 796 if (delpol)
1da177e4 797 xfrm_policy_kill(delpol);
1121994c
AD		 798 else if (xfrm_bydst_should_resize(net, dir, NULL))
799 schedule_work(&net->xfrm.policy_hash_work);
9b78a82c 800
1da177e4
LT		 801 return 0;
802}
803EXPORT_SYMBOL(xfrm_policy_insert);
804
8ca2e93b
JHS		 805struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
806 int dir, struct xfrm_selector *sel,
ef41aaa0
EP		 807 struct xfrm_sec_ctx *ctx, int delete,
808 int *err)
1da177e4 809{
2518c7c2
DM		 810 struct xfrm_policy *pol, *ret;
811 struct hlist_head *chain;
1da177e4 812
ef41aaa0 813 *err = 0;
283bc9f3 814 write_lock_bh(&net->xfrm.xfrm_policy_lock);
8d1211a6 815 chain = policy_hash_bysel(net, sel, sel->family, dir);
2518c7c2 816 ret = NULL;
b67bfe0d 817 hlist_for_each_entry(pol, chain, bydst) {
2518c7c2 818 if (pol->type == type &&
34f8d884 819 (mark & pol->mark.m) == pol->mark.v &&
2518c7c2
DM		 820 !selector_cmp(sel, &pol->selector) &&
821 xfrm_sec_ctx_match(ctx, pol->security)) {
1da177e4 822 xfrm_pol_hold(pol);
2518c7c2 823 if (delete) {
03e1ad7b
PM		 824 *err = security_xfrm_policy_delete(
825 pol->security);
ef41aaa0 826 if (*err) {
283bc9f3 827 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
ef41aaa0
EP		 828 return pol;
829 }
29fa0b30 830 __xfrm_policy_unlink(pol, dir);
2518c7c2
DM		 831 }
832 ret = pol;
1da177e4
LT		 833 break;
834 }
835 }
283bc9f3 836 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 837
fe1a5f03 838 if (ret && delete)
2518c7c2 839 xfrm_policy_kill(ret);
2518c7c2 840 return ret;
1da177e4 841}
df71837d 842EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1da177e4 843
8ca2e93b
JHS		 844struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
845 int dir, u32 id, int delete, int *err)
1da177e4 846{
2518c7c2
DM		 847 struct xfrm_policy *pol, *ret;
848 struct hlist_head *chain;
1da177e4 849
b5505c6e
HX		 850 *err = -ENOENT;
851 if (xfrm_policy_id2dir(id) != dir)
852 return NULL;
853
ef41aaa0 854 *err = 0;
283bc9f3 855 write_lock_bh(&net->xfrm.xfrm_policy_lock);
8d1211a6 856 chain = net->xfrm.policy_byidx + idx_hash(net, id);
2518c7c2 857 ret = NULL;
b67bfe0d 858 hlist_for_each_entry(pol, chain, byidx) {
34f8d884
JHS		 859 if (pol->type == type && pol->index == id &&
860 (mark & pol->mark.m) == pol->mark.v) {
1da177e4 861 xfrm_pol_hold(pol);
2518c7c2 862 if (delete) {
03e1ad7b
PM		 863 *err = security_xfrm_policy_delete(
864 pol->security);
ef41aaa0 865 if (*err) {
283bc9f3 866 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
ef41aaa0
EP		 867 return pol;
868 }
29fa0b30 869 __xfrm_policy_unlink(pol, dir);
2518c7c2
DM		 870 }
871 ret = pol;
1da177e4
LT		 872 break;
873 }
874 }
283bc9f3 875 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 876
fe1a5f03 877 if (ret && delete)
2518c7c2 878 xfrm_policy_kill(ret);
2518c7c2 879 return ret;
1da177e4
LT		 880}
881EXPORT_SYMBOL(xfrm_policy_byid);
882
4aa2e62c
JL		 883#ifdef CONFIG_SECURITY_NETWORK_XFRM
884static inline int
2e71029e 885xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1da177e4 886{
4aa2e62c
JL		 887 int dir, err = 0;
888
889 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
890 struct xfrm_policy *pol;
4aa2e62c
JL		 891 int i;
892
b67bfe0d 893 hlist_for_each_entry(pol,
33ffbbd5 894 &net->xfrm.policy_inexact[dir], bydst) {
4aa2e62c
JL		 895 if (pol->type != type)
896 continue;
03e1ad7b 897 err = security_xfrm_policy_delete(pol->security);
4aa2e62c 898 if (err) {
2e71029e 899 xfrm_audit_policy_delete(pol, 0, task_valid);
4aa2e62c
JL		 900 return err;
901 }
7dc12d6d 902 }
33ffbbd5 903 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
b67bfe0d 904 hlist_for_each_entry(pol,
33ffbbd5 905 net->xfrm.policy_bydst[dir].table + i,
4aa2e62c
JL		 906 bydst) {
907 if (pol->type != type)
908 continue;
03e1ad7b
PM		 909 err = security_xfrm_policy_delete(
910 pol->security);
4aa2e62c 911 if (err) {
ab5f5e8b 912 xfrm_audit_policy_delete(pol, 0,
2e71029e 913 task_valid);
4aa2e62c
JL		 914 return err;
915 }
916 }
917 }
918 }
919 return err;
920}
921#else
922static inline int
2e71029e 923xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
4aa2e62c
JL		 924{
925 return 0;
926}
927#endif
928
2e71029e 929int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
4aa2e62c 930{
2f1eb65f 931 int dir, err = 0, cnt = 0;
1da177e4 932
283bc9f3 933 write_lock_bh(&net->xfrm.xfrm_policy_lock);
4aa2e62c 934
2e71029e 935 err = xfrm_policy_flush_secctx_check(net, type, task_valid);
4aa2e62c
JL		 936 if (err)
937 goto out;
938
1da177e4 939 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
2518c7c2 940 struct xfrm_policy *pol;
29fa0b30 941 int i;
2518c7c2
DM		 942
943 again1:
b67bfe0d 944 hlist_for_each_entry(pol,
33ffbbd5 945 &net->xfrm.policy_inexact[dir], bydst) {
2518c7c2
DM		 946 if (pol->type != type)
947 continue;
ea2dea9d 948 __xfrm_policy_unlink(pol, dir);
283bc9f3 949 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
ea2dea9d 950 cnt++;
1da177e4 951
2e71029e 952 xfrm_audit_policy_delete(pol, 1, task_valid);
161a09e7 953
2518c7c2 954 xfrm_policy_kill(pol);
1da177e4 955
283bc9f3 956 write_lock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2
DM		 957 goto again1;
958 }
959
33ffbbd5 960 for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
2518c7c2 961 again2:
b67bfe0d 962 hlist_for_each_entry(pol,
33ffbbd5 963 net->xfrm.policy_bydst[dir].table + i,
2518c7c2
DM		 964 bydst) {
965 if (pol->type != type)
966 continue;
ea2dea9d 967 __xfrm_policy_unlink(pol, dir);
283bc9f3 968 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
ea2dea9d 969 cnt++;
2518c7c2 970
2e71029e 971 xfrm_audit_policy_delete(pol, 1, task_valid);
2518c7c2
DM		 972 xfrm_policy_kill(pol);
973
283bc9f3 974 write_lock_bh(&net->xfrm.xfrm_policy_lock);
2518c7c2
DM		 975 goto again2;
976 }
1da177e4 977 }
2518c7c2 978
1da177e4 979 }
2f1eb65f
JHS		 980 if (!cnt)
981 err = -ESRCH;
4aa2e62c 982out:
283bc9f3 983 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
4aa2e62c 984 return err;
1da177e4
LT		 985}
986EXPORT_SYMBOL(xfrm_policy_flush);
987
cdcbca7c 988int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
4c563f76 989 int (*func)(struct xfrm_policy *, int, int, void*),
1da177e4
LT		 990 void *data)
991{
12a169e7
HX		 992 struct xfrm_policy *pol;
993 struct xfrm_policy_walk_entry *x;
4c563f76
TT		 994 int error = 0;
995
996 if (walk->type >= XFRM_POLICY_TYPE_MAX &&
997 walk->type != XFRM_POLICY_TYPE_ANY)
998 return -EINVAL;
1da177e4 999
12a169e7 1000 if (list_empty(&walk->walk.all) && walk->seq != 0)
4c563f76
TT		 1001 return 0;
1002
283bc9f3 1003 write_lock_bh(&net->xfrm.xfrm_policy_lock);
12a169e7 1004 if (list_empty(&walk->walk.all))
cdcbca7c 1005 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
12a169e7
HX		 1006 else
1007 x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
cdcbca7c 1008 list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
12a169e7 1009 if (x->dead)
4c563f76 1010 continue;
12a169e7
HX		 1011 pol = container_of(x, struct xfrm_policy, walk);
1012 if (walk->type != XFRM_POLICY_TYPE_ANY &&
1013 walk->type != pol->type)
1014 continue;
1015 error = func(pol, xfrm_policy_id2dir(pol->index),
1016 walk->seq, data);
1017 if (error) {
1018 list_move_tail(&walk->walk.all, &x->all);
1019 goto out;
2518c7c2 1020 }
12a169e7 1021 walk->seq++;
1da177e4 1022 }
12a169e7 1023 if (walk->seq == 0) {
baf5d743
JHS		 1024 error = -ENOENT;
1025 goto out;
1026 }
12a169e7 1027 list_del_init(&walk->walk.all);
1da177e4 1028out:
283bc9f3 1029 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 1030 return error;
1031}
1032EXPORT_SYMBOL(xfrm_policy_walk);
1033
12a169e7
HX		 1034void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1035{
1036 INIT_LIST_HEAD(&walk->walk.all);
1037 walk->walk.dead = 1;
1038 walk->type = type;
1039 walk->seq = 0;
1040}
1041EXPORT_SYMBOL(xfrm_policy_walk_init);
1042
283bc9f3 1043void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
12a169e7
HX		 1044{
1045 if (list_empty(&walk->walk.all))
1046 return;
1047
283bc9f3 1048 write_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
12a169e7 1049 list_del(&walk->walk.all);
283bc9f3 1050 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
12a169e7
HX		 1051}
1052EXPORT_SYMBOL(xfrm_policy_walk_done);
1053
134b0fc5
JM		 1054/*
1055 * Find policy to apply to this flow.
1056 *
1057 * Returns 0 if policy found, else an -errno.
1058 */
f299d557
DM		 1059static int xfrm_policy_match(const struct xfrm_policy *pol,
1060 const struct flowi *fl,
2518c7c2 1061 u8 type, u16 family, int dir)
1da177e4 1062{
f299d557 1063 const struct xfrm_selector *sel = &pol->selector;
bc9b35ad
DM		 1064 int ret = -ESRCH;
1065 bool match;
1da177e4 1066
2518c7c2 1067 if (pol->family != family ||
1d28f42c 1068 (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
2518c7c2 1069 pol->type != type)
134b0fc5 1070 return ret;
1da177e4 1071
2518c7c2 1072 match = xfrm_selector_match(sel, fl, family);
134b0fc5 1073 if (match)
1d28f42c 1074 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
03e1ad7b 1075 dir);
2518c7c2 1076
134b0fc5 1077 return ret;
2518c7c2 1078}
1da177e4 1079
52479b62 1080static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
062cdb43 1081 const struct flowi *fl,
2518c7c2
DM		 1082 u16 family, u8 dir)
1083{
134b0fc5 1084 int err;
2518c7c2 1085 struct xfrm_policy *pol, *ret;
0b597e7e 1086 const xfrm_address_t *daddr, *saddr;
2518c7c2 1087 struct hlist_head *chain;
acba48e1 1088 u32 priority = ~0U;
df71837d 1089
2518c7c2
DM		 1090 daddr = xfrm_flowi_daddr(fl, family);
1091 saddr = xfrm_flowi_saddr(fl, family);
1092 if (unlikely(!daddr || !saddr))
1093 return NULL;
1094
283bc9f3 1095 read_lock_bh(&net->xfrm.xfrm_policy_lock);
52479b62 1096 chain = policy_hash_direct(net, daddr, saddr, family, dir);
2518c7c2 1097 ret = NULL;
b67bfe0d 1098 hlist_for_each_entry(pol, chain, bydst) {
134b0fc5
JM		 1099 err = xfrm_policy_match(pol, fl, type, family, dir);
1100 if (err) {
1101 if (err == -ESRCH)
1102 continue;
1103 else {
1104 ret = ERR_PTR(err);
1105 goto fail;
1106 }
1107 } else {
2518c7c2 1108 ret = pol;
acba48e1 1109 priority = ret->priority;
2518c7c2
DM		 1110 break;
1111 }
1112 }
52479b62 1113 chain = &net->xfrm.policy_inexact[dir];
b67bfe0d 1114 hlist_for_each_entry(pol, chain, bydst) {
134b0fc5
JM		 1115 err = xfrm_policy_match(pol, fl, type, family, dir);
1116 if (err) {
1117 if (err == -ESRCH)
1118 continue;
1119 else {
1120 ret = ERR_PTR(err);
1121 goto fail;
1122 }
1123 } else if (pol->priority < priority) {
acba48e1
DM		 1124 ret = pol;
1125 break;
1da177e4
LT		 1126 }
1127 }
acba48e1
DM		 1128 if (ret)
1129 xfrm_pol_hold(ret);
134b0fc5 1130fail:
283bc9f3 1131 read_unlock_bh(&net->xfrm.xfrm_policy_lock);
4e81bb83 1132
2518c7c2 1133 return ret;
4e81bb83
MN		 1134}
1135
80c802f3 1136static struct xfrm_policy *
73ff93cd 1137__xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
80c802f3
TT		 1138{
1139#ifdef CONFIG_XFRM_SUB_POLICY
1140 struct xfrm_policy *pol;
1141
1142 pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);
1143 if (pol != NULL)
1144 return pol;
1145#endif
1146 return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
1147}
1148
b5fb82c4
BZ		 1149static int flow_to_policy_dir(int dir)
1150{
1151 if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1152 XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1153 XFRM_POLICY_FWD == FLOW_DIR_FWD)
1154 return dir;
1155
1156 switch (dir) {
1157 default:
1158 case FLOW_DIR_IN:
1159 return XFRM_POLICY_IN;
1160 case FLOW_DIR_OUT:
1161 return XFRM_POLICY_OUT;
1162 case FLOW_DIR_FWD:
1163 return XFRM_POLICY_FWD;
1164 }
1165}
1166
fe1a5f03 1167static struct flow_cache_object *
dee9f4bc 1168xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family,
fe1a5f03 1169 u8 dir, struct flow_cache_object *old_obj, void *ctx)
4e81bb83
MN		 1170{
1171 struct xfrm_policy *pol;
fe1a5f03
TT		 1172
1173 if (old_obj)
1174 xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
4e81bb83 1175
b5fb82c4 1176 pol = __xfrm_policy_lookup(net, fl, family, flow_to_policy_dir(dir));
80c802f3 1177 if (IS_ERR_OR_NULL(pol))
fe1a5f03 1178 return ERR_CAST(pol);
fe1a5f03 1179
fe1a5f03
TT		 1180 /* Resolver returns two references:
1181 * one for cache and one for caller of flow_cache_lookup() */
1182 xfrm_pol_hold(pol);
1183
1184 return &pol->flo;
1da177e4
LT		 1185}
1186
df71837d
TJ		 1187static inline int policy_to_flow_dir(int dir)
1188{
1189 if (XFRM_POLICY_IN == FLOW_DIR_IN &&
a716c119
YH		 1190 XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1191 XFRM_POLICY_FWD == FLOW_DIR_FWD)
1192 return dir;
1193 switch (dir) {
1194 default:
1195 case XFRM_POLICY_IN:
1196 return FLOW_DIR_IN;
1197 case XFRM_POLICY_OUT:
1198 return FLOW_DIR_OUT;
1199 case XFRM_POLICY_FWD:
1200 return FLOW_DIR_FWD;
3ff50b79 1201 }
df71837d
TJ		 1202}
1203
dee9f4bc
DM		 1204static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir,
1205 const struct flowi *fl)
1da177e4
LT		 1206{
1207 struct xfrm_policy *pol;
283bc9f3 1208 struct net *net = sock_net(sk);
1da177e4 1209
283bc9f3 1210 read_lock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 1211 if ((pol = sk->sk_policy[dir]) != NULL) {
bc9b35ad
DM		 1212 bool match = xfrm_selector_match(&pol->selector, fl,
1213 sk->sk_family);
a716c119 1214 int err = 0;
df71837d 1215
3bccfbc7 1216 if (match) {
34f8d884
JHS		 1217 if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {
1218 pol = NULL;
1219 goto out;
1220 }
03e1ad7b 1221 err = security_xfrm_policy_lookup(pol->security,
1d28f42c 1222 fl->flowi_secid,
03e1ad7b 1223 policy_to_flow_dir(dir));
3bccfbc7
VY		 1224 if (!err)
1225 xfrm_pol_hold(pol);
1226 else if (err == -ESRCH)
1227 pol = NULL;
1228 else
1229 pol = ERR_PTR(err);
1230 } else
1da177e4
LT		 1231 pol = NULL;
1232 }
34f8d884 1233out:
283bc9f3 1234 read_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 1235 return pol;
1236}
1237
1238static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1239{
98806f75 1240 struct net *net = xp_net(pol);
4e81bb83 1241
98806f75 1242 list_add(&pol->walk.all, &net->xfrm.policy_all);
98806f75 1243 net->xfrm.policy_count[dir]++;
1da177e4
LT		 1244 xfrm_pol_hold(pol);
1245}
1246
1247static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1248 int dir)
1249{
98806f75
AD		 1250 struct net *net = xp_net(pol);
1251
53c2e285 1252 if (list_empty(&pol->walk.all))
2518c7c2 1253 return NULL;
1da177e4 1254
53c2e285
HX		 1255 /* Socket policies are not hashed. */
1256 if (!hlist_unhashed(&pol->bydst)) {
1257 hlist_del(&pol->bydst);
1258 hlist_del(&pol->byidx);
1259 }
1260
1261 list_del_init(&pol->walk.all);
98806f75 1262 net->xfrm.policy_count[dir]--;
2518c7c2
DM		 1263
1264 return pol;
1da177e4
LT		 1265}
1266
53c2e285
HX		 1267static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
1268{
1269 __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
1270}
1271
1272static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
1273{
1274 __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
1275}
1276
4666faab 1277int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1da177e4 1278{
283bc9f3
FD		 1279 struct net *net = xp_net(pol);
1280
1281 write_lock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 1282 pol = __xfrm_policy_unlink(pol, dir);
283bc9f3 1283 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4 1284 if (pol) {
1da177e4 1285 xfrm_policy_kill(pol);
4666faab 1286 return 0;
1da177e4 1287 }
4666faab 1288 return -ENOENT;
1da177e4 1289}
a70fcb0b 1290EXPORT_SYMBOL(xfrm_policy_delete);
1da177e4
LT		 1291
1292int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1293{
1121994c 1294 struct net *net = xp_net(pol);
1da177e4
LT		 1295 struct xfrm_policy *old_pol;
1296
4e81bb83
MN		 1297#ifdef CONFIG_XFRM_SUB_POLICY
1298 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1299 return -EINVAL;
1300#endif
1301
283bc9f3 1302 write_lock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 1303 old_pol = sk->sk_policy[dir];
1304 sk->sk_policy[dir] = pol;
1305 if (pol) {
9d729f72 1306 pol->curlft.add_time = get_seconds();
e682adf0 1307 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
53c2e285 1308 xfrm_sk_policy_link(pol, dir);
1da177e4 1309 }
a0073fe1
SK		 1310 if (old_pol) {
1311 if (pol)
1312 xfrm_policy_requeue(old_pol, pol);
1313
ea2dea9d
TT		 1314 /* Unlinking succeeds always. This is the only function
1315 * allowed to delete or replace socket policy.
1316 */
53c2e285 1317 xfrm_sk_policy_unlink(old_pol, dir);
a0073fe1 1318 }
283bc9f3 1319 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 1320
1321 if (old_pol) {
1322 xfrm_policy_kill(old_pol);
1323 }
1324 return 0;
1325}
1326
d3e40a9f 1327static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
1da177e4 1328{
0331b1f3 1329 struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
283bc9f3 1330 struct net *net = xp_net(old);
1da177e4
LT		 1331
1332 if (newp) {
1333 newp->selector = old->selector;
03e1ad7b
PM		 1334 if (security_xfrm_policy_clone(old->security,
1335 &newp->security)) {
df71837d
TJ		 1336 kfree(newp);
1337 return NULL; /* ENOMEM */
1338 }
1da177e4
LT		 1339 newp->lft = old->lft;
1340 newp->curlft = old->curlft;
fb977e2c 1341 newp->mark = old->mark;
1da177e4
LT		 1342 newp->action = old->action;
1343 newp->flags = old->flags;
1344 newp->xfrm_nr = old->xfrm_nr;
1345 newp->index = old->index;
4e81bb83 1346 newp->type = old->type;
1da177e4
LT		 1347 memcpy(newp->xfrm_vec, old->xfrm_vec,
1348 newp->xfrm_nr*sizeof(struct xfrm_tmpl));
283bc9f3 1349 write_lock_bh(&net->xfrm.xfrm_policy_lock);
53c2e285 1350 xfrm_sk_policy_link(newp, dir);
283bc9f3 1351 write_unlock_bh(&net->xfrm.xfrm_policy_lock);
1da177e4
LT		 1352 xfrm_pol_put(newp);
1353 }
1354 return newp;
1355}
1356
1357int __xfrm_sk_clone_policy(struct sock *sk)
1358{
1359 struct xfrm_policy *p0 = sk->sk_policy[0],
1360 *p1 = sk->sk_policy[1];
1361
1362 sk->sk_policy[0] = sk->sk_policy[1] = NULL;
1363 if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
1364 return -ENOMEM;
1365 if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
1366 return -ENOMEM;
1367 return 0;
1368}
1369
a1e59abf 1370static int
fbda33b2 1371xfrm_get_saddr(struct net *net, xfrm_address_t *local, xfrm_address_t *remote,
a1e59abf
PM		 1372 unsigned short family)
1373{
1374 int err;
1375 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1376
1377 if (unlikely(afinfo == NULL))
1378 return -EINVAL;
fbda33b2 1379 err = afinfo->get_saddr(net, local, remote);
a1e59abf
PM		 1380 xfrm_policy_put_afinfo(afinfo);
1381 return err;
1382}
1383
1da177e4
LT		 1384/* Resolve list of templates for the flow, given policy. */
1385
1386static int
a6c2e611
DM		 1387xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
1388 struct xfrm_state **xfrm, unsigned short family)
1da177e4 1389{
fbda33b2 1390 struct net *net = xp_net(policy);
1da177e4
LT		 1391 int nx;
1392 int i, error;
1393 xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1394 xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
a1e59abf 1395 xfrm_address_t tmp;
1da177e4 1396
9b7a787d 1397 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
1da177e4
LT		 1398 struct xfrm_state *x;
1399 xfrm_address_t *remote = daddr;
1400 xfrm_address_t *local = saddr;
1401 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1402
48b8d783
JK		 1403 if (tmpl->mode == XFRM_MODE_TUNNEL ||
1404 tmpl->mode == XFRM_MODE_BEET) {
1da177e4
LT		 1405 remote = &tmpl->id.daddr;
1406 local = &tmpl->saddr;
8444cf71
TE		 1407 if (xfrm_addr_any(local, tmpl->encap_family)) {
1408 error = xfrm_get_saddr(net, &tmp, remote, tmpl->encap_family);
a1e59abf
PM		 1409 if (error)
1410 goto fail;
1411 local = &tmp;
1412 }
1da177e4
LT		 1413 }
1414
1415 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1416
1417 if (x && x->km.state == XFRM_STATE_VALID) {
1418 xfrm[nx++] = x;
1419 daddr = remote;
1420 saddr = local;
1421 continue;
1422 }
1423 if (x) {
1424 error = (x->km.state == XFRM_STATE_ERROR ?
1425 -EINVAL : -EAGAIN);
1426 xfrm_state_put(x);
42054569 1427 } else if (error == -ESRCH) {
a4322266 1428 error = -EAGAIN;
42054569 1429 }
1da177e4
LT		 1430
1431 if (!tmpl->optional)
1432 goto fail;
1433 }
1434 return nx;
1435
1436fail:
9b7a787d 1437 for (nx--; nx >= 0; nx--)
1da177e4
LT		 1438 xfrm_state_put(xfrm[nx]);
1439 return error;
1440}
1441
4e81bb83 1442static int
a6c2e611
DM		 1443xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
1444 struct xfrm_state **xfrm, unsigned short family)
4e81bb83 1445{
41a49cc3
MN		 1446 struct xfrm_state *tp[XFRM_MAX_DEPTH];
1447 struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
4e81bb83
MN		 1448 int cnx = 0;
1449 int error;
1450 int ret;
1451 int i;
1452
1453 for (i = 0; i < npols; i++) {
1454 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1455 error = -ENOBUFS;
1456 goto fail;
1457 }
41a49cc3
MN		 1458
1459 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
4e81bb83
MN		 1460 if (ret < 0) {
1461 error = ret;
1462 goto fail;
1463 } else
1464 cnx += ret;
1465 }
1466
41a49cc3
MN		 1467 /* found states are sorted for outbound processing */
1468 if (npols > 1)
1469 xfrm_state_sort(xfrm, tpp, cnx, family);
1470
4e81bb83
MN		 1471 return cnx;
1472
1473 fail:
9b7a787d 1474 for (cnx--; cnx >= 0; cnx--)
41a49cc3 1475 xfrm_state_put(tpp[cnx]);
4e81bb83
MN		 1476 return error;
1477
1478}
1479
1da177e4
LT		 1480/* Check that the bundle accepts the flow and its components are
1481 * still valid.
1482 */
1483
05d84025 1484static inline int xfrm_get_tos(const struct flowi *fl, int family)
25ee3286
HX		 1485{
1486 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1487 int tos;
1da177e4 1488
25ee3286
HX		 1489 if (!afinfo)
1490 return -EINVAL;
1491
1492 tos = afinfo->get_tos(fl);
1493
1494 xfrm_policy_put_afinfo(afinfo);
1495
1496 return tos;
1497}
1498
80c802f3
TT		 1499static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
1500{
1501 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1502 struct dst_entry *dst = &xdst->u.dst;
1503
1504 if (xdst->route == NULL) {
1505 /* Dummy bundle - if it has xfrms we were not
1506 * able to build bundle as template resolution failed.
1507 * It means we need to try again resolving. */
1508 if (xdst->num_xfrms > 0)
1509 return NULL;
a0073fe1
SK		 1510 } else if (dst->flags & DST_XFRM_QUEUE) {
1511 return NULL;
80c802f3
TT		 1512 } else {
1513 /* Real bundle */
1514 if (stale_bundle(dst))
1515 return NULL;
1516 }
1517
1518 dst_hold(dst);
1519 return flo;
1520}
1521
1522static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
1523{
1524 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1525 struct dst_entry *dst = &xdst->u.dst;
1526
1527 if (!xdst->route)
1528 return 0;
1529 if (stale_bundle(dst))
1530 return 0;
1531
1532 return 1;
1533}
1534
1535static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
1536{
1537 struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1538 struct dst_entry *dst = &xdst->u.dst;
1539
1540 dst_free(dst);
1541}
1542
1543static const struct flow_cache_ops xfrm_bundle_fc_ops = {
1544 .get = xfrm_bundle_flo_get,
1545 .check = xfrm_bundle_flo_check,
1546 .delete = xfrm_bundle_flo_delete,
1547};
1548
d7c7544c 1549static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
1da177e4 1550{
1da177e4 1551 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
d7c7544c 1552 struct dst_ops *dst_ops;
25ee3286
HX		 1553 struct xfrm_dst *xdst;
1554
1555 if (!afinfo)
1556 return ERR_PTR(-EINVAL);
1557
d7c7544c
AD		 1558 switch (family) {
1559 case AF_INET:
1560 dst_ops = &net->xfrm.xfrm4_dst_ops;
1561 break;
dfd56b8b 1562#if IS_ENABLED(CONFIG_IPV6)
d7c7544c
AD		 1563 case AF_INET6:
1564 dst_ops = &net->xfrm.xfrm6_dst_ops;
1565 break;
1566#endif
1567 default:
1568 BUG();
1569 }
f5b0a874 1570 xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0);
25ee3286 1571
d4cae562 1572 if (likely(xdst)) {
141e369d
SK		 1573 struct dst_entry *dst = &xdst->u.dst;
1574
1575 memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
0b150932 1576 xdst->flo.ops = &xfrm_bundle_fc_ops;
9d7b0fc1
PM		 1577 if (afinfo->init_dst)
1578 afinfo->init_dst(net, xdst);
d4cae562 1579 } else
0b150932 1580 xdst = ERR_PTR(-ENOBUFS);
80c802f3 1581
d4cae562
MB		 1582 xfrm_policy_put_afinfo(afinfo);
1583
25ee3286
HX		 1584 return xdst;
1585}
1586
a1b05140
MN		 1587static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1588 int nfheader_len)
1589{
1590 struct xfrm_policy_afinfo *afinfo =
1591 xfrm_policy_get_afinfo(dst->ops->family);
1592 int err;
1593
1594 if (!afinfo)
1595 return -EINVAL;
1596
1597 err = afinfo->init_path(path, dst, nfheader_len);
1598
1599 xfrm_policy_put_afinfo(afinfo);
1600
1601 return err;
1602}
1603
87c1e12b 1604static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
0c7b3eef 1605 const struct flowi *fl)
25ee3286
HX		 1606{
1607 struct xfrm_policy_afinfo *afinfo =
1608 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1609 int err;
1610
1611 if (!afinfo)
1da177e4 1612 return -EINVAL;
25ee3286 1613
87c1e12b 1614 err = afinfo->fill_dst(xdst, dev, fl);
25ee3286 1615
1da177e4 1616 xfrm_policy_put_afinfo(afinfo);
25ee3286 1617
1da177e4
LT		 1618 return err;
1619}
1620
80c802f3 1621
25ee3286
HX		 1622/* Allocate chain of dst_entry's, attach known xfrm's, calculate
1623 * all the metrics... Shortly, bundle a bundle.
1624 */
1625
1626static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1627 struct xfrm_state **xfrm, int nx,
98313ada 1628 const struct flowi *fl,
25ee3286
HX		 1629 struct dst_entry *dst)
1630{
d7c7544c 1631 struct net *net = xp_net(policy);
25ee3286
HX		 1632 unsigned long now = jiffies;
1633 struct net_device *dev;
43a4dea4 1634 struct xfrm_mode *inner_mode;
25ee3286
HX		 1635 struct dst_entry *dst_prev = NULL;
1636 struct dst_entry *dst0 = NULL;
1637 int i = 0;
1638 int err;
1639 int header_len = 0;
a1b05140 1640 int nfheader_len = 0;
25ee3286
HX		 1641 int trailer_len = 0;
1642 int tos;
1643 int family = policy->selector.family;
9bb182a7
YH		 1644 xfrm_address_t saddr, daddr;
1645
1646 xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
25ee3286
HX		 1647
1648 tos = xfrm_get_tos(fl, family);
1649 err = tos;
1650 if (tos < 0)
1651 goto put_states;
1652
1653 dst_hold(dst);
1654
1655 for (; i < nx; i++) {
d7c7544c 1656 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
25ee3286
HX		 1657 struct dst_entry *dst1 = &xdst->u.dst;
1658
1659 err = PTR_ERR(xdst);
1660 if (IS_ERR(xdst)) {
1661 dst_release(dst);
1662 goto put_states;
1663 }
1664
43a4dea4
SK		 1665 if (xfrm[i]->sel.family == AF_UNSPEC) {
1666 inner_mode = xfrm_ip2inner_mode(xfrm[i],
1667 xfrm_af2proto(family));
1668 if (!inner_mode) {
1669 err = -EAFNOSUPPORT;
1670 dst_release(dst);
1671 goto put_states;
1672 }
1673 } else
1674 inner_mode = xfrm[i]->inner_mode;
1675
25ee3286
HX		 1676 if (!dst_prev)
1677 dst0 = dst1;
1678 else {
1679 dst_prev->child = dst_clone(dst1);
1680 dst1->flags |= DST_NOHASH;
1681 }
1682
1683 xdst->route = dst;
defb3519 1684 dst_copy_metrics(dst1, dst);
25ee3286
HX		 1685
1686 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1687 family = xfrm[i]->props.family;
9bb182a7
YH		 1688 dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr,
1689 family);
25ee3286
HX		 1690 err = PTR_ERR(dst);
1691 if (IS_ERR(dst))
1692 goto put_states;
1693 } else
1694 dst_hold(dst);
1695
1696 dst1->xfrm = xfrm[i];
80c802f3 1697 xdst->xfrm_genid = xfrm[i]->genid;
25ee3286 1698
f5b0a874 1699 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
25ee3286
HX		 1700 dst1->flags |= DST_HOST;
1701 dst1->lastuse = now;
1702
1703 dst1->input = dst_discard;
43a4dea4 1704 dst1->output = inner_mode->afinfo->output;
25ee3286
HX		 1705
1706 dst1->next = dst_prev;
1707 dst_prev = dst1;
1708
1709 header_len += xfrm[i]->props.header_len;
a1b05140
MN		 1710 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1711 nfheader_len += xfrm[i]->props.header_len;
25ee3286
HX		 1712 trailer_len += xfrm[i]->props.trailer_len;
1713 }
1714
1715 dst_prev->child = dst;
1716 dst0->path = dst;
1717
1718 err = -ENODEV;
1719 dev = dst->dev;
1720 if (!dev)
1721 goto free_dst;
1722
a1b05140 1723 xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
25ee3286
HX		 1724 xfrm_init_pmtu(dst_prev);
1725
1726 for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1727 struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev;
1728
87c1e12b 1729 err = xfrm_fill_dst(xdst, dev, fl);
25ee3286
HX		 1730 if (err)
1731 goto free_dst;
1732
1733 dst_prev->header_len = header_len;
1734 dst_prev->trailer_len = trailer_len;
1735 header_len -= xdst->u.dst.xfrm->props.header_len;
1736 trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1737 }
1738
1739out:
1740 return dst0;
1741
1742put_states:
1743 for (; i < nx; i++)
1744 xfrm_state_put(xfrm[i]);
1745free_dst:
1746 if (dst0)
1747 dst_free(dst0);
1748 dst0 = ERR_PTR(err);
1749 goto out;
1750}
1751
da7c224b 1752#ifdef CONFIG_XFRM_SUB_POLICY
be7928d2 1753static int xfrm_dst_alloc_copy(void **target, const void *src, int size)
157bfc25
MN		 1754{
1755 if (!*target) {
1756 *target = kmalloc(size, GFP_ATOMIC);
1757 if (!*target)
1758 return -ENOMEM;
1759 }
be7928d2 1760
157bfc25
MN		 1761 memcpy(*target, src, size);
1762 return 0;
1763}
da7c224b 1764#endif
157bfc25 1765
be7928d2
DB		 1766static int xfrm_dst_update_parent(struct dst_entry *dst,
1767 const struct xfrm_selector *sel)
157bfc25
MN		 1768{
1769#ifdef CONFIG_XFRM_SUB_POLICY
1770 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1771 return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1772 sel, sizeof(*sel));
1773#else
1774 return 0;
1775#endif
1776}
1777
be7928d2
DB		 1778static int xfrm_dst_update_origin(struct dst_entry *dst,
1779 const struct flowi *fl)
157bfc25
MN		 1780{
1781#ifdef CONFIG_XFRM_SUB_POLICY
1782 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1783 return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl));
1784#else
1785 return 0;
1786#endif
1787}
1da177e4 1788
73ff93cd 1789static int xfrm_expand_policies(const struct flowi *fl, u16 family,
80c802f3
TT		 1790 struct xfrm_policy **pols,
1791 int *num_pols, int *num_xfrms)
1792{
1793 int i;
1794
1795 if (*num_pols == 0 || !pols[0]) {
1796 *num_pols = 0;
1797 *num_xfrms = 0;
1798 return 0;
1799 }
1800 if (IS_ERR(pols[0]))
1801 return PTR_ERR(pols[0]);
1802
1803 *num_xfrms = pols[0]->xfrm_nr;
1804
1805#ifdef CONFIG_XFRM_SUB_POLICY
1806 if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1807 pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1808 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1809 XFRM_POLICY_TYPE_MAIN,
1810 fl, family,
1811 XFRM_POLICY_OUT);
1812 if (pols[1]) {
1813 if (IS_ERR(pols[1])) {
1814 xfrm_pols_put(pols, *num_pols);
1815 return PTR_ERR(pols[1]);
1816 }
02d0892f 1817 (*num_pols)++;
80c802f3
TT		 1818 (*num_xfrms) += pols[1]->xfrm_nr;
1819 }
1820 }
1821#endif
1822 for (i = 0; i < *num_pols; i++) {
1823 if (pols[i]->action != XFRM_POLICY_ALLOW) {
1824 *num_xfrms = -1;
1825 break;
1826 }
1827 }
1828
1829 return 0;
1830
1831}
1832
1833static struct xfrm_dst *
1834xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
4ca2e685 1835 const struct flowi *fl, u16 family,
80c802f3
TT		 1836 struct dst_entry *dst_orig)
1837{
1838 struct net *net = xp_net(pols[0]);
1839 struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1840 struct dst_entry *dst;
1841 struct xfrm_dst *xdst;
1842 int err;
1843
1844 /* Try to instantiate a bundle */
1845 err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
d809ec89
TT		 1846 if (err <= 0) {
1847 if (err != 0 && err != -EAGAIN)
80c802f3
TT		 1848 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1849 return ERR_PTR(err);
1850 }
1851
1852 dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig);
1853 if (IS_ERR(dst)) {
1854 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1855 return ERR_CAST(dst);
1856 }
1857
1858 xdst = (struct xfrm_dst *)dst;
1859 xdst->num_xfrms = err;
1860 if (num_pols > 1)
1861 err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1862 else
1863 err = xfrm_dst_update_origin(dst, fl);
1864 if (unlikely(err)) {
1865 dst_free(dst);
1866 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1867 return ERR_PTR(err);
1868 }
1869
1870 xdst->num_pols = num_pols;
3e94c2dc 1871 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
80c802f3
TT		 1872 xdst->policy_genid = atomic_read(&pols[0]->genid);
1873
1874 return xdst;
1875}
1876
a0073fe1
SK		 1877static void xfrm_policy_queue_process(unsigned long arg)
1878{
a0073fe1
SK		 1879 struct sk_buff *skb;
1880 struct sock *sk;
1881 struct dst_entry *dst;
a0073fe1
SK		 1882 struct xfrm_policy *pol = (struct xfrm_policy *)arg;
1883 struct xfrm_policy_queue *pq = &pol->polq;
1884 struct flowi fl;
1885 struct sk_buff_head list;
1886
1887 spin_lock(&pq->hold_queue.lock);
1888 skb = skb_peek(&pq->hold_queue);
2bb53e25
SK		 1889 if (!skb) {
1890 spin_unlock(&pq->hold_queue.lock);
1891 goto out;
1892 }
a0073fe1
SK		 1893 dst = skb_dst(skb);
1894 sk = skb->sk;
1895 xfrm_decode_session(skb, &fl, dst->ops->family);
1896 spin_unlock(&pq->hold_queue.lock);
1897
1898 dst_hold(dst->path);
1899 dst = xfrm_lookup(xp_net(pol), dst->path, &fl,
1900 sk, 0);
1901 if (IS_ERR(dst))
1902 goto purge_queue;
1903
1904 if (dst->flags & DST_XFRM_QUEUE) {
1905 dst_release(dst);
1906
1907 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
1908 goto purge_queue;
1909
1910 pq->timeout = pq->timeout << 1;
e7d8f6cb
SK		 1911 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
1912 xfrm_pol_hold(pol);
1913 goto out;
a0073fe1
SK		 1914 }
1915
1916 dst_release(dst);
1917
1918 __skb_queue_head_init(&list);
1919
1920 spin_lock(&pq->hold_queue.lock);
1921 pq->timeout = 0;
1922 skb_queue_splice_init(&pq->hold_queue, &list);
1923 spin_unlock(&pq->hold_queue.lock);
1924
1925 while (!skb_queue_empty(&list)) {
1926 skb = __skb_dequeue(&list);
1927
1928 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
1929 dst_hold(skb_dst(skb)->path);
1930 dst = xfrm_lookup(xp_net(pol), skb_dst(skb)->path,
1931 &fl, skb->sk, 0);
1932 if (IS_ERR(dst)) {
a0073fe1
SK		 1933 kfree_skb(skb);
1934 continue;
1935 }
1936
1937 nf_reset(skb);
1938 skb_dst_drop(skb);
1939 skb_dst_set(skb, dst);
1940
5c1e9f2c 1941 dst_output(skb);
a0073fe1
SK		 1942 }
1943
e7d8f6cb
SK		 1944out:
1945 xfrm_pol_put(pol);
a0073fe1
SK		 1946 return;
1947
1948purge_queue:
1949 pq->timeout = 0;
1ee5e667 1950 skb_queue_purge(&pq->hold_queue);
e7d8f6cb 1951 xfrm_pol_put(pol);
a0073fe1
SK		 1952}
1953
aad88724 1954static int xdst_queue_output(struct sock *sk, struct sk_buff *skb)
a0073fe1
SK		 1955{
1956 unsigned long sched_next;
1957 struct dst_entry *dst = skb_dst(skb);
1958 struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
e7d8f6cb
SK		 1959 struct xfrm_policy *pol = xdst->pols[0];
1960 struct xfrm_policy_queue *pq = &pol->polq;
4d53eff4 1961
39bb5e62 1962 if (unlikely(skb_fclone_busy(sk, skb))) {
4d53eff4
SK		 1963 kfree_skb(skb);
1964 return 0;
1965 }
a0073fe1
SK		 1966
1967 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
1968 kfree_skb(skb);
1969 return -EAGAIN;
1970 }
1971
1972 skb_dst_force(skb);
a0073fe1
SK		 1973
1974 spin_lock_bh(&pq->hold_queue.lock);
1975
1976 if (!pq->timeout)
1977 pq->timeout = XFRM_QUEUE_TMO_MIN;
1978
1979 sched_next = jiffies + pq->timeout;
1980
1981 if (del_timer(&pq->hold_timer)) {
1982 if (time_before(pq->hold_timer.expires, sched_next))
1983 sched_next = pq->hold_timer.expires;
e7d8f6cb 1984 xfrm_pol_put(pol);
a0073fe1
SK		 1985 }
1986
1987 __skb_queue_tail(&pq->hold_queue, skb);
e7d8f6cb
SK		 1988 if (!mod_timer(&pq->hold_timer, sched_next))
1989 xfrm_pol_hold(pol);
a0073fe1
SK		 1990
1991 spin_unlock_bh(&pq->hold_queue.lock);
1992
1993 return 0;
1994}
1995
1996static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
b8c203b2 1997 struct xfrm_flo *xflo,
a0073fe1
SK		 1998 const struct flowi *fl,
1999 int num_xfrms,
2000 u16 family)
2001{
2002 int err;
2003 struct net_device *dev;
b8c203b2 2004 struct dst_entry *dst;
a0073fe1
SK		 2005 struct dst_entry *dst1;
2006 struct xfrm_dst *xdst;
2007
2008 xdst = xfrm_alloc_dst(net, family);
2009 if (IS_ERR(xdst))
2010 return xdst;
2011
b8c203b2
SK		 2012 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
2013 net->xfrm.sysctl_larval_drop ||
2014 num_xfrms <= 0)
a0073fe1
SK		 2015 return xdst;
2016
b8c203b2 2017 dst = xflo->dst_orig;
a0073fe1
SK		 2018 dst1 = &xdst->u.dst;
2019 dst_hold(dst);
2020 xdst->route = dst;
2021
2022 dst_copy_metrics(dst1, dst);
2023
2024 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2025 dst1->flags |= DST_HOST | DST_XFRM_QUEUE;
2026 dst1->lastuse = jiffies;
2027
2028 dst1->input = dst_discard;
2029 dst1->output = xdst_queue_output;
2030
2031 dst_hold(dst);
2032 dst1->child = dst;
2033 dst1->path = dst;
2034
2035 xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
2036
2037 err = -ENODEV;
2038 dev = dst->dev;
2039 if (!dev)
2040 goto free_dst;
2041
2042 err = xfrm_fill_dst(xdst, dev, fl);
2043 if (err)
2044 goto free_dst;
2045
2046out:
2047 return xdst;
2048
2049free_dst:
2050 dst_release(dst1);
2051 xdst = ERR_PTR(err);
2052 goto out;
2053}
2054
80c802f3 2055static struct flow_cache_object *
dee9f4bc 2056xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,
80c802f3
TT		 2057 struct flow_cache_object *oldflo, void *ctx)
2058{
b8c203b2 2059 struct xfrm_flo *xflo = (struct xfrm_flo *)ctx;
80c802f3
TT		 2060 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2061 struct xfrm_dst *xdst, *new_xdst;
2062 int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
2063
2064 /* Check if the policies from old bundle are usable */
2065 xdst = NULL;
2066 if (oldflo) {
2067 xdst = container_of(oldflo, struct xfrm_dst, flo);
2068 num_pols = xdst->num_pols;
2069 num_xfrms = xdst->num_xfrms;
2070 pol_dead = 0;
2071 for (i = 0; i < num_pols; i++) {
2072 pols[i] = xdst->pols[i];
2073 pol_dead |= pols[i]->walk.dead;
2074 }
2075 if (pol_dead) {
2076 dst_free(&xdst->u.dst);
2077 xdst = NULL;
2078 num_pols = 0;
2079 num_xfrms = 0;
2080 oldflo = NULL;
2081 }
2082 }
2083
2084 /* Resolve policies to use if we couldn't get them from
2085 * previous cache entry */
2086 if (xdst == NULL) {
2087 num_pols = 1;
b5fb82c4
BZ		 2088 pols[0] = __xfrm_policy_lookup(net, fl, family,
2089 flow_to_policy_dir(dir));
80c802f3
TT		 2090 err = xfrm_expand_policies(fl, family, pols,
2091 &num_pols, &num_xfrms);
2092 if (err < 0)
2093 goto inc_error;
2094 if (num_pols == 0)
2095 return NULL;
2096 if (num_xfrms <= 0)
2097 goto make_dummy_bundle;
2098 }
2099
b8c203b2
SK		 2100 new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
2101 xflo->dst_orig);
80c802f3
TT		 2102 if (IS_ERR(new_xdst)) {
2103 err = PTR_ERR(new_xdst);
2104 if (err != -EAGAIN)
2105 goto error;
2106 if (oldflo == NULL)
2107 goto make_dummy_bundle;
2108 dst_hold(&xdst->u.dst);
2109 return oldflo;
d809ec89
TT		 2110 } else if (new_xdst == NULL) {
2111 num_xfrms = 0;
2112 if (oldflo == NULL)
2113 goto make_dummy_bundle;
2114 xdst->num_xfrms = 0;
2115 dst_hold(&xdst->u.dst);
2116 return oldflo;
80c802f3
TT		 2117 }
2118
2119 /* Kill the previous bundle */
2120 if (xdst) {
2121 /* The policies were stolen for newly generated bundle */
2122 xdst->num_pols = 0;
2123 dst_free(&xdst->u.dst);
2124 }
2125
2126 /* Flow cache does not have reference, it dst_free()'s,
2127 * but we do need to return one reference for original caller */
2128 dst_hold(&new_xdst->u.dst);
2129 return &new_xdst->flo;
2130
2131make_dummy_bundle:
2132 /* We found policies, but there's no bundles to instantiate:
2133 * either because the policy blocks, has no transformations or
2134 * we could not build template (no xfrm_states).*/
b8c203b2 2135 xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
80c802f3
TT		 2136 if (IS_ERR(xdst)) {
2137 xfrm_pols_put(pols, num_pols);
2138 return ERR_CAST(xdst);
2139 }
2140 xdst->num_pols = num_pols;
2141 xdst->num_xfrms = num_xfrms;
3e94c2dc 2142 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
80c802f3
TT		 2143
2144 dst_hold(&xdst->u.dst);
2145 return &xdst->flo;
2146
2147inc_error:
2148 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2149error:
2150 if (xdst != NULL)
2151 dst_free(&xdst->u.dst);
2152 else
2153 xfrm_pols_put(pols, num_pols);
2154 return ERR_PTR(err);
2155}
1da177e4 2156
2774c131
DM		 2157static struct dst_entry *make_blackhole(struct net *net, u16 family,
2158 struct dst_entry *dst_orig)
2159{
2160 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2161 struct dst_entry *ret;
2162
2163 if (!afinfo) {
2164 dst_release(dst_orig);
433a1954 2165 return ERR_PTR(-EINVAL);
2774c131
DM		 2166 } else {
2167 ret = afinfo->blackhole_route(net, dst_orig);
2168 }
2169 xfrm_policy_put_afinfo(afinfo);
2170
2171 return ret;
2172}
2173
1da177e4
LT		 2174/* Main function: finds/creates a bundle for given flow.
2175 *
2176 * At the moment we eat a raw IP route. Mostly to speed up lookups
2177 * on interfaces with disabled IPsec.
2178 */
452edd59
DM		 2179struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
2180 const struct flowi *fl,
2181 struct sock *sk, int flags)
1da177e4 2182{
4e81bb83 2183 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
80c802f3
TT		 2184 struct flow_cache_object *flo;
2185 struct xfrm_dst *xdst;
452edd59 2186 struct dst_entry *dst, *route;
80c802f3 2187 u16 family = dst_orig->ops->family;
df71837d 2188 u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
4b021628 2189 int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
e0d1caa7 2190
80c802f3
TT		 2191 dst = NULL;
2192 xdst = NULL;
2193 route = NULL;
4e81bb83 2194
f7944fb1 2195 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
80c802f3
TT		 2196 num_pols = 1;
2197 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
2198 err = xfrm_expand_policies(fl, family, pols,
2199 &num_pols, &num_xfrms);
2200 if (err < 0)
75b8c133 2201 goto dropdst;
80c802f3
TT		 2202
2203 if (num_pols) {
2204 if (num_xfrms <= 0) {
2205 drop_pols = num_pols;
2206 goto no_transform;
2207 }
2208
2209 xdst = xfrm_resolve_and_create_bundle(
2210 pols, num_pols, fl,
2211 family, dst_orig);
2212 if (IS_ERR(xdst)) {
2213 xfrm_pols_put(pols, num_pols);
2214 err = PTR_ERR(xdst);
2215 goto dropdst;
d809ec89
TT		 2216 } else if (xdst == NULL) {
2217 num_xfrms = 0;
2218 drop_pols = num_pols;
2219 goto no_transform;
80c802f3
TT		 2220 }
2221
b7eea454
SK		 2222 dst_hold(&xdst->u.dst);
2223 xdst->u.dst.flags |= DST_NOCACHE;
80c802f3 2224 route = xdst->route;
0aa64774 2225 }
3bccfbc7 2226 }
1da177e4 2227
80c802f3 2228 if (xdst == NULL) {
b8c203b2
SK		 2229 struct xfrm_flo xflo;
2230
2231 xflo.dst_orig = dst_orig;
2232 xflo.flags = flags;
2233
1da177e4 2234 /* To accelerate a bit... */
2518c7c2 2235 if ((dst_orig->flags & DST_NOXFRM) ||
52479b62 2236 !net->xfrm.policy_count[XFRM_POLICY_OUT])
8b7817f3 2237 goto nopol;
1da177e4 2238
80c802f3 2239 flo = flow_cache_lookup(net, fl, family, dir,
b8c203b2 2240 xfrm_bundle_lookup, &xflo);
80c802f3
TT		 2241 if (flo == NULL)
2242 goto nopol;
fe1a5f03 2243 if (IS_ERR(flo)) {
80c802f3 2244 err = PTR_ERR(flo);
75b8c133 2245 goto dropdst;
d66e37a9 2246 }
80c802f3
TT		 2247 xdst = container_of(flo, struct xfrm_dst, flo);
2248
2249 num_pols = xdst->num_pols;
2250 num_xfrms = xdst->num_xfrms;
3e94c2dc 2251 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
80c802f3
TT		 2252 route = xdst->route;
2253 }
2254
2255 dst = &xdst->u.dst;
2256 if (route == NULL && num_xfrms > 0) {
2257 /* The only case when xfrm_bundle_lookup() returns a
2258 * bundle with null route, is when the template could
2259 * not be resolved. It means policies are there, but
2260 * bundle could not be created, since we don't yet
2261 * have the xfrm_state's. We need to wait for KM to
2262 * negotiate new SA's or bail out with error.*/
2263 if (net->xfrm.sysctl_larval_drop) {
80c802f3 2264 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
ac37e251
W		 2265 err = -EREMOTE;
2266 goto error;
80c802f3 2267 }
80c802f3 2268
5b8ef341 2269 err = -EAGAIN;
80c802f3
TT		 2270
2271 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2272 goto error;
1da177e4
LT		 2273 }
2274
80c802f3
TT		 2275no_transform:
2276 if (num_pols == 0)
8b7817f3 2277 goto nopol;
1da177e4 2278
80c802f3
TT		 2279 if ((flags & XFRM_LOOKUP_ICMP) &&
2280 !(pols[0]->flags & XFRM_POLICY_ICMP)) {
2281 err = -ENOENT;
8b7817f3 2282 goto error;
80c802f3 2283 }
8b7817f3 2284
80c802f3
TT		 2285 for (i = 0; i < num_pols; i++)
2286 pols[i]->curlft.use_time = get_seconds();
8b7817f3 2287
80c802f3 2288 if (num_xfrms < 0) {
1da177e4 2289 /* Prohibit the flow */
59c9940e 2290 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
e104411b
PM		 2291 err = -EPERM;
2292 goto error;
80c802f3
TT		 2293 } else if (num_xfrms > 0) {
2294 /* Flow transformed */
80c802f3
TT		 2295 dst_release(dst_orig);
2296 } else {
2297 /* Flow passes untransformed */
2298 dst_release(dst);
452edd59 2299 dst = dst_orig;
1da177e4 2300 }
80c802f3
TT		 2301ok:
2302 xfrm_pols_put(pols, drop_pols);
0c183379
G		 2303 if (dst && dst->xfrm &&
2304 dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
2305 dst->flags |= DST_XFRM_TUNNEL;
452edd59 2306 return dst;
1da177e4 2307
80c802f3 2308nopol:
452edd59
DM		 2309 if (!(flags & XFRM_LOOKUP_ICMP)) {
2310 dst = dst_orig;
80c802f3 2311 goto ok;
452edd59 2312 }
80c802f3 2313 err = -ENOENT;
1da177e4 2314error:
80c802f3 2315 dst_release(dst);
75b8c133 2316dropdst:
ac37e251
W		 2317 if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
2318 dst_release(dst_orig);
80c802f3 2319 xfrm_pols_put(pols, drop_pols);
452edd59 2320 return ERR_PTR(err);
1da177e4
LT		 2321}
2322EXPORT_SYMBOL(xfrm_lookup);
2323
f92ee619
SK		 2324/* Callers of xfrm_lookup_route() must ensure a call to dst_output().
2325 * Otherwise we may send out blackholed packets.
2326 */
2327struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
2328 const struct flowi *fl,
2329 struct sock *sk, int flags)
2330{
b8c203b2 2331 struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
ac37e251
W		 2332 flags | XFRM_LOOKUP_QUEUE |
2333 XFRM_LOOKUP_KEEP_DST_REF);
f92ee619
SK		 2334
2335 if (IS_ERR(dst) && PTR_ERR(dst) == -EREMOTE)
2336 return make_blackhole(net, dst_orig->ops->family, dst_orig);
2337
2338 return dst;
2339}
2340EXPORT_SYMBOL(xfrm_lookup_route);
2341
df0ba92a 2342static inline int
8f029de2 2343xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
df0ba92a
MN		 2344{
2345 struct xfrm_state *x;
df0ba92a
MN		 2346
2347 if (!skb->sp || idx < 0 || idx >= skb->sp->len)
2348 return 0;
2349 x = skb->sp->xvec[idx];
2350 if (!x->type->reject)
2351 return 0;
1ecafede 2352 return x->type->reject(x, skb, fl);
df0ba92a
MN		 2353}
2354
1da177e4
LT		 2355/* When skb is transformed back to its "native" form, we have to
2356 * check policy restrictions. At the moment we make this in maximally
2357 * stupid way. Shame on me. :-) Of course, connected sockets must
2358 * have policy cached at them.
2359 */
2360
2361static inline int
7db454b9 2362xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
1da177e4
LT		 2363 unsigned short family)
2364{
2365 if (xfrm_state_kern(x))
928ba416 2366 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
1da177e4
LT		 2367 return x->id.proto == tmpl->id.proto &&
2368 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
2369 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
2370 x->props.mode == tmpl->mode &&
c5d18e98 2371 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
f3bd4840 2372 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
7e49e6de
MN		 2373 !(x->props.mode != XFRM_MODE_TRANSPORT &&
2374 xfrm_state_addr_cmp(tmpl, x, family));
1da177e4
LT		 2375}
2376
df0ba92a
MN		 2377/*
2378 * 0 or more than 0 is returned when validation is succeeded (either bypass
2379 * because of optional transport mode, or next index of the mathced secpath
2380 * state with the template.
2381 * -1 is returned when no matching template is found.
2382 * Otherwise "-2 - errored_index" is returned.
2383 */
1da177e4 2384static inline int
22cccb7e 2385xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
1da177e4
LT		 2386 unsigned short family)
2387{
2388 int idx = start;
2389
2390 if (tmpl->optional) {
7e49e6de 2391 if (tmpl->mode == XFRM_MODE_TRANSPORT)
1da177e4
LT		 2392 return start;
2393 } else
2394 start = -1;
2395 for (; idx < sp->len; idx++) {
dbe5b4aa 2396 if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
1da177e4 2397 return ++idx;
df0ba92a
MN		 2398 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2399 if (start == -1)
2400 start = -2-idx;
1da177e4 2401 break;
df0ba92a 2402 }
1da177e4
LT		 2403 }
2404 return start;
2405}
2406
d5422efe
HX		 2407int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
2408 unsigned int family, int reverse)
1da177e4
LT		 2409{
2410 struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
e0d1caa7 2411 int err;
1da177e4
LT		 2412
2413 if (unlikely(afinfo == NULL))
2414 return -EAFNOSUPPORT;
2415
d5422efe 2416 afinfo->decode_session(skb, fl, reverse);
1d28f42c 2417 err = security_xfrm_decode_session(skb, &fl->flowi_secid);
1da177e4 2418 xfrm_policy_put_afinfo(afinfo);
e0d1caa7 2419 return err;
1da177e4 2420}
d5422efe 2421EXPORT_SYMBOL(__xfrm_decode_session);
1da177e4 2422
9a7386ec 2423static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
1da177e4
LT		 2424{
2425 for (; k < sp->len; k++) {
df0ba92a 2426 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
d1d9facf 2427 *idxp = k;
1da177e4 2428 return 1;
df0ba92a 2429 }
1da177e4
LT		 2430 }
2431
2432 return 0;
2433}
2434
a716c119 2435int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
1da177e4
LT		 2436 unsigned short family)
2437{
f6e1e25d 2438 struct net *net = dev_net(skb->dev);
1da177e4 2439 struct xfrm_policy *pol;
4e81bb83
MN		 2440 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2441 int npols = 0;
2442 int xfrm_nr;
2443 int pi;
d5422efe 2444 int reverse;
1da177e4 2445 struct flowi fl;
d5422efe 2446 u8 fl_dir;
df0ba92a 2447 int xerr_idx = -1;
1da177e4 2448
d5422efe
HX		 2449 reverse = dir & ~XFRM_POLICY_MASK;
2450 dir &= XFRM_POLICY_MASK;
2451 fl_dir = policy_to_flow_dir(dir);
2452
0aa64774 2453 if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
59c9940e 2454 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
1da177e4 2455 return 0;
0aa64774
MN		 2456 }
2457
eb9c7ebe 2458 nf_nat_decode_session(skb, &fl, family);
1da177e4
LT		 2459
2460 /* First, check used SA against their selectors. */
2461 if (skb->sp) {
2462 int i;
2463
9b7a787d 2464 for (i = skb->sp->len-1; i >= 0; i--) {
dbe5b4aa 2465 struct xfrm_state *x = skb->sp->xvec[i];
0aa64774 2466 if (!xfrm_selector_match(&x->sel, &fl, family)) {
59c9940e 2467 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
1da177e4 2468 return 0;
0aa64774 2469 }
1da177e4
LT		 2470 }
2471 }
2472
2473 pol = NULL;
3bccfbc7 2474 if (sk && sk->sk_policy[dir]) {
e0d1caa7 2475 pol = xfrm_sk_policy_lookup(sk, dir, &fl);
0aa64774 2476 if (IS_ERR(pol)) {
59c9940e 2477 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3bccfbc7 2478 return 0;
0aa64774 2479 }
3bccfbc7 2480 }
1da177e4 2481
fe1a5f03
TT		 2482 if (!pol) {
2483 struct flow_cache_object *flo;
2484
2485 flo = flow_cache_lookup(net, &fl, family, fl_dir,
2486 xfrm_policy_lookup, NULL);
2487 if (IS_ERR_OR_NULL(flo))
2488 pol = ERR_CAST(flo);
2489 else
2490 pol = container_of(flo, struct xfrm_policy, flo);
2491 }
1da177e4 2492
0aa64774 2493 if (IS_ERR(pol)) {
59c9940e 2494 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
134b0fc5 2495 return 0;
0aa64774 2496 }
134b0fc5 2497
df0ba92a 2498 if (!pol) {
d1d9facf 2499 if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
df0ba92a 2500 xfrm_secpath_reject(xerr_idx, skb, &fl);
59c9940e 2501 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
df0ba92a
MN		 2502 return 0;
2503 }
2504 return 1;
2505 }
1da177e4 2506
9d729f72 2507 pol->curlft.use_time = get_seconds();
1da177e4 2508
4e81bb83 2509 pols[0] = pol;
02d0892f 2510 npols++;
4e81bb83
MN		 2511#ifdef CONFIG_XFRM_SUB_POLICY
2512 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
f6e1e25d 2513 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
4e81bb83
MN		 2514 &fl, family,
2515 XFRM_POLICY_IN);
2516 if (pols[1]) {
0aa64774 2517 if (IS_ERR(pols[1])) {
59c9940e 2518 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
134b0fc5 2519 return 0;
0aa64774 2520 }
9d729f72 2521 pols[1]->curlft.use_time = get_seconds();
02d0892f 2522 npols++;
4e81bb83
MN		 2523 }
2524 }
2525#endif
2526
1da177e4
LT		 2527 if (pol->action == XFRM_POLICY_ALLOW) {
2528 struct sec_path *sp;
2529 static struct sec_path dummy;
4e81bb83 2530 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
41a49cc3 2531 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
4e81bb83
MN		 2532 struct xfrm_tmpl **tpp = tp;
2533 int ti = 0;
1da177e4
LT		 2534 int i, k;
2535
2536 if ((sp = skb->sp) == NULL)
2537 sp = &dummy;
2538
4e81bb83
MN		 2539 for (pi = 0; pi < npols; pi++) {
2540 if (pols[pi] != pol &&
0aa64774 2541 pols[pi]->action != XFRM_POLICY_ALLOW) {
59c9940e 2542 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
4e81bb83 2543 goto reject;
0aa64774
MN		 2544 }
2545 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
59c9940e 2546 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
4e81bb83 2547 goto reject_error;
0aa64774 2548 }
4e81bb83
MN		 2549 for (i = 0; i < pols[pi]->xfrm_nr; i++)
2550 tpp[ti++] = &pols[pi]->xfrm_vec[i];
2551 }
2552 xfrm_nr = ti;
41a49cc3 2553 if (npols > 1) {
283bc9f3 2554 xfrm_tmpl_sort(stp, tpp, xfrm_nr, family, net);
41a49cc3
MN		 2555 tpp = stp;
2556 }
4e81bb83 2557
1da177e4
LT		 2558 /* For each tunnel xfrm, find the first matching tmpl.
2559 * For each tmpl before that, find corresponding xfrm.
2560 * Order is _important_. Later we will implement
2561 * some barriers, but at the moment barriers
2562 * are implied between each two transformations.
2563 */
4e81bb83
MN		 2564 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2565 k = xfrm_policy_ok(tpp[i], sp, k, family);
df0ba92a 2566 if (k < 0) {
d1d9facf
JM		 2567 if (k < -1)
2568 /* "-2 - errored_index" returned */
2569 xerr_idx = -(2+k);
59c9940e 2570 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
1da177e4 2571 goto reject;
df0ba92a 2572 }
1da177e4
LT		 2573 }
2574
0aa64774 2575 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
59c9940e 2576 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
1da177e4 2577 goto reject;
0aa64774 2578 }
1da177e4 2579
4e81bb83 2580 xfrm_pols_put(pols, npols);
1da177e4
LT		 2581 return 1;
2582 }
59c9940e 2583 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
1da177e4
LT		 2584
2585reject:
df0ba92a 2586 xfrm_secpath_reject(xerr_idx, skb, &fl);
4e81bb83
MN		 2587reject_error:
2588 xfrm_pols_put(pols, npols);
1da177e4
LT		 2589 return 0;
2590}
2591EXPORT_SYMBOL(__xfrm_policy_check);
2592
2593int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2594{
99a66657 2595 struct net *net = dev_net(skb->dev);
1da177e4 2596 struct flowi fl;
adf30907 2597 struct dst_entry *dst;
73137147 2598 int res = 1;
1da177e4 2599
0aa64774 2600 if (xfrm_decode_session(skb, &fl, family) < 0) {
72032fdb 2601 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
1da177e4 2602 return 0;
0aa64774 2603 }
1da177e4 2604
fafeeb6c 2605 skb_dst_force(skb);
adf30907 2606
b8c203b2 2607 dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
452edd59 2608 if (IS_ERR(dst)) {
73137147 2609 res = 0;
452edd59
DM		 2610 dst = NULL;
2611 }
adf30907
ED		 2612 skb_dst_set(skb, dst);
2613 return res;
1da177e4
LT		 2614}
2615EXPORT_SYMBOL(__xfrm_route_forward);
2616
d49c73c7
DM		 2617/* Optimize later using cookies and generation ids. */
2618
1da177e4
LT		 2619static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2620{
d49c73c7 2621 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
f5b0a874
DM		 2622 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2623 * get validated by dst_ops->check on every use. We do this
2624 * because when a normal route referenced by an XFRM dst is
2625 * obsoleted we do not go looking around for all parent
2626 * referencing XFRM dsts so that we can invalidate them. It
2627 * is just too much work. Instead we make the checks here on
2628 * every use. For example:
d49c73c7
DM		 2629 *
2630 * XFRM dst A --> IPv4 dst X
2631 *
2632 * X is the "xdst->route" of A (X is also the "dst->path" of A
2633 * in this example). If X is marked obsolete, "A" will not
2634 * notice. That's what we are validating here via the
2635 * stale_bundle() check.
2636 *
2637 * When a policy's bundle is pruned, we dst_free() the XFRM
f5b0a874
DM		 2638 * dst which causes it's ->obsolete field to be set to
2639 * DST_OBSOLETE_DEAD. If an XFRM dst has been pruned like
2640 * this, we want to force a new route lookup.
399c180a 2641 */
d49c73c7
DM		 2642 if (dst->obsolete < 0 && !stale_bundle(dst))
2643 return dst;
2644
1da177e4
LT		 2645 return NULL;
2646}
2647
2648static int stale_bundle(struct dst_entry *dst)
2649{
12fdb4d3 2650 return !xfrm_bundle_ok((struct xfrm_dst *)dst);
1da177e4
LT		 2651}
2652
aabc9761 2653void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
1da177e4 2654{
1da177e4 2655 while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
c346dca1 2656 dst->dev = dev_net(dev)->loopback_dev;
de3cb747 2657 dev_hold(dst->dev);
1da177e4
LT		 2658 dev_put(dev);
2659 }
2660}
aabc9761 2661EXPORT_SYMBOL(xfrm_dst_ifdown);
1da177e4
LT		 2662
2663static void xfrm_link_failure(struct sk_buff *skb)
2664{
2665 /* Impossible. Such dst must be popped before reaches point of failure. */
1da177e4
LT		 2666}
2667
2668static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2669{
2670 if (dst) {
2671 if (dst->obsolete) {
2672 dst_release(dst);
2673 dst = NULL;
2674 }
2675 }
2676 return dst;
2677}
2678
e4c17216 2679void xfrm_garbage_collect(struct net *net)
c0ed1c14 2680{
ca925cf1 2681 flow_cache_flush(net);
c0ed1c14 2682}
e4c17216 2683EXPORT_SYMBOL(xfrm_garbage_collect);
c0ed1c14
SK		 2684
2685static void xfrm_garbage_collect_deferred(struct net *net)
2686{
ca925cf1 2687 flow_cache_flush_deferred(net);
c0ed1c14
SK		 2688}
2689
25ee3286 2690static void xfrm_init_pmtu(struct dst_entry *dst)
1da177e4
LT		 2691{
2692 do {
2693 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2694 u32 pmtu, route_mtu_cached;
2695
2696 pmtu = dst_mtu(dst->child);
2697 xdst->child_mtu_cached = pmtu;
2698
2699 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2700
2701 route_mtu_cached = dst_mtu(xdst->route);
2702 xdst->route_mtu_cached = route_mtu_cached;
2703
2704 if (pmtu > route_mtu_cached)
2705 pmtu = route_mtu_cached;
2706
defb3519 2707 dst_metric_set(dst, RTAX_MTU, pmtu);
1da177e4
LT		 2708 } while ((dst = dst->next));
2709}
2710
1da177e4
LT		 2711/* Check that the bundle accepts the flow and its components are
2712 * still valid.
2713 */
2714
12fdb4d3 2715static int xfrm_bundle_ok(struct xfrm_dst *first)
1da177e4
LT		 2716{
2717 struct dst_entry *dst = &first->u.dst;
2718 struct xfrm_dst *last;
2719 u32 mtu;
2720
92d63dec 2721 if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
1da177e4
LT		 2722 (dst->dev && !netif_running(dst->dev)))
2723 return 0;
2724
a0073fe1
SK		 2725 if (dst->flags & DST_XFRM_QUEUE)
2726 return 1;
2727
1da177e4
LT		 2728 last = NULL;
2729
2730 do {
2731 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2732
1da177e4
LT		 2733 if (dst->xfrm->km.state != XFRM_STATE_VALID)
2734 return 0;
80c802f3
TT		 2735 if (xdst->xfrm_genid != dst->xfrm->genid)
2736 return 0;
b1312c89
TT		 2737 if (xdst->num_pols > 0 &&
2738 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
9d4a706d 2739 return 0;
e53820de 2740
1da177e4
LT		 2741 mtu = dst_mtu(dst->child);
2742 if (xdst->child_mtu_cached != mtu) {
2743 last = xdst;
2744 xdst->child_mtu_cached = mtu;
2745 }
2746
92d63dec 2747 if (!dst_check(xdst->route, xdst->route_cookie))
1da177e4
LT		 2748 return 0;
2749 mtu = dst_mtu(xdst->route);
2750 if (xdst->route_mtu_cached != mtu) {
2751 last = xdst;
2752 xdst->route_mtu_cached = mtu;
2753 }
2754
2755 dst = dst->child;
2756 } while (dst->xfrm);
2757
2758 if (likely(!last))
2759 return 1;
2760
2761 mtu = last->child_mtu_cached;
2762 for (;;) {
2763 dst = &last->u.dst;
2764
2765 mtu = xfrm_state_mtu(dst->xfrm, mtu);
2766 if (mtu > last->route_mtu_cached)
2767 mtu = last->route_mtu_cached;
defb3519 2768 dst_metric_set(dst, RTAX_MTU, mtu);
1da177e4
LT		 2769
2770 if (last == first)
2771 break;
2772
bd0bf076 2773 last = (struct xfrm_dst *)last->u.dst.next;
1da177e4
LT		 2774 last->child_mtu_cached = mtu;
2775 }
2776
2777 return 1;
2778}
2779
0dbaee3b
DM		 2780static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2781{
2782 return dst_metric_advmss(dst->path);
2783}
2784
ebb762f2 2785static unsigned int xfrm_mtu(const struct dst_entry *dst)
d33e4553 2786{
618f9bc7
SK		 2787 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2788
2789 return mtu ? : dst_mtu(dst->path);
d33e4553
DM		 2790}
2791
f894cbf8
DM		 2792static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
2793 struct sk_buff *skb,
2794 const void *daddr)
d3aaeb38 2795{
f894cbf8 2796 return dst->path->ops->neigh_lookup(dst, skb, daddr);
d3aaeb38
DM		 2797}
2798
1da177e4
LT		 2799int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2800{
d7c7544c 2801 struct net *net;
1da177e4
LT		 2802 int err = 0;
2803 if (unlikely(afinfo == NULL))
2804 return -EINVAL;
2805 if (unlikely(afinfo->family >= NPROTO))
2806 return -EAFNOSUPPORT;
ef8531b6 2807 spin_lock(&xfrm_policy_afinfo_lock);
1da177e4
LT		 2808 if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2809 err = -ENOBUFS;
2810 else {
2811 struct dst_ops *dst_ops = afinfo->dst_ops;
2812 if (likely(dst_ops->kmem_cachep == NULL))
2813 dst_ops->kmem_cachep = xfrm_dst_cache;
2814 if (likely(dst_ops->check == NULL))
2815 dst_ops->check = xfrm_dst_check;
0dbaee3b
DM		 2816 if (likely(dst_ops->default_advmss == NULL))
2817 dst_ops->default_advmss = xfrm_default_advmss;
ebb762f2
SK		 2818 if (likely(dst_ops->mtu == NULL))
2819 dst_ops->mtu = xfrm_mtu;
1da177e4
LT		 2820 if (likely(dst_ops->negative_advice == NULL))
2821 dst_ops->negative_advice = xfrm_negative_advice;
2822 if (likely(dst_ops->link_failure == NULL))
2823 dst_ops->link_failure = xfrm_link_failure;
d3aaeb38
DM		 2824 if (likely(dst_ops->neigh_lookup == NULL))
2825 dst_ops->neigh_lookup = xfrm_neigh_lookup;
1da177e4 2826 if (likely(afinfo->garbage_collect == NULL))
c0ed1c14 2827 afinfo->garbage_collect = xfrm_garbage_collect_deferred;
418a99ac 2828 rcu_assign_pointer(xfrm_policy_afinfo[afinfo->family], afinfo);
1da177e4 2829 }
ef8531b6 2830 spin_unlock(&xfrm_policy_afinfo_lock);
d7c7544c
AD		 2831
2832 rtnl_lock();
2833 for_each_net(net) {
2834 struct dst_ops *xfrm_dst_ops;
2835
2836 switch (afinfo->family) {
2837 case AF_INET:
2838 xfrm_dst_ops = &net->xfrm.xfrm4_dst_ops;
2839 break;
dfd56b8b 2840#if IS_ENABLED(CONFIG_IPV6)
d7c7544c
AD		 2841 case AF_INET6:
2842 xfrm_dst_ops = &net->xfrm.xfrm6_dst_ops;
2843 break;
2844#endif
2845 default:
2846 BUG();
2847 }
2848 *xfrm_dst_ops = *afinfo->dst_ops;
2849 }
2850 rtnl_unlock();
2851
1da177e4
LT		 2852 return err;
2853}
2854EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2855
2856int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2857{
2858 int err = 0;
2859 if (unlikely(afinfo == NULL))
2860 return -EINVAL;
2861 if (unlikely(afinfo->family >= NPROTO))
2862 return -EAFNOSUPPORT;
ef8531b6 2863 spin_lock(&xfrm_policy_afinfo_lock);
1da177e4
LT		 2864 if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2865 if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2866 err = -EINVAL;
ef8531b6
ED		 2867 else
2868 RCU_INIT_POINTER(xfrm_policy_afinfo[afinfo->family],
2869 NULL);
2870 }
2871 spin_unlock(&xfrm_policy_afinfo_lock);
2872 if (!err) {
2873 struct dst_ops *dst_ops = afinfo->dst_ops;
2874
2875 synchronize_rcu();
2876
2877 dst_ops->kmem_cachep = NULL;
2878 dst_ops->check = NULL;
2879 dst_ops->negative_advice = NULL;
2880 dst_ops->link_failure = NULL;
2881 afinfo->garbage_collect = NULL;
1da177e4 2882 }
1da177e4
LT		 2883 return err;
2884}
2885EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2886
d7c7544c
AD		 2887static void __net_init xfrm_dst_ops_init(struct net *net)
2888{
2889 struct xfrm_policy_afinfo *afinfo;
2890
ef8531b6
ED		 2891 rcu_read_lock();
2892 afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET]);
d7c7544c
AD		 2893 if (afinfo)
2894 net->xfrm.xfrm4_dst_ops = *afinfo->dst_ops;
dfd56b8b 2895#if IS_ENABLED(CONFIG_IPV6)
ef8531b6 2896 afinfo = rcu_dereference(xfrm_policy_afinfo[AF_INET6]);
d7c7544c
AD		 2897 if (afinfo)
2898 net->xfrm.xfrm6_dst_ops = *afinfo->dst_ops;
2899#endif
418a99ac 2900 rcu_read_unlock();
546be240
HX		 2901}
2902
1da177e4
LT		 2903static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
2904{
351638e7 2905 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
e9dc8653 2906
1da177e4
LT		 2907 switch (event) {
2908 case NETDEV_DOWN:
c0ed1c14 2909 xfrm_garbage_collect(dev_net(dev));
1da177e4
LT		 2910 }
2911 return NOTIFY_DONE;
2912}
2913
2914static struct notifier_block xfrm_dev_notifier = {
d5917a35 2915 .notifier_call = xfrm_dev_event,
1da177e4
LT		 2916};
2917
558f82ef 2918#ifdef CONFIG_XFRM_STATISTICS
59c9940e 2919static int __net_init xfrm_statistics_init(struct net *net)
558f82ef 2920{
c68cd1a0 2921 int rv;
698365fa
WC		 2922 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
2923 if (!net->mib.xfrm_statistics)
558f82ef 2924 return -ENOMEM;
c68cd1a0
AD		 2925 rv = xfrm_proc_init(net);
2926 if (rv < 0)
698365fa 2927 free_percpu(net->mib.xfrm_statistics);
c68cd1a0 2928 return rv;
558f82ef 2929}
59c9940e
AD		 2930
2931static void xfrm_statistics_fini(struct net *net)
2932{
c68cd1a0 2933 xfrm_proc_fini(net);
698365fa 2934 free_percpu(net->mib.xfrm_statistics);
59c9940e
AD		 2935}
2936#else
2937static int __net_init xfrm_statistics_init(struct net *net)
2938{
2939 return 0;
2940}
2941
2942static void xfrm_statistics_fini(struct net *net)
2943{
2944}
558f82ef
MN		 2945#endif
2946
d62ddc21 2947static int __net_init xfrm_policy_init(struct net *net)
1da177e4 2948{
2518c7c2
DM		 2949 unsigned int hmask, sz;
2950 int dir;
2951
d62ddc21
AD		 2952 if (net_eq(net, &init_net))
2953 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
1da177e4 2954 sizeof(struct xfrm_dst),
e5d679f3 2955 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
20c2df83 2956 NULL);
1da177e4 2957
2518c7c2
DM		 2958 hmask = 8 - 1;
2959 sz = (hmask+1) * sizeof(struct hlist_head);
2960
93b851c1
AD		 2961 net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2962 if (!net->xfrm.policy_byidx)
2963 goto out_byidx;
8100bea7 2964 net->xfrm.policy_idx_hmask = hmask;
2518c7c2 2965
53c2e285 2966 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
2518c7c2
DM		 2967 struct xfrm_policy_hash *htab;
2968
dc2caba7 2969 net->xfrm.policy_count[dir] = 0;
53c2e285 2970 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
8b18f8ea 2971 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2518c7c2 2972
a35f6c5d 2973 htab = &net->xfrm.policy_bydst[dir];
44e36b42 2974 htab->table = xfrm_hash_alloc(sz);
2518c7c2 2975 if (!htab->table)
a35f6c5d
AD		 2976 goto out_bydst;
2977 htab->hmask = hmask;
b58555f1
CG		 2978 htab->dbits4 = 32;
2979 htab->sbits4 = 32;
2980 htab->dbits6 = 128;
2981 htab->sbits6 = 128;
2518c7c2 2982 }
880a6fab
CG		 2983 net->xfrm.policy_hthresh.lbits4 = 32;
2984 net->xfrm.policy_hthresh.rbits4 = 32;
2985 net->xfrm.policy_hthresh.lbits6 = 128;
2986 net->xfrm.policy_hthresh.rbits6 = 128;
2987
2988 seqlock_init(&net->xfrm.policy_hthresh.lock);
2518c7c2 2989
adfcf0b2 2990 INIT_LIST_HEAD(&net->xfrm.policy_all);
66caf628 2991 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
880a6fab 2992 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
d62ddc21
AD		 2993 if (net_eq(net, &init_net))
2994 register_netdevice_notifier(&xfrm_dev_notifier);
2995 return 0;
93b851c1 2996
a35f6c5d
AD		 2997out_bydst:
2998 for (dir--; dir >= 0; dir--) {
2999 struct xfrm_policy_hash *htab;
3000
3001 htab = &net->xfrm.policy_bydst[dir];
3002 xfrm_hash_free(htab->table, sz);
3003 }
3004 xfrm_hash_free(net->xfrm.policy_byidx, sz);
93b851c1
AD		 3005out_byidx:
3006 return -ENOMEM;
d62ddc21
AD		 3007}
3008
3009static void xfrm_policy_fini(struct net *net)
3010{
93b851c1 3011 unsigned int sz;
8b18f8ea 3012 int dir;
93b851c1 3013
7c2776ee
AD		 3014 flush_work(&net->xfrm.policy_hash_work);
3015#ifdef CONFIG_XFRM_SUB_POLICY
2e71029e 3016 xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
7c2776ee 3017#endif
2e71029e 3018 xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
7c2776ee 3019
adfcf0b2 3020 WARN_ON(!list_empty(&net->xfrm.policy_all));
93b851c1 3021
53c2e285 3022 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
a35f6c5d
AD		 3023 struct xfrm_policy_hash *htab;
3024
8b18f8ea 3025 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
a35f6c5d
AD		 3026
3027 htab = &net->xfrm.policy_bydst[dir];
5b653b2a 3028 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
a35f6c5d
AD		 3029 WARN_ON(!hlist_empty(htab->table));
3030 xfrm_hash_free(htab->table, sz);
8b18f8ea
AD		 3031 }
3032
8100bea7 3033 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
93b851c1
AD		 3034 WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
3035 xfrm_hash_free(net->xfrm.policy_byidx, sz);
1da177e4
LT		 3036}
3037
d62ddc21
AD		 3038static int __net_init xfrm_net_init(struct net *net)
3039{
3040 int rv;
3041
59c9940e
AD		 3042 rv = xfrm_statistics_init(net);
3043 if (rv < 0)
3044 goto out_statistics;
d62ddc21
AD		 3045 rv = xfrm_state_init(net);
3046 if (rv < 0)
3047 goto out_state;
3048 rv = xfrm_policy_init(net);
3049 if (rv < 0)
3050 goto out_policy;
d7c7544c 3051 xfrm_dst_ops_init(net);
b27aeadb
AD		 3052 rv = xfrm_sysctl_init(net);
3053 if (rv < 0)
3054 goto out_sysctl;
4a93f509
SK		 3055 rv = flow_cache_init(net);
3056 if (rv < 0)
3057 goto out;
283bc9f3
FD		 3058
3059 /* Initialize the per-net locks here */
3060 spin_lock_init(&net->xfrm.xfrm_state_lock);
3061 rwlock_init(&net->xfrm.xfrm_policy_lock);
283bc9f3
FD		 3062 mutex_init(&net->xfrm.xfrm_cfg_mutex);
3063
d62ddc21
AD		 3064 return 0;
3065
4a93f509
SK		 3066out:
3067 xfrm_sysctl_fini(net);
b27aeadb
AD		 3068out_sysctl:
3069 xfrm_policy_fini(net);
d62ddc21
AD		 3070out_policy:
3071 xfrm_state_fini(net);
3072out_state:
59c9940e
AD		 3073 xfrm_statistics_fini(net);
3074out_statistics:
d62ddc21
AD		 3075 return rv;
3076}
3077
3078static void __net_exit xfrm_net_exit(struct net *net)
3079{
4a93f509 3080 flow_cache_fini(net);
b27aeadb 3081 xfrm_sysctl_fini(net);
d62ddc21
AD		 3082 xfrm_policy_fini(net);
3083 xfrm_state_fini(net);
59c9940e 3084 xfrm_statistics_fini(net);
d62ddc21
AD		 3085}
3086
3087static struct pernet_operations __net_initdata xfrm_net_ops = {
3088 .init = xfrm_net_init,
3089 .exit = xfrm_net_exit,
3090};
3091
1da177e4
LT		 3092void __init xfrm_init(void)
3093{
d62ddc21 3094 register_pernet_subsys(&xfrm_net_ops);
1da177e4
LT		 3095 xfrm_input_init();
3096}
3097
ab5f5e8b 3098#ifdef CONFIG_AUDITSYSCALL
1486cbd7
IJ		 3099static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
3100 struct audit_buffer *audit_buf)
ab5f5e8b 3101{
875179fa
PM		 3102 struct xfrm_sec_ctx *ctx = xp->security;
3103 struct xfrm_selector *sel = &xp->selector;
3104
3105 if (ctx)
ab5f5e8b 3106 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
875179fa 3107 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
ab5f5e8b 3108
9b7a787d 3109 switch (sel->family) {
ab5f5e8b 3110 case AF_INET:
21454aaa 3111 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
875179fa
PM		 3112 if (sel->prefixlen_s != 32)
3113 audit_log_format(audit_buf, " src_prefixlen=%d",
3114 sel->prefixlen_s);
21454aaa 3115 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
875179fa
PM		 3116 if (sel->prefixlen_d != 32)
3117 audit_log_format(audit_buf, " dst_prefixlen=%d",
3118 sel->prefixlen_d);
ab5f5e8b
JL		 3119 break;
3120 case AF_INET6:
5b095d98 3121 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
875179fa
PM		 3122 if (sel->prefixlen_s != 128)
3123 audit_log_format(audit_buf, " src_prefixlen=%d",
3124 sel->prefixlen_s);
5b095d98 3125 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
875179fa
PM		 3126 if (sel->prefixlen_d != 128)
3127 audit_log_format(audit_buf, " dst_prefixlen=%d",
3128 sel->prefixlen_d);
ab5f5e8b
JL		 3129 break;
3130 }
3131}
3132
2e71029e 3133void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
ab5f5e8b
JL		 3134{
3135 struct audit_buffer *audit_buf;
ab5f5e8b 3136
afeb14b4 3137 audit_buf = xfrm_audit_start("SPD-add");
ab5f5e8b
JL		 3138 if (audit_buf == NULL)
3139 return;
2e71029e 3140 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
afeb14b4 3141 audit_log_format(audit_buf, " res=%u", result);
ab5f5e8b
JL		 3142 xfrm_audit_common_policyinfo(xp, audit_buf);
3143 audit_log_end(audit_buf);
3144}
3145EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
3146
68277acc 3147void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
2e71029e 3148 bool task_valid)
ab5f5e8b
JL		 3149{
3150 struct audit_buffer *audit_buf;
ab5f5e8b 3151
afeb14b4 3152 audit_buf = xfrm_audit_start("SPD-delete");
ab5f5e8b
JL		 3153 if (audit_buf == NULL)
3154 return;
2e71029e 3155 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
afeb14b4 3156 audit_log_format(audit_buf, " res=%u", result);
ab5f5e8b
JL		 3157 xfrm_audit_common_policyinfo(xp, audit_buf);
3158 audit_log_end(audit_buf);
3159}
3160EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
3161#endif
3162
80c9abaa 3163#ifdef CONFIG_XFRM_MIGRATE
bc9b35ad
DM		 3164static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
3165 const struct xfrm_selector *sel_tgt)
80c9abaa
SS		 3166{
3167 if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
3168 if (sel_tgt->family == sel_cmp->family &&
70e94e66
YH		 3169 xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
3170 sel_cmp->family) &&
3171 xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
3172 sel_cmp->family) &&
80c9abaa
SS		 3173 sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
3174 sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
bc9b35ad 3175 return true;
80c9abaa
SS		 3176 }
3177 } else {
3178 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
bc9b35ad 3179 return true;
80c9abaa
SS		 3180 }
3181 }
bc9b35ad 3182 return false;
80c9abaa
SS		 3183}
3184
3e94c2dc
WC		 3185static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
3186 u8 dir, u8 type, struct net *net)
80c9abaa
SS		 3187{
3188 struct xfrm_policy *pol, *ret = NULL;
80c9abaa
SS		 3189 struct hlist_head *chain;
3190 u32 priority = ~0U;
3191
283bc9f3 3192 read_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME*/
8d549c4f 3193 chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
b67bfe0d 3194 hlist_for_each_entry(pol, chain, bydst) {
80c9abaa
SS		 3195 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3196 pol->type == type) {
3197 ret = pol;
3198 priority = ret->priority;
3199 break;
3200 }
3201 }
8d549c4f 3202 chain = &net->xfrm.policy_inexact[dir];
b67bfe0d 3203 hlist_for_each_entry(pol, chain, bydst) {
80c9abaa
SS		 3204 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3205 pol->type == type &&
3206 pol->priority < priority) {
3207 ret = pol;
3208 break;
3209 }
3210 }
3211
3212 if (ret)
3213 xfrm_pol_hold(ret);
3214
283bc9f3 3215 read_unlock_bh(&net->xfrm.xfrm_policy_lock);
80c9abaa
SS		 3216
3217 return ret;
3218}
3219
dd701754 3220static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
80c9abaa
SS		 3221{
3222 int match = 0;
3223
3224 if (t->mode == m->mode && t->id.proto == m->proto &&
3225 (m->reqid == 0 || t->reqid == m->reqid)) {
3226 switch (t->mode) {
3227 case XFRM_MODE_TUNNEL:
3228 case XFRM_MODE_BEET:
70e94e66
YH		 3229 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
3230 m->old_family) &&
3231 xfrm_addr_equal(&t->saddr, &m->old_saddr,
3232 m->old_family)) {
80c9abaa
SS		 3233 match = 1;
3234 }
3235 break;
3236 case XFRM_MODE_TRANSPORT:
3237 /* in case of transport mode, template does not store
3238 any IP addresses, hence we just compare mode and
3239 protocol */
3240 match = 1;
3241 break;
3242 default:
3243 break;
3244 }
3245 }
3246 return match;
3247}
3248
3249/* update endpoint address(es) of template(s) */
3250static int xfrm_policy_migrate(struct xfrm_policy *pol,
3251 struct xfrm_migrate *m, int num_migrate)
3252{
3253 struct xfrm_migrate *mp;
80c9abaa
SS		 3254 int i, j, n = 0;
3255
3256 write_lock_bh(&pol->lock);
12a169e7 3257 if (unlikely(pol->walk.dead)) {
80c9abaa
SS		 3258 /* target policy has been deleted */
3259 write_unlock_bh(&pol->lock);
3260 return -ENOENT;
3261 }
3262
3263 for (i = 0; i < pol->xfrm_nr; i++) {
3264 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
3265 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
3266 continue;
3267 n++;
1bfcb10f
HX		 3268 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
3269 pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
80c9abaa
SS		 3270 continue;
3271 /* update endpoints */
3272 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
3273 sizeof(pol->xfrm_vec[i].id.daddr));
3274 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
3275 sizeof(pol->xfrm_vec[i].saddr));
3276 pol->xfrm_vec[i].encap_family = mp->new_family;
3277 /* flush bundles */
80c802f3 3278 atomic_inc(&pol->genid);
80c9abaa
SS		 3279 }
3280 }
3281
3282 write_unlock_bh(&pol->lock);
3283
3284 if (!n)
3285 return -ENODATA;
3286
3287 return 0;
3288}
3289
dd701754 3290static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
80c9abaa
SS		 3291{
3292 int i, j;
3293
3294 if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
3295 return -EINVAL;
3296
3297 for (i = 0; i < num_migrate; i++) {
70e94e66
YH		 3298 if (xfrm_addr_equal(&m[i].old_daddr, &m[i].new_daddr,
3299 m[i].old_family) &&
3300 xfrm_addr_equal(&m[i].old_saddr, &m[i].new_saddr,
3301 m[i].old_family))
80c9abaa
SS		 3302 return -EINVAL;
3303 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
3304 xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
3305 return -EINVAL;
3306
3307 /* check if there is any duplicated entry */
3308 for (j = i + 1; j < num_migrate; j++) {
3309 if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
3310 sizeof(m[i].old_daddr)) &&
3311 !memcmp(&m[i].old_saddr, &m[j].old_saddr,
3312 sizeof(m[i].old_saddr)) &&
3313 m[i].proto == m[j].proto &&
3314 m[i].mode == m[j].mode &&
3315 m[i].reqid == m[j].reqid &&
3316 m[i].old_family == m[j].old_family)
3317 return -EINVAL;
3318 }
3319 }
3320
3321 return 0;
3322}
3323
b4b7c0b3 3324int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
13c1d189 3325 struct xfrm_migrate *m, int num_migrate,
8d549c4f 3326 struct xfrm_kmaddress *k, struct net *net)
80c9abaa
SS		 3327{
3328 int i, err, nx_cur = 0, nx_new = 0;
3329 struct xfrm_policy *pol = NULL;
3330 struct xfrm_state *x, *xc;
3331 struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
3332 struct xfrm_state *x_new[XFRM_MAX_DEPTH];
3333 struct xfrm_migrate *mp;
3334
3335 if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
3336 goto out;
3337
3338 /* Stage 1 - find policy */
8d549c4f 3339 if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {
80c9abaa
SS		 3340 err = -ENOENT;
3341 goto out;
3342 }
3343
3344 /* Stage 2 - find and update state(s) */
3345 for (i = 0, mp = m; i < num_migrate; i++, mp++) {
283bc9f3 3346 if ((x = xfrm_migrate_state_find(mp, net))) {
80c9abaa
SS		 3347 x_cur[nx_cur] = x;
3348 nx_cur++;
3349 if ((xc = xfrm_state_migrate(x, mp))) {
3350 x_new[nx_new] = xc;
3351 nx_new++;
3352 } else {
3353 err = -ENODATA;
3354 goto restore_state;
3355 }
3356 }
3357 }
3358
3359 /* Stage 3 - update policy */
3360 if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
3361 goto restore_state;
3362
3363 /* Stage 4 - delete old state(s) */
3364 if (nx_cur) {
3365 xfrm_states_put(x_cur, nx_cur);
3366 xfrm_states_delete(x_cur, nx_cur);
3367 }
3368
3369 /* Stage 5 - announce */
13c1d189 3370 km_migrate(sel, dir, type, m, num_migrate, k);
80c9abaa
SS		 3371
3372 xfrm_pol_put(pol);
3373
3374 return 0;
3375out:
3376 return err;
3377
3378restore_state:
3379 if (pol)
3380 xfrm_pol_put(pol);
3381 if (nx_cur)
3382 xfrm_states_put(x_cur, nx_cur);
3383 if (nx_new)
3384 xfrm_states_delete(x_new, nx_new);
3385
3386 return err;
3387}
e610e679 3388EXPORT_SYMBOL(xfrm_migrate);
80c9abaa 3389#endif
Linux 6.x block layer and io_uring tree(s)