net/xfrm/xfrm_policy.c

   1 /*
   2  * xfrm_policy.c
   3  *
   4  * Changes:
   5  *      Mitsuru KANDA @USAGI
   6  *      Kazunori MIYAZAWA @USAGI
   7  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
   8  *              IPv6 support
   9  *      Kazunori MIYAZAWA @USAGI
  10  *      YOSHIFUJI Hideaki
  11  *              Split up af-specific portion
  12  *      Derek Atkins <derek@ihtfp.com>          Add the post_input processor
  13  *
  14  */
  15
  16 #include <linux/err.h>
  17 #include <linux/slab.h>
  18 #include <linux/kmod.h>
  19 #include <linux/list.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/notifier.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/netfilter.h>
  25 #include <linux/module.h>
  26 #include <linux/cache.h>
  27 #include <linux/cpu.h>
  28 #include <linux/audit.h>
  29 #include <linux/rhashtable.h>
  30 #include <net/dst.h>
  31 #include <net/flow.h>
  32 #include <net/xfrm.h>
  33 #include <net/ip.h>
  34 #ifdef CONFIG_XFRM_STATISTICS
  35 #include <net/snmp.h>
  36 #endif
  37
  38 #include "xfrm_hash.h"
  39
  40 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
  41 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
  42 #define XFRM_MAX_QUEUE_LEN      100
  43
  44 struct xfrm_flo {
  45         struct dst_entry *dst_orig;
  46         u8 flags;
  47 };
  48
  49 struct xfrm_pol_inexact_key {
  50         possible_net_t net;
  51         u16 family;
  52         u8 dir, type;
  53 };
  54
  55 struct xfrm_pol_inexact_bin {
  56         struct xfrm_pol_inexact_key k;
  57         struct rhash_head head;
  58         struct hlist_head hhead;
  59
  60         /* slow path below */
  61         struct list_head inexact_bins;
  62         struct rcu_head rcu;
  63 };
  64
  65 static DEFINE_SPINLOCK(xfrm_if_cb_lock);
  66 static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
  67
  68 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
  69 static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
  70                                                 __read_mostly;
  71
  72 static struct kmem_cache *xfrm_dst_cache __ro_after_init;
  73 static __read_mostly seqcount_t xfrm_policy_hash_generation;
  74
  75 static struct rhashtable xfrm_policy_inexact_table;
  76 static const struct rhashtable_params xfrm_pol_inexact_params;
  77
  78 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
  79 static int stale_bundle(struct dst_entry *dst);
  80 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
  81 static void xfrm_policy_queue_process(struct timer_list *t);
  82
  83 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
  84 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
  85                                                 int dir);
  86
  87 static struct xfrm_pol_inexact_bin *
  88 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir);
  89
  90 static struct xfrm_pol_inexact_bin *
  91 xfrm_policy_inexact_lookup_rcu(struct net *net,
  92                                u8 type, u16 family, u8 dir);
  93 static struct xfrm_policy *
  94 xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
  95                         bool excl);
  96 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
  97                                             struct xfrm_policy *policy);
  98
  99 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
 100 {
 101         return refcount_inc_not_zero(&policy->refcnt);
 102 }
 103
 104 static inline bool
 105 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
 106 {
 107         const struct flowi4 *fl4 = &fl->u.ip4;
 108
 109         return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
 110                 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
 111                 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
 112                 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
 113                 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
 114                 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
 115 }
 116
 117 static inline bool
 118 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
 119 {
 120         const struct flowi6 *fl6 = &fl->u.ip6;
 121
 122         return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
 123                 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
 124                 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
 125                 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
 126                 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
 127                 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
 128 }
 129
 130 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
 131                          unsigned short family)
 132 {
 133         switch (family) {
 134         case AF_INET:
 135                 return __xfrm4_selector_match(sel, fl);
 136         case AF_INET6:
 137                 return __xfrm6_selector_match(sel, fl);
 138         }
 139         return false;
 140 }
 141
 142 static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
 143 {
 144         const struct xfrm_policy_afinfo *afinfo;
 145
 146         if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
 147                 return NULL;
 148         rcu_read_lock();
 149         afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
 150         if (unlikely(!afinfo))
 151                 rcu_read_unlock();
 152         return afinfo;
 153 }
 154
 155 /* Called with rcu_read_lock(). */
 156 static const struct xfrm_if_cb *xfrm_if_get_cb(void)
 157 {
 158         return rcu_dereference(xfrm_if_cb);
 159 }
 160
 161 struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
 162                                     const xfrm_address_t *saddr,
 163                                     const xfrm_address_t *daddr,
 164                                     int family, u32 mark)
 165 {
 166         const struct xfrm_policy_afinfo *afinfo;
 167         struct dst_entry *dst;
 168
 169         afinfo = xfrm_policy_get_afinfo(family);
 170         if (unlikely(afinfo == NULL))
 171                 return ERR_PTR(-EAFNOSUPPORT);
 172
 173         dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark);
 174
 175         rcu_read_unlock();
 176
 177         return dst;
 178 }
 179 EXPORT_SYMBOL(__xfrm_dst_lookup);
 180
 181 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
 182                                                 int tos, int oif,
 183                                                 xfrm_address_t *prev_saddr,
 184                                                 xfrm_address_t *prev_daddr,
 185                                                 int family, u32 mark)
 186 {
 187         struct net *net = xs_net(x);
 188         xfrm_address_t *saddr = &x->props.saddr;
 189         xfrm_address_t *daddr = &x->id.daddr;
 190         struct dst_entry *dst;
 191
 192         if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
 193                 saddr = x->coaddr;
 194                 daddr = prev_daddr;
 195         }
 196         if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
 197                 saddr = prev_saddr;
 198                 daddr = x->coaddr;
 199         }
 200
 201         dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark);
 202
 203         if (!IS_ERR(dst)) {
 204                 if (prev_saddr != saddr)
 205                         memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
 206                 if (prev_daddr != daddr)
 207                         memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
 208         }
 209
 210         return dst;
 211 }
 212
 213 static inline unsigned long make_jiffies(long secs)
 214 {
 215         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
 216                 return MAX_SCHEDULE_TIMEOUT-1;
 217         else
 218                 return secs*HZ;
 219 }
 220
 221 static void xfrm_policy_timer(struct timer_list *t)
 222 {
 223         struct xfrm_policy *xp = from_timer(xp, t, timer);
 224         time64_t now = ktime_get_real_seconds();
 225         time64_t next = TIME64_MAX;
 226         int warn = 0;
 227         int dir;
 228
 229         read_lock(&xp->lock);
 230
 231         if (unlikely(xp->walk.dead))
 232                 goto out;
 233
 234         dir = xfrm_policy_id2dir(xp->index);
 235
 236         if (xp->lft.hard_add_expires_seconds) {
 237                 time64_t tmo = xp->lft.hard_add_expires_seconds +
 238                         xp->curlft.add_time - now;
 239                 if (tmo <= 0)
 240                         goto expired;
 241                 if (tmo < next)
 242                         next = tmo;
 243         }
 244         if (xp->lft.hard_use_expires_seconds) {
 245                 time64_t tmo = xp->lft.hard_use_expires_seconds +
 246                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 247                 if (tmo <= 0)
 248                         goto expired;
 249                 if (tmo < next)
 250                         next = tmo;
 251         }
 252         if (xp->lft.soft_add_expires_seconds) {
 253                 time64_t tmo = xp->lft.soft_add_expires_seconds +
 254                         xp->curlft.add_time - now;
 255                 if (tmo <= 0) {
 256                         warn = 1;
 257                         tmo = XFRM_KM_TIMEOUT;
 258                 }
 259                 if (tmo < next)
 260                         next = tmo;
 261         }
 262         if (xp->lft.soft_use_expires_seconds) {
 263                 time64_t tmo = xp->lft.soft_use_expires_seconds +
 264                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 265                 if (tmo <= 0) {
 266                         warn = 1;
 267                         tmo = XFRM_KM_TIMEOUT;
 268                 }
 269                 if (tmo < next)
 270                         next = tmo;
 271         }
 272
 273         if (warn)
 274                 km_policy_expired(xp, dir, 0, 0);
 275         if (next != TIME64_MAX &&
 276             !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
 277                 xfrm_pol_hold(xp);
 278
 279 out:
 280         read_unlock(&xp->lock);
 281         xfrm_pol_put(xp);
 282         return;
 283
 284 expired:
 285         read_unlock(&xp->lock);
 286         if (!xfrm_policy_delete(xp, dir))
 287                 km_policy_expired(xp, dir, 1, 0);
 288         xfrm_pol_put(xp);
 289 }
 290
 291 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 292  * SPD calls.
 293  */
 294
 295 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
 296 {
 297         struct xfrm_policy *policy;
 298
 299         policy = kzalloc(sizeof(struct xfrm_policy), gfp);
 300
 301         if (policy) {
 302                 write_pnet(&policy->xp_net, net);
 303                 INIT_LIST_HEAD(&policy->walk.all);
 304                 INIT_HLIST_NODE(&policy->bydst_inexact_list);
 305                 INIT_HLIST_NODE(&policy->bydst);
 306                 INIT_HLIST_NODE(&policy->byidx);
 307                 rwlock_init(&policy->lock);
 308                 refcount_set(&policy->refcnt, 1);
 309                 skb_queue_head_init(&policy->polq.hold_queue);
 310                 timer_setup(&policy->timer, xfrm_policy_timer, 0);
 311                 timer_setup(&policy->polq.hold_timer,
 312                             xfrm_policy_queue_process, 0);
 313         }
 314         return policy;
 315 }
 316 EXPORT_SYMBOL(xfrm_policy_alloc);
 317
 318 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
 319 {
 320         struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
 321
 322         security_xfrm_policy_free(policy->security);
 323         kfree(policy);
 324 }
 325
 326 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
 327
 328 void xfrm_policy_destroy(struct xfrm_policy *policy)
 329 {
 330         BUG_ON(!policy->walk.dead);
 331
 332         if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
 333                 BUG();
 334
 335         call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
 336 }
 337 EXPORT_SYMBOL(xfrm_policy_destroy);
 338
 339 /* Rule must be locked. Release descendant resources, announce
 340  * entry dead. The rule must be unlinked from lists to the moment.
 341  */
 342
 343 static void xfrm_policy_kill(struct xfrm_policy *policy)
 344 {
 345         policy->walk.dead = 1;
 346
 347         atomic_inc(&policy->genid);
 348
 349         if (del_timer(&policy->polq.hold_timer))
 350                 xfrm_pol_put(policy);
 351         skb_queue_purge(&policy->polq.hold_queue);
 352
 353         if (del_timer(&policy->timer))
 354                 xfrm_pol_put(policy);
 355
 356         xfrm_pol_put(policy);
 357 }
 358
 359 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
 360
 361 static inline unsigned int idx_hash(struct net *net, u32 index)
 362 {
 363         return __idx_hash(index, net->xfrm.policy_idx_hmask);
 364 }
 365
 366 /* calculate policy hash thresholds */
 367 static void __get_hash_thresh(struct net *net,
 368                               unsigned short family, int dir,
 369                               u8 *dbits, u8 *sbits)
 370 {
 371         switch (family) {
 372         case AF_INET:
 373                 *dbits = net->xfrm.policy_bydst[dir].dbits4;
 374                 *sbits = net->xfrm.policy_bydst[dir].sbits4;
 375                 break;
 376
 377         case AF_INET6:
 378                 *dbits = net->xfrm.policy_bydst[dir].dbits6;
 379                 *sbits = net->xfrm.policy_bydst[dir].sbits6;
 380                 break;
 381
 382         default:
 383                 *dbits = 0;
 384                 *sbits = 0;
 385         }
 386 }
 387
 388 static struct hlist_head *policy_hash_bysel(struct net *net,
 389                                             const struct xfrm_selector *sel,
 390                                             unsigned short family, int dir)
 391 {
 392         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 393         unsigned int hash;
 394         u8 dbits;
 395         u8 sbits;
 396
 397         __get_hash_thresh(net, family, dir, &dbits, &sbits);
 398         hash = __sel_hash(sel, family, hmask, dbits, sbits);
 399
 400         if (hash == hmask + 1)
 401                 return NULL;
 402
 403         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
 404                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
 405 }
 406
 407 static struct hlist_head *policy_hash_direct(struct net *net,
 408                                              const xfrm_address_t *daddr,
 409                                              const xfrm_address_t *saddr,
 410                                              unsigned short family, int dir)
 411 {
 412         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 413         unsigned int hash;
 414         u8 dbits;
 415         u8 sbits;
 416
 417         __get_hash_thresh(net, family, dir, &dbits, &sbits);
 418         hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
 419
 420         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
 421                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
 422 }
 423
 424 static void xfrm_dst_hash_transfer(struct net *net,
 425                                    struct hlist_head *list,
 426                                    struct hlist_head *ndsttable,
 427                                    unsigned int nhashmask,
 428                                    int dir)
 429 {
 430         struct hlist_node *tmp, *entry0 = NULL;
 431         struct xfrm_policy *pol;
 432         unsigned int h0 = 0;
 433         u8 dbits;
 434         u8 sbits;
 435
 436 redo:
 437         hlist_for_each_entry_safe(pol, tmp, list, bydst) {
 438                 unsigned int h;
 439
 440                 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
 441                 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
 442                                 pol->family, nhashmask, dbits, sbits);
 443                 if (!entry0) {
 444                         hlist_del_rcu(&pol->bydst);
 445                         hlist_add_head_rcu(&pol->bydst, ndsttable + h);
 446                         h0 = h;
 447                 } else {
 448                         if (h != h0)
 449                                 continue;
 450                         hlist_del_rcu(&pol->bydst);
 451                         hlist_add_behind_rcu(&pol->bydst, entry0);
 452                 }
 453                 entry0 = &pol->bydst;
 454         }
 455         if (!hlist_empty(list)) {
 456                 entry0 = NULL;
 457                 goto redo;
 458         }
 459 }
 460
 461 static void xfrm_idx_hash_transfer(struct hlist_head *list,
 462                                    struct hlist_head *nidxtable,
 463                                    unsigned int nhashmask)
 464 {
 465         struct hlist_node *tmp;
 466         struct xfrm_policy *pol;
 467
 468         hlist_for_each_entry_safe(pol, tmp, list, byidx) {
 469                 unsigned int h;
 470
 471                 h = __idx_hash(pol->index, nhashmask);
 472                 hlist_add_head(&pol->byidx, nidxtable+h);
 473         }
 474 }
 475
 476 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
 477 {
 478         return ((old_hmask + 1) << 1) - 1;
 479 }
 480
 481 static void xfrm_bydst_resize(struct net *net, int dir)
 482 {
 483         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 484         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 485         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 486         struct hlist_head *ndst = xfrm_hash_alloc(nsize);
 487         struct hlist_head *odst;
 488         int i;
 489
 490         if (!ndst)
 491                 return;
 492
 493         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
 494         write_seqcount_begin(&xfrm_policy_hash_generation);
 495
 496         odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
 497                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
 498
 499         odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
 500                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
 501
 502         for (i = hmask; i >= 0; i--)
 503                 xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
 504
 505         rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
 506         net->xfrm.policy_bydst[dir].hmask = nhashmask;
 507
 508         write_seqcount_end(&xfrm_policy_hash_generation);
 509         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
 510
 511         synchronize_rcu();
 512
 513         xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
 514 }
 515
 516 static void xfrm_byidx_resize(struct net *net, int total)
 517 {
 518         unsigned int hmask = net->xfrm.policy_idx_hmask;
 519         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 520         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 521         struct hlist_head *oidx = net->xfrm.policy_byidx;
 522         struct hlist_head *nidx = xfrm_hash_alloc(nsize);
 523         int i;
 524
 525         if (!nidx)
 526                 return;
 527
 528         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
 529
 530         for (i = hmask; i >= 0; i--)
 531                 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
 532
 533         net->xfrm.policy_byidx = nidx;
 534         net->xfrm.policy_idx_hmask = nhashmask;
 535
 536         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
 537
 538         xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
 539 }
 540
 541 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
 542 {
 543         unsigned int cnt = net->xfrm.policy_count[dir];
 544         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 545
 546         if (total)
 547                 *total += cnt;
 548
 549         if ((hmask + 1) < xfrm_policy_hashmax &&
 550             cnt > hmask)
 551                 return 1;
 552
 553         return 0;
 554 }
 555
 556 static inline int xfrm_byidx_should_resize(struct net *net, int total)
 557 {
 558         unsigned int hmask = net->xfrm.policy_idx_hmask;
 559
 560         if ((hmask + 1) < xfrm_policy_hashmax &&
 561             total > hmask)
 562                 return 1;
 563
 564         return 0;
 565 }
 566
 567 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
 568 {
 569         si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
 570         si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
 571         si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
 572         si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
 573         si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
 574         si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
 575         si->spdhcnt = net->xfrm.policy_idx_hmask;
 576         si->spdhmcnt = xfrm_policy_hashmax;
 577 }
 578 EXPORT_SYMBOL(xfrm_spd_getinfo);
 579
 580 static DEFINE_MUTEX(hash_resize_mutex);
 581 static void xfrm_hash_resize(struct work_struct *work)
 582 {
 583         struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
 584         int dir, total;
 585
 586         mutex_lock(&hash_resize_mutex);
 587
 588         total = 0;
 589         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 590                 if (xfrm_bydst_should_resize(net, dir, &total))
 591                         xfrm_bydst_resize(net, dir);
 592         }
 593         if (xfrm_byidx_should_resize(net, total))
 594                 xfrm_byidx_resize(net, total);
 595
 596         mutex_unlock(&hash_resize_mutex);
 597 }
 598
 599 static void xfrm_hash_reset_inexact_table(struct net *net)
 600 {
 601         struct xfrm_pol_inexact_bin *b;
 602
 603         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
 604
 605         list_for_each_entry(b, &net->xfrm.inexact_bins, inexact_bins)
 606                 INIT_HLIST_HEAD(&b->hhead);
 607 }
 608
 609 /* Make sure *pol can be inserted into fastbin.
 610  * Useful to check that later insert requests will be sucessful
 611  * (provided xfrm_policy_lock is held throughout).
 612  */
 613 static struct xfrm_pol_inexact_bin *
 614 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
 615 {
 616         struct xfrm_pol_inexact_bin *bin, *prev;
 617         struct xfrm_pol_inexact_key k = {
 618                 .family = pol->family,
 619                 .type = pol->type,
 620                 .dir = dir,
 621         };
 622         struct net *net = xp_net(pol);
 623
 624         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
 625
 626         write_pnet(&k.net, net);
 627         bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
 628                                      xfrm_pol_inexact_params);
 629         if (bin)
 630                 return bin;
 631
 632         bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
 633         if (!bin)
 634                 return NULL;
 635
 636         bin->k = k;
 637         INIT_HLIST_HEAD(&bin->hhead);
 638
 639         prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
 640                                                 &bin->k, &bin->head,
 641                                                 xfrm_pol_inexact_params);
 642         if (!prev) {
 643                 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
 644                 return bin;
 645         }
 646
 647         kfree(bin);
 648
 649         return IS_ERR(prev) ? NULL : prev;
 650 }
 651
 652 static void xfrm_policy_inexact_delete_bin(struct net *net,
 653                                            struct xfrm_pol_inexact_bin *b)
 654 {
 655         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
 656
 657         if (!hlist_empty(&b->hhead))
 658                 return;
 659
 660         if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
 661                                    xfrm_pol_inexact_params) == 0) {
 662                 list_del(&b->inexact_bins);
 663                 kfree_rcu(b, rcu);
 664         }
 665 }
 666
 667 static void __xfrm_policy_inexact_flush(struct net *net)
 668 {
 669         struct xfrm_pol_inexact_bin *bin;
 670
 671         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
 672
 673         list_for_each_entry(bin, &net->xfrm.inexact_bins, inexact_bins)
 674                 xfrm_policy_inexact_delete_bin(net, bin);
 675 }
 676
 677 static struct xfrm_policy *
 678 xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
 679 {
 680         struct xfrm_pol_inexact_bin *bin;
 681         struct xfrm_policy *delpol;
 682         struct hlist_head *chain;
 683         struct net *net;
 684
 685         bin = xfrm_policy_inexact_alloc_bin(policy, dir);
 686         if (!bin)
 687                 return ERR_PTR(-ENOMEM);
 688
 689         delpol = xfrm_policy_insert_list(&bin->hhead, policy, excl);
 690         if (delpol && excl)
 691                 return ERR_PTR(-EEXIST);
 692
 693         net = xp_net(policy);
 694         chain = &net->xfrm.policy_inexact[dir];
 695         xfrm_policy_insert_inexact_list(chain, policy);
 696
 697         return delpol;
 698 }
 699
 700 static void xfrm_hash_rebuild(struct work_struct *work)
 701 {
 702         struct net *net = container_of(work, struct net,
 703                                        xfrm.policy_hthresh.work);
 704         unsigned int hmask;
 705         struct xfrm_policy *pol;
 706         struct xfrm_policy *policy;
 707         struct hlist_head *chain;
 708         struct hlist_head *odst;
 709         struct hlist_node *newpos;
 710         int i;
 711         int dir;
 712         unsigned seq;
 713         u8 lbits4, rbits4, lbits6, rbits6;
 714
 715         mutex_lock(&hash_resize_mutex);
 716
 717         /* read selector prefixlen thresholds */
 718         do {
 719                 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
 720
 721                 lbits4 = net->xfrm.policy_hthresh.lbits4;
 722                 rbits4 = net->xfrm.policy_hthresh.rbits4;
 723                 lbits6 = net->xfrm.policy_hthresh.lbits6;
 724                 rbits6 = net->xfrm.policy_hthresh.rbits6;
 725         } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
 726
 727         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
 728
 729         /* make sure that we can insert the indirect policies again before
 730          * we start with destructive action.
 731          */
 732         list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
 733                 u8 dbits, sbits;
 734
 735                 dir = xfrm_policy_id2dir(policy->index);
 736                 if (policy->walk.dead || dir >= XFRM_POLICY_MAX)
 737                         continue;
 738
 739                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
 740                         if (policy->family == AF_INET) {
 741                                 dbits = rbits4;
 742                                 sbits = lbits4;
 743                         } else {
 744                                 dbits = rbits6;
 745                                 sbits = lbits6;
 746                         }
 747                 } else {
 748                         if (policy->family == AF_INET) {
 749                                 dbits = lbits4;
 750                                 sbits = rbits4;
 751                         } else {
 752                                 dbits = lbits6;
 753                                 sbits = rbits6;
 754                         }
 755                 }
 756
 757                 if (policy->selector.prefixlen_d < dbits ||
 758                     policy->selector.prefixlen_s < sbits)
 759                         continue;
 760
 761                 if (!xfrm_policy_inexact_alloc_bin(policy, dir))
 762                         goto out_unlock;
 763         }
 764
 765         /* reset the bydst and inexact table in all directions */
 766         xfrm_hash_reset_inexact_table(net);
 767
 768         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 769                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
 770                 hmask = net->xfrm.policy_bydst[dir].hmask;
 771                 odst = net->xfrm.policy_bydst[dir].table;
 772                 for (i = hmask; i >= 0; i--)
 773                         INIT_HLIST_HEAD(odst + i);
 774                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
 775                         /* dir out => dst = remote, src = local */
 776                         net->xfrm.policy_bydst[dir].dbits4 = rbits4;
 777                         net->xfrm.policy_bydst[dir].sbits4 = lbits4;
 778                         net->xfrm.policy_bydst[dir].dbits6 = rbits6;
 779                         net->xfrm.policy_bydst[dir].sbits6 = lbits6;
 780                 } else {
 781                         /* dir in/fwd => dst = local, src = remote */
 782                         net->xfrm.policy_bydst[dir].dbits4 = lbits4;
 783                         net->xfrm.policy_bydst[dir].sbits4 = rbits4;
 784                         net->xfrm.policy_bydst[dir].dbits6 = lbits6;
 785                         net->xfrm.policy_bydst[dir].sbits6 = rbits6;
 786                 }
 787         }
 788
 789         /* re-insert all policies by order of creation */
 790         list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
 791                 if (policy->walk.dead ||
 792                     xfrm_policy_id2dir(policy->index) >= XFRM_POLICY_MAX) {
 793                         /* skip socket policies */
 794                         continue;
 795                 }
 796                 newpos = NULL;
 797                 chain = policy_hash_bysel(net, &policy->selector,
 798                                           policy->family,
 799                                           xfrm_policy_id2dir(policy->index));
 800                 if (!chain) {
 801                         void *p = xfrm_policy_inexact_insert(policy, dir, 0);
 802
 803                         WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
 804                         continue;
 805                 }
 806
 807                 hlist_for_each_entry(pol, chain, bydst) {
 808                         if (policy->priority >= pol->priority)
 809                                 newpos = &pol->bydst;
 810                         else
 811                                 break;
 812                 }
 813                 if (newpos)
 814                         hlist_add_behind_rcu(&policy->bydst, newpos);
 815                 else
 816                         hlist_add_head_rcu(&policy->bydst, chain);
 817         }
 818
 819 out_unlock:
 820         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
 821
 822         mutex_unlock(&hash_resize_mutex);
 823 }
 824
 825 void xfrm_policy_hash_rebuild(struct net *net)
 826 {
 827         schedule_work(&net->xfrm.policy_hthresh.work);
 828 }
 829 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
 830
 831 /* Generate new index... KAME seems to generate them ordered by cost
 832  * of an absolute inpredictability of ordering of rules. This will not pass. */
 833 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
 834 {
 835         static u32 idx_generator;
 836
 837         for (;;) {
 838                 struct hlist_head *list;
 839                 struct xfrm_policy *p;
 840                 u32 idx;
 841                 int found;
 842
 843                 if (!index) {
 844                         idx = (idx_generator | dir);
 845                         idx_generator += 8;
 846                 } else {
 847                         idx = index;
 848                         index = 0;
 849                 }
 850
 851                 if (idx == 0)
 852                         idx = 8;
 853                 list = net->xfrm.policy_byidx + idx_hash(net, idx);
 854                 found = 0;
 855                 hlist_for_each_entry(p, list, byidx) {
 856                         if (p->index == idx) {
 857                                 found = 1;
 858                                 break;
 859                         }
 860                 }
 861                 if (!found)
 862                         return idx;
 863         }
 864 }
 865
 866 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
 867 {
 868         u32 *p1 = (u32 *) s1;
 869         u32 *p2 = (u32 *) s2;
 870         int len = sizeof(struct xfrm_selector) / sizeof(u32);
 871         int i;
 872
 873         for (i = 0; i < len; i++) {
 874                 if (p1[i] != p2[i])
 875                         return 1;
 876         }
 877
 878         return 0;
 879 }
 880
 881 static void xfrm_policy_requeue(struct xfrm_policy *old,
 882                                 struct xfrm_policy *new)
 883 {
 884         struct xfrm_policy_queue *pq = &old->polq;
 885         struct sk_buff_head list;
 886
 887         if (skb_queue_empty(&pq->hold_queue))
 888                 return;
 889
 890         __skb_queue_head_init(&list);
 891
 892         spin_lock_bh(&pq->hold_queue.lock);
 893         skb_queue_splice_init(&pq->hold_queue, &list);
 894         if (del_timer(&pq->hold_timer))
 895                 xfrm_pol_put(old);
 896         spin_unlock_bh(&pq->hold_queue.lock);
 897
 898         pq = &new->polq;
 899
 900         spin_lock_bh(&pq->hold_queue.lock);
 901         skb_queue_splice(&list, &pq->hold_queue);
 902         pq->timeout = XFRM_QUEUE_TMO_MIN;
 903         if (!mod_timer(&pq->hold_timer, jiffies))
 904                 xfrm_pol_hold(new);
 905         spin_unlock_bh(&pq->hold_queue.lock);
 906 }
 907
 908 static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
 909                                    struct xfrm_policy *pol)
 910 {
 911         u32 mark = policy->mark.v & policy->mark.m;
 912
 913         if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
 914                 return true;
 915
 916         if ((mark & pol->mark.m) == pol->mark.v &&
 917             policy->priority == pol->priority)
 918                 return true;
 919
 920         return false;
 921 }
 922
 923 static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
 924 {
 925         const struct xfrm_pol_inexact_key *k = data;
 926         u32 a = k->type << 24 | k->dir << 16 | k->family;
 927
 928         return jhash_2words(a, net_hash_mix(read_pnet(&k->net)), seed);
 929 }
 930
 931 static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
 932 {
 933         const struct xfrm_pol_inexact_bin *b = data;
 934
 935         return xfrm_pol_bin_key(&b->k, 0, seed);
 936 }
 937
 938 static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
 939                             const void *ptr)
 940 {
 941         const struct xfrm_pol_inexact_key *key = arg->key;
 942         const struct xfrm_pol_inexact_bin *b = ptr;
 943         int ret;
 944
 945         if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
 946                 return -1;
 947
 948         ret = b->k.dir ^ key->dir;
 949         if (ret)
 950                 return ret;
 951
 952         ret = b->k.type ^ key->type;
 953         if (ret)
 954                 return ret;
 955
 956         ret = b->k.family ^ key->family;
 957         if (ret)
 958                 return ret;
 959
 960         return 0;
 961 }
 962
 963 static const struct rhashtable_params xfrm_pol_inexact_params = {
 964         .head_offset            = offsetof(struct xfrm_pol_inexact_bin, head),
 965         .hashfn                 = xfrm_pol_bin_key,
 966         .obj_hashfn             = xfrm_pol_bin_obj,
 967         .obj_cmpfn              = xfrm_pol_bin_cmp,
 968         .automatic_shrinking    = true,
 969 };
 970
 971 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
 972                                             struct xfrm_policy *policy)
 973 {
 974         struct xfrm_policy *pol, *delpol = NULL;
 975         struct hlist_node *newpos = NULL;
 976
 977         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
 978                 if (pol->type == policy->type &&
 979                     pol->if_id == policy->if_id &&
 980                     !selector_cmp(&pol->selector, &policy->selector) &&
 981                     xfrm_policy_mark_match(policy, pol) &&
 982                     xfrm_sec_ctx_match(pol->security, policy->security) &&
 983                     !WARN_ON(delpol)) {
 984                         delpol = pol;
 985                         if (policy->priority > pol->priority)
 986                                 continue;
 987                 } else if (policy->priority >= pol->priority) {
 988                         newpos = &pol->bydst_inexact_list;
 989                         continue;
 990                 }
 991                 if (delpol)
 992                         break;
 993         }
 994
 995         if (newpos)
 996                 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos);
 997         else
 998                 hlist_add_head_rcu(&policy->bydst_inexact_list, chain);
 999 }
1000
1001 static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
1002                                                    struct xfrm_policy *policy,
1003                                                    bool excl)
1004 {
1005         struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
1006
1007         hlist_for_each_entry(pol, chain, bydst) {
1008                 if (pol->type == policy->type &&
1009                     pol->if_id == policy->if_id &&
1010                     !selector_cmp(&pol->selector, &policy->selector) &&
1011                     xfrm_policy_mark_match(policy, pol) &&
1012                     xfrm_sec_ctx_match(pol->security, policy->security) &&
1013                     !WARN_ON(delpol)) {
1014                         if (excl)
1015                                 return ERR_PTR(-EEXIST);
1016                         delpol = pol;
1017                         if (policy->priority > pol->priority)
1018                                 continue;
1019                 } else if (policy->priority >= pol->priority) {
1020                         newpos = pol;
1021                         continue;
1022                 }
1023                 if (delpol)
1024                         break;
1025         }
1026
1027         if (newpos)
1028                 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
1029         else
1030                 hlist_add_head_rcu(&policy->bydst, chain);
1031
1032         return delpol;
1033 }
1034
1035 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
1036 {
1037         struct net *net = xp_net(policy);
1038         struct xfrm_policy *delpol;
1039         struct hlist_head *chain;
1040
1041         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1042         chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
1043         if (chain)
1044                 delpol = xfrm_policy_insert_list(chain, policy, excl);
1045         else
1046                 delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1047
1048         if (IS_ERR(delpol)) {
1049                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1050                 return PTR_ERR(delpol);
1051         }
1052
1053         __xfrm_policy_link(policy, dir);
1054
1055         /* After previous checking, family can either be AF_INET or AF_INET6 */
1056         if (policy->family == AF_INET)
1057                 rt_genid_bump_ipv4(net);
1058         else
1059                 rt_genid_bump_ipv6(net);
1060
1061         if (delpol) {
1062                 xfrm_policy_requeue(delpol, policy);
1063                 __xfrm_policy_unlink(delpol, dir);
1064         }
1065         policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1066         hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
1067         policy->curlft.add_time = ktime_get_real_seconds();
1068         policy->curlft.use_time = 0;
1069         if (!mod_timer(&policy->timer, jiffies + HZ))
1070                 xfrm_pol_hold(policy);
1071         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1072
1073         if (delpol)
1074                 xfrm_policy_kill(delpol);
1075         else if (xfrm_bydst_should_resize(net, dir, NULL))
1076                 schedule_work(&net->xfrm.policy_hash_work);
1077
1078         return 0;
1079 }
1080 EXPORT_SYMBOL(xfrm_policy_insert);
1081
1082 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u32 if_id,
1083                                           u8 type, int dir,
1084                                           struct xfrm_selector *sel,
1085                                           struct xfrm_sec_ctx *ctx, int delete,
1086                                           int *err)
1087 {
1088         struct xfrm_pol_inexact_bin *bin = NULL;
1089         struct xfrm_policy *pol, *ret = NULL;
1090         struct hlist_head *chain;
1091
1092         *err = 0;
1093         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1094         chain = policy_hash_bysel(net, sel, sel->family, dir);
1095         if (!chain) {
1096                 bin = xfrm_policy_inexact_lookup(net, type,
1097                                                  sel->family, dir);
1098                 if (!bin) {
1099                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1100                         return NULL;
1101                 }
1102
1103                 chain = &bin->hhead;
1104         }
1105
1106         ret = NULL;
1107         hlist_for_each_entry(pol, chain, bydst) {
1108                 if (pol->type == type &&
1109                     pol->if_id == if_id &&
1110                     (mark & pol->mark.m) == pol->mark.v &&
1111                     !selector_cmp(sel, &pol->selector) &&
1112                     xfrm_sec_ctx_match(ctx, pol->security)) {
1113                         xfrm_pol_hold(pol);
1114                         if (delete) {
1115                                 *err = security_xfrm_policy_delete(
1116                                                                 pol->security);
1117                                 if (*err) {
1118                                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1119                                         return pol;
1120                                 }
1121                                 __xfrm_policy_unlink(pol, dir);
1122                                 xfrm_policy_inexact_delete_bin(net, bin);
1123                         }
1124                         ret = pol;
1125                         break;
1126                 }
1127         }
1128         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1129
1130         if (ret && delete)
1131                 xfrm_policy_kill(ret);
1132         return ret;
1133 }
1134 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1135
1136 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u32 if_id,
1137                                      u8 type, int dir, u32 id, int delete,
1138                                      int *err)
1139 {
1140         struct xfrm_policy *pol, *ret;
1141         struct hlist_head *chain;
1142
1143         *err = -ENOENT;
1144         if (xfrm_policy_id2dir(id) != dir)
1145                 return NULL;
1146
1147         *err = 0;
1148         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1149         chain = net->xfrm.policy_byidx + idx_hash(net, id);
1150         ret = NULL;
1151         hlist_for_each_entry(pol, chain, byidx) {
1152                 if (pol->type == type && pol->index == id &&
1153                     pol->if_id == if_id &&
1154                     (mark & pol->mark.m) == pol->mark.v) {
1155                         xfrm_pol_hold(pol);
1156                         if (delete) {
1157                                 *err = security_xfrm_policy_delete(
1158                                                                 pol->security);
1159                                 if (*err) {
1160                                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1161                                         return pol;
1162                                 }
1163                                 __xfrm_policy_unlink(pol, dir);
1164                         }
1165                         ret = pol;
1166                         break;
1167                 }
1168         }
1169         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1170
1171         if (ret && delete)
1172                 xfrm_policy_kill(ret);
1173         return ret;
1174 }
1175 EXPORT_SYMBOL(xfrm_policy_byid);
1176
1177 #ifdef CONFIG_SECURITY_NETWORK_XFRM
1178 static inline int
1179 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1180 {
1181         struct xfrm_policy *pol;
1182         int err = 0;
1183
1184         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1185                 if (pol->walk.dead ||
1186                     xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1187                     pol->type != type)
1188                         continue;
1189
1190                 err = security_xfrm_policy_delete(pol->security);
1191                 if (err) {
1192                         xfrm_audit_policy_delete(pol, 0, task_valid);
1193                         return err;
1194                 }
1195         }
1196         return err;
1197 }
1198 #else
1199 static inline int
1200 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1201 {
1202         return 0;
1203 }
1204 #endif
1205
1206 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1207 {
1208         int dir, err = 0, cnt = 0;
1209         struct xfrm_policy *pol;
1210
1211         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1212
1213         err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1214         if (err)
1215                 goto out;
1216
1217 again:
1218         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1219                 dir = xfrm_policy_id2dir(pol->index);
1220                 if (pol->walk.dead ||
1221                     dir >= XFRM_POLICY_MAX ||
1222                     pol->type != type)
1223                         continue;
1224
1225                 __xfrm_policy_unlink(pol, dir);
1226                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1227                 cnt++;
1228                 xfrm_audit_policy_delete(pol, 1, task_valid);
1229                 xfrm_policy_kill(pol);
1230                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1231                 goto again;
1232         }
1233         if (cnt)
1234                 __xfrm_policy_inexact_flush(net);
1235         else
1236                 err = -ESRCH;
1237 out:
1238         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1239         return err;
1240 }
1241 EXPORT_SYMBOL(xfrm_policy_flush);
1242
1243 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1244                      int (*func)(struct xfrm_policy *, int, int, void*),
1245                      void *data)
1246 {
1247         struct xfrm_policy *pol;
1248         struct xfrm_policy_walk_entry *x;
1249         int error = 0;
1250
1251         if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1252             walk->type != XFRM_POLICY_TYPE_ANY)
1253                 return -EINVAL;
1254
1255         if (list_empty(&walk->walk.all) && walk->seq != 0)
1256                 return 0;
1257
1258         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1259         if (list_empty(&walk->walk.all))
1260                 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1261         else
1262                 x = list_first_entry(&walk->walk.all,
1263                                      struct xfrm_policy_walk_entry, all);
1264
1265         list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1266                 if (x->dead)
1267                         continue;
1268                 pol = container_of(x, struct xfrm_policy, walk);
1269                 if (walk->type != XFRM_POLICY_TYPE_ANY &&
1270                     walk->type != pol->type)
1271                         continue;
1272                 error = func(pol, xfrm_policy_id2dir(pol->index),
1273                              walk->seq, data);
1274                 if (error) {
1275                         list_move_tail(&walk->walk.all, &x->all);
1276                         goto out;
1277                 }
1278                 walk->seq++;
1279         }
1280         if (walk->seq == 0) {
1281                 error = -ENOENT;
1282                 goto out;
1283         }
1284         list_del_init(&walk->walk.all);
1285 out:
1286         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1287         return error;
1288 }
1289 EXPORT_SYMBOL(xfrm_policy_walk);
1290
1291 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1292 {
1293         INIT_LIST_HEAD(&walk->walk.all);
1294         walk->walk.dead = 1;
1295         walk->type = type;
1296         walk->seq = 0;
1297 }
1298 EXPORT_SYMBOL(xfrm_policy_walk_init);
1299
1300 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1301 {
1302         if (list_empty(&walk->walk.all))
1303                 return;
1304
1305         spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1306         list_del(&walk->walk.all);
1307         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1308 }
1309 EXPORT_SYMBOL(xfrm_policy_walk_done);
1310
1311 /*
1312  * Find policy to apply to this flow.
1313  *
1314  * Returns 0 if policy found, else an -errno.
1315  */
1316 static int xfrm_policy_match(const struct xfrm_policy *pol,
1317                              const struct flowi *fl,
1318                              u8 type, u16 family, int dir, u32 if_id)
1319 {
1320         const struct xfrm_selector *sel = &pol->selector;
1321         int ret = -ESRCH;
1322         bool match;
1323
1324         if (pol->family != family ||
1325             pol->if_id != if_id ||
1326             (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1327             pol->type != type)
1328                 return ret;
1329
1330         match = xfrm_selector_match(sel, fl, family);
1331         if (match)
1332                 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
1333                                                   dir);
1334         return ret;
1335 }
1336
1337 static struct xfrm_pol_inexact_bin *
1338 xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family, u8 dir)
1339 {
1340         struct xfrm_pol_inexact_key k = {
1341                 .family = family,
1342                 .type = type,
1343                 .dir = dir,
1344         };
1345
1346         write_pnet(&k.net, net);
1347
1348         return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
1349                                  xfrm_pol_inexact_params);
1350 }
1351
1352 static struct xfrm_pol_inexact_bin *
1353 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir)
1354 {
1355         struct xfrm_pol_inexact_bin *bin;
1356
1357         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1358
1359         rcu_read_lock();
1360         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir);
1361         rcu_read_unlock();
1362
1363         return bin;
1364 }
1365
1366 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
1367                                                      const struct flowi *fl,
1368                                                      u16 family, u8 dir,
1369                                                      u32 if_id)
1370 {
1371         const xfrm_address_t *daddr, *saddr;
1372         struct xfrm_pol_inexact_bin *bin;
1373         struct xfrm_policy *pol, *ret;
1374         struct hlist_head *chain;
1375         unsigned int sequence;
1376         u32 priority;
1377         int err;
1378
1379         daddr = xfrm_flowi_daddr(fl, family);
1380         saddr = xfrm_flowi_saddr(fl, family);
1381         if (unlikely(!daddr || !saddr))
1382                 return NULL;
1383
1384         rcu_read_lock();
1385  retry:
1386         do {
1387                 sequence = read_seqcount_begin(&xfrm_policy_hash_generation);
1388                 chain = policy_hash_direct(net, daddr, saddr, family, dir);
1389         } while (read_seqcount_retry(&xfrm_policy_hash_generation, sequence));
1390
1391         priority = ~0U;
1392         ret = NULL;
1393         hlist_for_each_entry_rcu(pol, chain, bydst) {
1394                 err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
1395                 if (err) {
1396                         if (err == -ESRCH)
1397                                 continue;
1398                         else {
1399                                 ret = ERR_PTR(err);
1400                                 goto fail;
1401                         }
1402                 } else {
1403                         ret = pol;
1404                         priority = ret->priority;
1405                         break;
1406                 }
1407         }
1408         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir);
1409         if (!bin)
1410                 goto skip_inexact;
1411         chain = &bin->hhead;
1412         hlist_for_each_entry_rcu(pol, chain, bydst) {
1413                 if ((pol->priority >= priority) && ret)
1414                         break;
1415
1416                 err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
1417                 if (err) {
1418                         if (err == -ESRCH)
1419                                 continue;
1420                         else {
1421                                 ret = ERR_PTR(err);
1422                                 goto fail;
1423                         }
1424                 } else {
1425                         ret = pol;
1426                         break;
1427                 }
1428         }
1429
1430 skip_inexact:
1431         if (read_seqcount_retry(&xfrm_policy_hash_generation, sequence))
1432                 goto retry;
1433
1434         if (ret && !xfrm_pol_hold_rcu(ret))
1435                 goto retry;
1436 fail:
1437         rcu_read_unlock();
1438
1439         return ret;
1440 }
1441
1442 static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
1443                                               const struct flowi *fl,
1444                                               u16 family, u8 dir, u32 if_id)
1445 {
1446 #ifdef CONFIG_XFRM_SUB_POLICY
1447         struct xfrm_policy *pol;
1448
1449         pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
1450                                         dir, if_id);
1451         if (pol != NULL)
1452                 return pol;
1453 #endif
1454         return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
1455                                          dir, if_id);
1456 }
1457
1458 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
1459                                                  const struct flowi *fl,
1460                                                  u16 family, u32 if_id)
1461 {
1462         struct xfrm_policy *pol;
1463
1464         rcu_read_lock();
1465  again:
1466         pol = rcu_dereference(sk->sk_policy[dir]);
1467         if (pol != NULL) {
1468                 bool match;
1469                 int err = 0;
1470
1471                 if (pol->family != family) {
1472                         pol = NULL;
1473                         goto out;
1474                 }
1475
1476                 match = xfrm_selector_match(&pol->selector, fl, family);
1477                 if (match) {
1478                         if ((sk->sk_mark & pol->mark.m) != pol->mark.v ||
1479                             pol->if_id != if_id) {
1480                                 pol = NULL;
1481                                 goto out;
1482                         }
1483                         err = security_xfrm_policy_lookup(pol->security,
1484                                                       fl->flowi_secid,
1485                                                       dir);
1486                         if (!err) {
1487                                 if (!xfrm_pol_hold_rcu(pol))
1488                                         goto again;
1489                         } else if (err == -ESRCH) {
1490                                 pol = NULL;
1491                         } else {
1492                                 pol = ERR_PTR(err);
1493                         }
1494                 } else
1495                         pol = NULL;
1496         }
1497 out:
1498         rcu_read_unlock();
1499         return pol;
1500 }
1501
1502 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1503 {
1504         struct net *net = xp_net(pol);
1505
1506         list_add(&pol->walk.all, &net->xfrm.policy_all);
1507         net->xfrm.policy_count[dir]++;
1508         xfrm_pol_hold(pol);
1509 }
1510
1511 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1512                                                 int dir)
1513 {
1514         struct net *net = xp_net(pol);
1515
1516         if (list_empty(&pol->walk.all))
1517                 return NULL;
1518
1519         /* Socket policies are not hashed. */
1520         if (!hlist_unhashed(&pol->bydst)) {
1521                 hlist_del_rcu(&pol->bydst);
1522                 hlist_del_init(&pol->bydst_inexact_list);
1523                 hlist_del(&pol->byidx);
1524         }
1525
1526         list_del_init(&pol->walk.all);
1527         net->xfrm.policy_count[dir]--;
1528
1529         return pol;
1530 }
1531
1532 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
1533 {
1534         __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
1535 }
1536
1537 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
1538 {
1539         __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
1540 }
1541
1542 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1543 {
1544         struct net *net = xp_net(pol);
1545
1546         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1547         pol = __xfrm_policy_unlink(pol, dir);
1548         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1549         if (pol) {
1550                 xfrm_policy_kill(pol);
1551                 return 0;
1552         }
1553         return -ENOENT;
1554 }
1555 EXPORT_SYMBOL(xfrm_policy_delete);
1556
1557 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1558 {
1559         struct net *net = sock_net(sk);
1560         struct xfrm_policy *old_pol;
1561
1562 #ifdef CONFIG_XFRM_SUB_POLICY
1563         if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1564                 return -EINVAL;
1565 #endif
1566
1567         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1568         old_pol = rcu_dereference_protected(sk->sk_policy[dir],
1569                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
1570         if (pol) {
1571                 pol->curlft.add_time = ktime_get_real_seconds();
1572                 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
1573                 xfrm_sk_policy_link(pol, dir);
1574         }
1575         rcu_assign_pointer(sk->sk_policy[dir], pol);
1576         if (old_pol) {
1577                 if (pol)
1578                         xfrm_policy_requeue(old_pol, pol);
1579
1580                 /* Unlinking succeeds always. This is the only function
1581                  * allowed to delete or replace socket policy.
1582                  */
1583                 xfrm_sk_policy_unlink(old_pol, dir);
1584         }
1585         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1586
1587         if (old_pol) {
1588                 xfrm_policy_kill(old_pol);
1589         }
1590         return 0;
1591 }
1592
1593 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
1594 {
1595         struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1596         struct net *net = xp_net(old);
1597
1598         if (newp) {
1599                 newp->selector = old->selector;
1600                 if (security_xfrm_policy_clone(old->security,
1601                                                &newp->security)) {
1602                         kfree(newp);
1603                         return NULL;  /* ENOMEM */
1604                 }
1605                 newp->lft = old->lft;
1606                 newp->curlft = old->curlft;
1607                 newp->mark = old->mark;
1608                 newp->if_id = old->if_id;
1609                 newp->action = old->action;
1610                 newp->flags = old->flags;
1611                 newp->xfrm_nr = old->xfrm_nr;
1612                 newp->index = old->index;
1613                 newp->type = old->type;
1614                 newp->family = old->family;
1615                 memcpy(newp->xfrm_vec, old->xfrm_vec,
1616                        newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1617                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1618                 xfrm_sk_policy_link(newp, dir);
1619                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1620                 xfrm_pol_put(newp);
1621         }
1622         return newp;
1623 }
1624
1625 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
1626 {
1627         const struct xfrm_policy *p;
1628         struct xfrm_policy *np;
1629         int i, ret = 0;
1630
1631         rcu_read_lock();
1632         for (i = 0; i < 2; i++) {
1633                 p = rcu_dereference(osk->sk_policy[i]);
1634                 if (p) {
1635                         np = clone_policy(p, i);
1636                         if (unlikely(!np)) {
1637                                 ret = -ENOMEM;
1638                                 break;
1639                         }
1640                         rcu_assign_pointer(sk->sk_policy[i], np);
1641                 }
1642         }
1643         rcu_read_unlock();
1644         return ret;
1645 }
1646
1647 static int
1648 xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
1649                xfrm_address_t *remote, unsigned short family, u32 mark)
1650 {
1651         int err;
1652         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1653
1654         if (unlikely(afinfo == NULL))
1655                 return -EINVAL;
1656         err = afinfo->get_saddr(net, oif, local, remote, mark);
1657         rcu_read_unlock();
1658         return err;
1659 }
1660
1661 /* Resolve list of templates for the flow, given policy. */
1662
1663 static int
1664 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
1665                       struct xfrm_state **xfrm, unsigned short family)
1666 {
1667         struct net *net = xp_net(policy);
1668         int nx;
1669         int i, error;
1670         xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1671         xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1672         xfrm_address_t tmp;
1673
1674         for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
1675                 struct xfrm_state *x;
1676                 xfrm_address_t *remote = daddr;
1677                 xfrm_address_t *local  = saddr;
1678                 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1679
1680                 if (tmpl->mode == XFRM_MODE_TUNNEL ||
1681                     tmpl->mode == XFRM_MODE_BEET) {
1682                         remote = &tmpl->id.daddr;
1683                         local = &tmpl->saddr;
1684                         if (xfrm_addr_any(local, tmpl->encap_family)) {
1685                                 error = xfrm_get_saddr(net, fl->flowi_oif,
1686                                                        &tmp, remote,
1687                                                        tmpl->encap_family, 0);
1688                                 if (error)
1689                                         goto fail;
1690                                 local = &tmp;
1691                         }
1692                 }
1693
1694                 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
1695                                     family, policy->if_id);
1696
1697                 if (x && x->km.state == XFRM_STATE_VALID) {
1698                         xfrm[nx++] = x;
1699                         daddr = remote;
1700                         saddr = local;
1701                         continue;
1702                 }
1703                 if (x) {
1704                         error = (x->km.state == XFRM_STATE_ERROR ?
1705                                  -EINVAL : -EAGAIN);
1706                         xfrm_state_put(x);
1707                 } else if (error == -ESRCH) {
1708                         error = -EAGAIN;
1709                 }
1710
1711                 if (!tmpl->optional)
1712                         goto fail;
1713         }
1714         return nx;
1715
1716 fail:
1717         for (nx--; nx >= 0; nx--)
1718                 xfrm_state_put(xfrm[nx]);
1719         return error;
1720 }
1721
1722 static int
1723 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
1724                   struct xfrm_state **xfrm, unsigned short family)
1725 {
1726         struct xfrm_state *tp[XFRM_MAX_DEPTH];
1727         struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1728         int cnx = 0;
1729         int error;
1730         int ret;
1731         int i;
1732
1733         for (i = 0; i < npols; i++) {
1734                 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1735                         error = -ENOBUFS;
1736                         goto fail;
1737                 }
1738
1739                 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1740                 if (ret < 0) {
1741                         error = ret;
1742                         goto fail;
1743                 } else
1744                         cnx += ret;
1745         }
1746
1747         /* found states are sorted for outbound processing */
1748         if (npols > 1)
1749                 xfrm_state_sort(xfrm, tpp, cnx, family);
1750
1751         return cnx;
1752
1753  fail:
1754         for (cnx--; cnx >= 0; cnx--)
1755                 xfrm_state_put(tpp[cnx]);
1756         return error;
1757
1758 }
1759
1760 static int xfrm_get_tos(const struct flowi *fl, int family)
1761 {
1762         const struct xfrm_policy_afinfo *afinfo;
1763         int tos;
1764
1765         afinfo = xfrm_policy_get_afinfo(family);
1766         if (!afinfo)
1767                 return 0;
1768
1769         tos = afinfo->get_tos(fl);
1770
1771         rcu_read_unlock();
1772
1773         return tos;
1774 }
1775
1776 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
1777 {
1778         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1779         struct dst_ops *dst_ops;
1780         struct xfrm_dst *xdst;
1781
1782         if (!afinfo)
1783                 return ERR_PTR(-EINVAL);
1784
1785         switch (family) {
1786         case AF_INET:
1787                 dst_ops = &net->xfrm.xfrm4_dst_ops;
1788                 break;
1789 #if IS_ENABLED(CONFIG_IPV6)
1790         case AF_INET6:
1791                 dst_ops = &net->xfrm.xfrm6_dst_ops;
1792                 break;
1793 #endif
1794         default:
1795                 BUG();
1796         }
1797         xdst = dst_alloc(dst_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
1798
1799         if (likely(xdst)) {
1800                 struct dst_entry *dst = &xdst->u.dst;
1801
1802                 memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
1803         } else
1804                 xdst = ERR_PTR(-ENOBUFS);
1805
1806         rcu_read_unlock();
1807
1808         return xdst;
1809 }
1810
1811 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1812                                  int nfheader_len)
1813 {
1814         const struct xfrm_policy_afinfo *afinfo =
1815                 xfrm_policy_get_afinfo(dst->ops->family);
1816         int err;
1817
1818         if (!afinfo)
1819                 return -EINVAL;
1820
1821         err = afinfo->init_path(path, dst, nfheader_len);
1822
1823         rcu_read_unlock();
1824
1825         return err;
1826 }
1827
1828 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
1829                                 const struct flowi *fl)
1830 {
1831         const struct xfrm_policy_afinfo *afinfo =
1832                 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1833         int err;
1834
1835         if (!afinfo)
1836                 return -EINVAL;
1837
1838         err = afinfo->fill_dst(xdst, dev, fl);
1839
1840         rcu_read_unlock();
1841
1842         return err;
1843 }
1844
1845
1846 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
1847  * all the metrics... Shortly, bundle a bundle.
1848  */
1849
1850 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1851                                             struct xfrm_state **xfrm,
1852                                             struct xfrm_dst **bundle,
1853                                             int nx,
1854                                             const struct flowi *fl,
1855                                             struct dst_entry *dst)
1856 {
1857         struct net *net = xp_net(policy);
1858         unsigned long now = jiffies;
1859         struct net_device *dev;
1860         struct xfrm_mode *inner_mode;
1861         struct xfrm_dst *xdst_prev = NULL;
1862         struct xfrm_dst *xdst0 = NULL;
1863         int i = 0;
1864         int err;
1865         int header_len = 0;
1866         int nfheader_len = 0;
1867         int trailer_len = 0;
1868         int tos;
1869         int family = policy->selector.family;
1870         xfrm_address_t saddr, daddr;
1871
1872         xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1873
1874         tos = xfrm_get_tos(fl, family);
1875
1876         dst_hold(dst);
1877
1878         for (; i < nx; i++) {
1879                 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
1880                 struct dst_entry *dst1 = &xdst->u.dst;
1881
1882                 err = PTR_ERR(xdst);
1883                 if (IS_ERR(xdst)) {
1884                         dst_release(dst);
1885                         goto put_states;
1886                 }
1887
1888                 bundle[i] = xdst;
1889                 if (!xdst_prev)
1890                         xdst0 = xdst;
1891                 else
1892                         /* Ref count is taken during xfrm_alloc_dst()
1893                          * No need to do dst_clone() on dst1
1894                          */
1895                         xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
1896
1897                 if (xfrm[i]->sel.family == AF_UNSPEC) {
1898                         inner_mode = xfrm_ip2inner_mode(xfrm[i],
1899                                                         xfrm_af2proto(family));
1900                         if (!inner_mode) {
1901                                 err = -EAFNOSUPPORT;
1902                                 dst_release(dst);
1903                                 goto put_states;
1904                         }
1905                 } else
1906                         inner_mode = xfrm[i]->inner_mode;
1907
1908                 xdst->route = dst;
1909                 dst_copy_metrics(dst1, dst);
1910
1911                 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1912                         __u32 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
1913
1914                         family = xfrm[i]->props.family;
1915                         dst = xfrm_dst_lookup(xfrm[i], tos, fl->flowi_oif,
1916                                               &saddr, &daddr, family, mark);
1917                         err = PTR_ERR(dst);
1918                         if (IS_ERR(dst))
1919                                 goto put_states;
1920                 } else
1921                         dst_hold(dst);
1922
1923                 dst1->xfrm = xfrm[i];
1924                 xdst->xfrm_genid = xfrm[i]->genid;
1925
1926                 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1927                 dst1->flags |= DST_HOST;
1928                 dst1->lastuse = now;
1929
1930                 dst1->input = dst_discard;
1931                 dst1->output = inner_mode->afinfo->output;
1932
1933                 xdst_prev = xdst;
1934
1935                 header_len += xfrm[i]->props.header_len;
1936                 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1937                         nfheader_len += xfrm[i]->props.header_len;
1938                 trailer_len += xfrm[i]->props.trailer_len;
1939         }
1940
1941         xfrm_dst_set_child(xdst_prev, dst);
1942         xdst0->path = dst;
1943
1944         err = -ENODEV;
1945         dev = dst->dev;
1946         if (!dev)
1947                 goto free_dst;
1948
1949         xfrm_init_path(xdst0, dst, nfheader_len);
1950         xfrm_init_pmtu(bundle, nx);
1951
1952         for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
1953              xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
1954                 err = xfrm_fill_dst(xdst_prev, dev, fl);
1955                 if (err)
1956                         goto free_dst;
1957
1958                 xdst_prev->u.dst.header_len = header_len;
1959                 xdst_prev->u.dst.trailer_len = trailer_len;
1960                 header_len -= xdst_prev->u.dst.xfrm->props.header_len;
1961                 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
1962         }
1963
1964         return &xdst0->u.dst;
1965
1966 put_states:
1967         for (; i < nx; i++)
1968                 xfrm_state_put(xfrm[i]);
1969 free_dst:
1970         if (xdst0)
1971                 dst_release_immediate(&xdst0->u.dst);
1972
1973         return ERR_PTR(err);
1974 }
1975
1976 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
1977                                 struct xfrm_policy **pols,
1978                                 int *num_pols, int *num_xfrms)
1979 {
1980         int i;
1981
1982         if (*num_pols == 0 || !pols[0]) {
1983                 *num_pols = 0;
1984                 *num_xfrms = 0;
1985                 return 0;
1986         }
1987         if (IS_ERR(pols[0]))
1988                 return PTR_ERR(pols[0]);
1989
1990         *num_xfrms = pols[0]->xfrm_nr;
1991
1992 #ifdef CONFIG_XFRM_SUB_POLICY
1993         if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1994             pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1995                 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1996                                                     XFRM_POLICY_TYPE_MAIN,
1997                                                     fl, family,
1998                                                     XFRM_POLICY_OUT,
1999                                                     pols[0]->if_id);
2000                 if (pols[1]) {
2001                         if (IS_ERR(pols[1])) {
2002                                 xfrm_pols_put(pols, *num_pols);
2003                                 return PTR_ERR(pols[1]);
2004                         }
2005                         (*num_pols)++;
2006                         (*num_xfrms) += pols[1]->xfrm_nr;
2007                 }
2008         }
2009 #endif
2010         for (i = 0; i < *num_pols; i++) {
2011                 if (pols[i]->action != XFRM_POLICY_ALLOW) {
2012                         *num_xfrms = -1;
2013                         break;
2014                 }
2015         }
2016
2017         return 0;
2018
2019 }
2020
2021 static struct xfrm_dst *
2022 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
2023                                const struct flowi *fl, u16 family,
2024                                struct dst_entry *dst_orig)
2025 {
2026         struct net *net = xp_net(pols[0]);
2027         struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2028         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2029         struct xfrm_dst *xdst;
2030         struct dst_entry *dst;
2031         int err;
2032
2033         /* Try to instantiate a bundle */
2034         err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
2035         if (err <= 0) {
2036                 if (err == 0)
2037                         return NULL;
2038
2039                 if (err != -EAGAIN)
2040                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2041                 return ERR_PTR(err);
2042         }
2043
2044         dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
2045         if (IS_ERR(dst)) {
2046                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2047                 return ERR_CAST(dst);
2048         }
2049
2050         xdst = (struct xfrm_dst *)dst;
2051         xdst->num_xfrms = err;
2052         xdst->num_pols = num_pols;
2053         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2054         xdst->policy_genid = atomic_read(&pols[0]->genid);
2055
2056         return xdst;
2057 }
2058
2059 static void xfrm_policy_queue_process(struct timer_list *t)
2060 {
2061         struct sk_buff *skb;
2062         struct sock *sk;
2063         struct dst_entry *dst;
2064         struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
2065         struct net *net = xp_net(pol);
2066         struct xfrm_policy_queue *pq = &pol->polq;
2067         struct flowi fl;
2068         struct sk_buff_head list;
2069
2070         spin_lock(&pq->hold_queue.lock);
2071         skb = skb_peek(&pq->hold_queue);
2072         if (!skb) {
2073                 spin_unlock(&pq->hold_queue.lock);
2074                 goto out;
2075         }
2076         dst = skb_dst(skb);
2077         sk = skb->sk;
2078         xfrm_decode_session(skb, &fl, dst->ops->family);
2079         spin_unlock(&pq->hold_queue.lock);
2080
2081         dst_hold(xfrm_dst_path(dst));
2082         dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
2083         if (IS_ERR(dst))
2084                 goto purge_queue;
2085
2086         if (dst->flags & DST_XFRM_QUEUE) {
2087                 dst_release(dst);
2088
2089                 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2090                         goto purge_queue;
2091
2092                 pq->timeout = pq->timeout << 1;
2093                 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
2094                         xfrm_pol_hold(pol);
2095         goto out;
2096         }
2097
2098         dst_release(dst);
2099
2100         __skb_queue_head_init(&list);
2101
2102         spin_lock(&pq->hold_queue.lock);
2103         pq->timeout = 0;
2104         skb_queue_splice_init(&pq->hold_queue, &list);
2105         spin_unlock(&pq->hold_queue.lock);
2106
2107         while (!skb_queue_empty(&list)) {
2108                 skb = __skb_dequeue(&list);
2109
2110                 xfrm_decode_session(skb, &fl, skb_dst(skb)->ops->family);
2111                 dst_hold(xfrm_dst_path(skb_dst(skb)));
2112                 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
2113                 if (IS_ERR(dst)) {
2114                         kfree_skb(skb);
2115                         continue;
2116                 }
2117
2118                 nf_reset(skb);
2119                 skb_dst_drop(skb);
2120                 skb_dst_set(skb, dst);
2121
2122                 dst_output(net, skb->sk, skb);
2123         }
2124
2125 out:
2126         xfrm_pol_put(pol);
2127         return;
2128
2129 purge_queue:
2130         pq->timeout = 0;
2131         skb_queue_purge(&pq->hold_queue);
2132         xfrm_pol_put(pol);
2133 }
2134
2135 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2136 {
2137         unsigned long sched_next;
2138         struct dst_entry *dst = skb_dst(skb);
2139         struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2140         struct xfrm_policy *pol = xdst->pols[0];
2141         struct xfrm_policy_queue *pq = &pol->polq;
2142
2143         if (unlikely(skb_fclone_busy(sk, skb))) {
2144                 kfree_skb(skb);
2145                 return 0;
2146         }
2147
2148         if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2149                 kfree_skb(skb);
2150                 return -EAGAIN;
2151         }
2152
2153         skb_dst_force(skb);
2154
2155         spin_lock_bh(&pq->hold_queue.lock);
2156
2157         if (!pq->timeout)
2158                 pq->timeout = XFRM_QUEUE_TMO_MIN;
2159
2160         sched_next = jiffies + pq->timeout;
2161
2162         if (del_timer(&pq->hold_timer)) {
2163                 if (time_before(pq->hold_timer.expires, sched_next))
2164                         sched_next = pq->hold_timer.expires;
2165                 xfrm_pol_put(pol);
2166         }
2167
2168         __skb_queue_tail(&pq->hold_queue, skb);
2169         if (!mod_timer(&pq->hold_timer, sched_next))
2170                 xfrm_pol_hold(pol);
2171
2172         spin_unlock_bh(&pq->hold_queue.lock);
2173
2174         return 0;
2175 }
2176
2177 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
2178                                                  struct xfrm_flo *xflo,
2179                                                  const struct flowi *fl,
2180                                                  int num_xfrms,
2181                                                  u16 family)
2182 {
2183         int err;
2184         struct net_device *dev;
2185         struct dst_entry *dst;
2186         struct dst_entry *dst1;
2187         struct xfrm_dst *xdst;
2188
2189         xdst = xfrm_alloc_dst(net, family);
2190         if (IS_ERR(xdst))
2191                 return xdst;
2192
2193         if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
2194             net->xfrm.sysctl_larval_drop ||
2195             num_xfrms <= 0)
2196                 return xdst;
2197
2198         dst = xflo->dst_orig;
2199         dst1 = &xdst->u.dst;
2200         dst_hold(dst);
2201         xdst->route = dst;
2202
2203         dst_copy_metrics(dst1, dst);
2204
2205         dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2206         dst1->flags |= DST_HOST | DST_XFRM_QUEUE;
2207         dst1->lastuse = jiffies;
2208
2209         dst1->input = dst_discard;
2210         dst1->output = xdst_queue_output;
2211
2212         dst_hold(dst);
2213         xfrm_dst_set_child(xdst, dst);
2214         xdst->path = dst;
2215
2216         xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
2217
2218         err = -ENODEV;
2219         dev = dst->dev;
2220         if (!dev)
2221                 goto free_dst;
2222
2223         err = xfrm_fill_dst(xdst, dev, fl);
2224         if (err)
2225                 goto free_dst;
2226
2227 out:
2228         return xdst;
2229
2230 free_dst:
2231         dst_release(dst1);
2232         xdst = ERR_PTR(err);
2233         goto out;
2234 }
2235
2236 static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
2237                                            const struct flowi *fl,
2238                                            u16 family, u8 dir,
2239                                            struct xfrm_flo *xflo, u32 if_id)
2240 {
2241         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2242         int num_pols = 0, num_xfrms = 0, err;
2243         struct xfrm_dst *xdst;
2244
2245         /* Resolve policies to use if we couldn't get them from
2246          * previous cache entry */
2247         num_pols = 1;
2248         pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
2249         err = xfrm_expand_policies(fl, family, pols,
2250                                            &num_pols, &num_xfrms);
2251         if (err < 0)
2252                 goto inc_error;
2253         if (num_pols == 0)
2254                 return NULL;
2255         if (num_xfrms <= 0)
2256                 goto make_dummy_bundle;
2257
2258         xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
2259                                               xflo->dst_orig);
2260         if (IS_ERR(xdst)) {
2261                 err = PTR_ERR(xdst);
2262                 if (err == -EREMOTE) {
2263                         xfrm_pols_put(pols, num_pols);
2264                         return NULL;
2265                 }
2266
2267                 if (err != -EAGAIN)
2268                         goto error;
2269                 goto make_dummy_bundle;
2270         } else if (xdst == NULL) {
2271                 num_xfrms = 0;
2272                 goto make_dummy_bundle;
2273         }
2274
2275         return xdst;
2276
2277 make_dummy_bundle:
2278         /* We found policies, but there's no bundles to instantiate:
2279          * either because the policy blocks, has no transformations or
2280          * we could not build template (no xfrm_states).*/
2281         xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
2282         if (IS_ERR(xdst)) {
2283                 xfrm_pols_put(pols, num_pols);
2284                 return ERR_CAST(xdst);
2285         }
2286         xdst->num_pols = num_pols;
2287         xdst->num_xfrms = num_xfrms;
2288         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2289
2290         return xdst;
2291
2292 inc_error:
2293         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2294 error:
2295         xfrm_pols_put(pols, num_pols);
2296         return ERR_PTR(err);
2297 }
2298
2299 static struct dst_entry *make_blackhole(struct net *net, u16 family,
2300                                         struct dst_entry *dst_orig)
2301 {
2302         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2303         struct dst_entry *ret;
2304
2305         if (!afinfo) {
2306                 dst_release(dst_orig);
2307                 return ERR_PTR(-EINVAL);
2308         } else {
2309                 ret = afinfo->blackhole_route(net, dst_orig);
2310         }
2311         rcu_read_unlock();
2312
2313         return ret;
2314 }
2315
2316 /* Finds/creates a bundle for given flow and if_id
2317  *
2318  * At the moment we eat a raw IP route. Mostly to speed up lookups
2319  * on interfaces with disabled IPsec.
2320  *
2321  * xfrm_lookup uses an if_id of 0 by default, and is provided for
2322  * compatibility
2323  */
2324 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
2325                                         struct dst_entry *dst_orig,
2326                                         const struct flowi *fl,
2327                                         const struct sock *sk,
2328                                         int flags, u32 if_id)
2329 {
2330         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2331         struct xfrm_dst *xdst;
2332         struct dst_entry *dst, *route;
2333         u16 family = dst_orig->ops->family;
2334         u8 dir = XFRM_POLICY_OUT;
2335         int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
2336
2337         dst = NULL;
2338         xdst = NULL;
2339         route = NULL;
2340
2341         sk = sk_const_to_full_sk(sk);
2342         if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
2343                 num_pols = 1;
2344                 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
2345                                                 if_id);
2346                 err = xfrm_expand_policies(fl, family, pols,
2347                                            &num_pols, &num_xfrms);
2348                 if (err < 0)
2349                         goto dropdst;
2350
2351                 if (num_pols) {
2352                         if (num_xfrms <= 0) {
2353                                 drop_pols = num_pols;
2354                                 goto no_transform;
2355                         }
2356
2357                         xdst = xfrm_resolve_and_create_bundle(
2358                                         pols, num_pols, fl,
2359                                         family, dst_orig);
2360
2361                         if (IS_ERR(xdst)) {
2362                                 xfrm_pols_put(pols, num_pols);
2363                                 err = PTR_ERR(xdst);
2364                                 if (err == -EREMOTE)
2365                                         goto nopol;
2366
2367                                 goto dropdst;
2368                         } else if (xdst == NULL) {
2369                                 num_xfrms = 0;
2370                                 drop_pols = num_pols;
2371                                 goto no_transform;
2372                         }
2373
2374                         route = xdst->route;
2375                 }
2376         }
2377
2378         if (xdst == NULL) {
2379                 struct xfrm_flo xflo;
2380
2381                 xflo.dst_orig = dst_orig;
2382                 xflo.flags = flags;
2383
2384                 /* To accelerate a bit...  */
2385                 if ((dst_orig->flags & DST_NOXFRM) ||
2386                     !net->xfrm.policy_count[XFRM_POLICY_OUT])
2387                         goto nopol;
2388
2389                 xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
2390                 if (xdst == NULL)
2391                         goto nopol;
2392                 if (IS_ERR(xdst)) {
2393                         err = PTR_ERR(xdst);
2394                         goto dropdst;
2395                 }
2396
2397                 num_pols = xdst->num_pols;
2398                 num_xfrms = xdst->num_xfrms;
2399                 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
2400                 route = xdst->route;
2401         }
2402
2403         dst = &xdst->u.dst;
2404         if (route == NULL && num_xfrms > 0) {
2405                 /* The only case when xfrm_bundle_lookup() returns a
2406                  * bundle with null route, is when the template could
2407                  * not be resolved. It means policies are there, but
2408                  * bundle could not be created, since we don't yet
2409                  * have the xfrm_state's. We need to wait for KM to
2410                  * negotiate new SA's or bail out with error.*/
2411                 if (net->xfrm.sysctl_larval_drop) {
2412                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2413                         err = -EREMOTE;
2414                         goto error;
2415                 }
2416
2417                 err = -EAGAIN;
2418
2419                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
2420                 goto error;
2421         }
2422
2423 no_transform:
2424         if (num_pols == 0)
2425                 goto nopol;
2426
2427         if ((flags & XFRM_LOOKUP_ICMP) &&
2428             !(pols[0]->flags & XFRM_POLICY_ICMP)) {
2429                 err = -ENOENT;
2430                 goto error;
2431         }
2432
2433         for (i = 0; i < num_pols; i++)
2434                 pols[i]->curlft.use_time = ktime_get_real_seconds();
2435
2436         if (num_xfrms < 0) {
2437                 /* Prohibit the flow */
2438                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
2439                 err = -EPERM;
2440                 goto error;
2441         } else if (num_xfrms > 0) {
2442                 /* Flow transformed */
2443                 dst_release(dst_orig);
2444         } else {
2445                 /* Flow passes untransformed */
2446                 dst_release(dst);
2447                 dst = dst_orig;
2448         }
2449 ok:
2450         xfrm_pols_put(pols, drop_pols);
2451         if (dst && dst->xfrm &&
2452             dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
2453                 dst->flags |= DST_XFRM_TUNNEL;
2454         return dst;
2455
2456 nopol:
2457         if (!(flags & XFRM_LOOKUP_ICMP)) {
2458                 dst = dst_orig;
2459                 goto ok;
2460         }
2461         err = -ENOENT;
2462 error:
2463         dst_release(dst);
2464 dropdst:
2465         if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
2466                 dst_release(dst_orig);
2467         xfrm_pols_put(pols, drop_pols);
2468         return ERR_PTR(err);
2469 }
2470 EXPORT_SYMBOL(xfrm_lookup_with_ifid);
2471
2472 /* Main function: finds/creates a bundle for given flow.
2473  *
2474  * At the moment we eat a raw IP route. Mostly to speed up lookups
2475  * on interfaces with disabled IPsec.
2476  */
2477 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
2478                               const struct flowi *fl, const struct sock *sk,
2479                               int flags)
2480 {
2481         return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
2482 }
2483 EXPORT_SYMBOL(xfrm_lookup);
2484
2485 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
2486  * Otherwise we may send out blackholed packets.
2487  */
2488 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
2489                                     const struct flowi *fl,
2490                                     const struct sock *sk, int flags)
2491 {
2492         struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
2493                                             flags | XFRM_LOOKUP_QUEUE |
2494                                             XFRM_LOOKUP_KEEP_DST_REF);
2495
2496         if (IS_ERR(dst) && PTR_ERR(dst) == -EREMOTE)
2497                 return make_blackhole(net, dst_orig->ops->family, dst_orig);
2498
2499         if (IS_ERR(dst))
2500                 dst_release(dst_orig);
2501
2502         return dst;
2503 }
2504 EXPORT_SYMBOL(xfrm_lookup_route);
2505
2506 static inline int
2507 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
2508 {
2509         struct xfrm_state *x;
2510
2511         if (!skb->sp || idx < 0 || idx >= skb->sp->len)
2512                 return 0;
2513         x = skb->sp->xvec[idx];
2514         if (!x->type->reject)
2515                 return 0;
2516         return x->type->reject(x, skb, fl);
2517 }
2518
2519 /* When skb is transformed back to its "native" form, we have to
2520  * check policy restrictions. At the moment we make this in maximally
2521  * stupid way. Shame on me. :-) Of course, connected sockets must
2522  * have policy cached at them.
2523  */
2524
2525 static inline int
2526 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
2527               unsigned short family)
2528 {
2529         if (xfrm_state_kern(x))
2530                 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
2531         return  x->id.proto == tmpl->id.proto &&
2532                 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
2533                 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
2534                 x->props.mode == tmpl->mode &&
2535                 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
2536                  !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
2537                 !(x->props.mode != XFRM_MODE_TRANSPORT &&
2538                   xfrm_state_addr_cmp(tmpl, x, family));
2539 }
2540
2541 /*
2542  * 0 or more than 0 is returned when validation is succeeded (either bypass
2543  * because of optional transport mode, or next index of the mathced secpath
2544  * state with the template.
2545  * -1 is returned when no matching template is found.
2546  * Otherwise "-2 - errored_index" is returned.
2547  */
2548 static inline int
2549 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
2550                unsigned short family)
2551 {
2552         int idx = start;
2553
2554         if (tmpl->optional) {
2555                 if (tmpl->mode == XFRM_MODE_TRANSPORT)
2556                         return start;
2557         } else
2558                 start = -1;
2559         for (; idx < sp->len; idx++) {
2560                 if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
2561                         return ++idx;
2562                 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2563                         if (start == -1)
2564                                 start = -2-idx;
2565                         break;
2566                 }
2567         }
2568         return start;
2569 }
2570
2571 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
2572                           unsigned int family, int reverse)
2573 {
2574         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2575         int err;
2576
2577         if (unlikely(afinfo == NULL))
2578                 return -EAFNOSUPPORT;
2579
2580         afinfo->decode_session(skb, fl, reverse);
2581
2582         err = security_xfrm_decode_session(skb, &fl->flowi_secid);
2583         rcu_read_unlock();
2584         return err;
2585 }
2586 EXPORT_SYMBOL(__xfrm_decode_session);
2587
2588 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
2589 {
2590         for (; k < sp->len; k++) {
2591                 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
2592                         *idxp = k;
2593                         return 1;
2594                 }
2595         }
2596
2597         return 0;
2598 }
2599
2600 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
2601                         unsigned short family)
2602 {
2603         struct net *net = dev_net(skb->dev);
2604         struct xfrm_policy *pol;
2605         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2606         int npols = 0;
2607         int xfrm_nr;
2608         int pi;
2609         int reverse;
2610         struct flowi fl;
2611         int xerr_idx = -1;
2612         const struct xfrm_if_cb *ifcb;
2613         struct xfrm_if *xi;
2614         u32 if_id = 0;
2615
2616         rcu_read_lock();
2617         ifcb = xfrm_if_get_cb();
2618
2619         if (ifcb) {
2620                 xi = ifcb->decode_session(skb);
2621                 if (xi)
2622                         if_id = xi->p.if_id;
2623         }
2624         rcu_read_unlock();
2625
2626         reverse = dir & ~XFRM_POLICY_MASK;
2627         dir &= XFRM_POLICY_MASK;
2628
2629         if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
2630                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
2631                 return 0;
2632         }
2633
2634         nf_nat_decode_session(skb, &fl, family);
2635
2636         /* First, check used SA against their selectors. */
2637         if (skb->sp) {
2638                 int i;
2639
2640                 for (i = skb->sp->len-1; i >= 0; i--) {
2641                         struct xfrm_state *x = skb->sp->xvec[i];
2642                         if (!xfrm_selector_match(&x->sel, &fl, family)) {
2643                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
2644                                 return 0;
2645                         }
2646                 }
2647         }
2648
2649         pol = NULL;
2650         sk = sk_to_full_sk(sk);
2651         if (sk && sk->sk_policy[dir]) {
2652                 pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
2653                 if (IS_ERR(pol)) {
2654                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2655                         return 0;
2656                 }
2657         }
2658
2659         if (!pol)
2660                 pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
2661
2662         if (IS_ERR(pol)) {
2663                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2664                 return 0;
2665         }
2666
2667         if (!pol) {
2668                 if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
2669                         xfrm_secpath_reject(xerr_idx, skb, &fl);
2670                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
2671                         return 0;
2672                 }
2673                 return 1;
2674         }
2675
2676         pol->curlft.use_time = ktime_get_real_seconds();
2677
2678         pols[0] = pol;
2679         npols++;
2680 #ifdef CONFIG_XFRM_SUB_POLICY
2681         if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2682                 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
2683                                                     &fl, family,
2684                                                     XFRM_POLICY_IN, if_id);
2685                 if (pols[1]) {
2686                         if (IS_ERR(pols[1])) {
2687                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2688                                 return 0;
2689                         }
2690                         pols[1]->curlft.use_time = ktime_get_real_seconds();
2691                         npols++;
2692                 }
2693         }
2694 #endif
2695
2696         if (pol->action == XFRM_POLICY_ALLOW) {
2697                 struct sec_path *sp;
2698                 static struct sec_path dummy;
2699                 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
2700                 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
2701                 struct xfrm_tmpl **tpp = tp;
2702                 int ti = 0;
2703                 int i, k;
2704
2705                 if ((sp = skb->sp) == NULL)
2706                         sp = &dummy;
2707
2708                 for (pi = 0; pi < npols; pi++) {
2709                         if (pols[pi] != pol &&
2710                             pols[pi]->action != XFRM_POLICY_ALLOW) {
2711                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2712                                 goto reject;
2713                         }
2714                         if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2715                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
2716                                 goto reject_error;
2717                         }
2718                         for (i = 0; i < pols[pi]->xfrm_nr; i++)
2719                                 tpp[ti++] = &pols[pi]->xfrm_vec[i];
2720                 }
2721                 xfrm_nr = ti;
2722                 if (npols > 1) {
2723                         xfrm_tmpl_sort(stp, tpp, xfrm_nr, family, net);
2724                         tpp = stp;
2725                 }
2726
2727                 /* For each tunnel xfrm, find the first matching tmpl.
2728                  * For each tmpl before that, find corresponding xfrm.
2729                  * Order is _important_. Later we will implement
2730                  * some barriers, but at the moment barriers
2731                  * are implied between each two transformations.
2732                  */
2733                 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2734                         k = xfrm_policy_ok(tpp[i], sp, k, family);
2735                         if (k < 0) {
2736                                 if (k < -1)
2737                                         /* "-2 - errored_index" returned */
2738                                         xerr_idx = -(2+k);
2739                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2740                                 goto reject;
2741                         }
2742                 }
2743
2744                 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2745                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2746                         goto reject;
2747                 }
2748
2749                 xfrm_pols_put(pols, npols);
2750                 return 1;
2751         }
2752         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2753
2754 reject:
2755         xfrm_secpath_reject(xerr_idx, skb, &fl);
2756 reject_error:
2757         xfrm_pols_put(pols, npols);
2758         return 0;
2759 }
2760 EXPORT_SYMBOL(__xfrm_policy_check);
2761
2762 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2763 {
2764         struct net *net = dev_net(skb->dev);
2765         struct flowi fl;
2766         struct dst_entry *dst;
2767         int res = 1;
2768
2769         if (xfrm_decode_session(skb, &fl, family) < 0) {
2770                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2771                 return 0;
2772         }
2773
2774         skb_dst_force(skb);
2775         if (!skb_dst(skb)) {
2776                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2777                 return 0;
2778         }
2779
2780         dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
2781         if (IS_ERR(dst)) {
2782                 res = 0;
2783                 dst = NULL;
2784         }
2785         skb_dst_set(skb, dst);
2786         return res;
2787 }
2788 EXPORT_SYMBOL(__xfrm_route_forward);
2789
2790 /* Optimize later using cookies and generation ids. */
2791
2792 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2793 {
2794         /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2795          * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2796          * get validated by dst_ops->check on every use.  We do this
2797          * because when a normal route referenced by an XFRM dst is
2798          * obsoleted we do not go looking around for all parent
2799          * referencing XFRM dsts so that we can invalidate them.  It
2800          * is just too much work.  Instead we make the checks here on
2801          * every use.  For example:
2802          *
2803          *      XFRM dst A --> IPv4 dst X
2804          *
2805          * X is the "xdst->route" of A (X is also the "dst->path" of A
2806          * in this example).  If X is marked obsolete, "A" will not
2807          * notice.  That's what we are validating here via the
2808          * stale_bundle() check.
2809          *
2810          * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
2811          * be marked on it.
2812          * This will force stale_bundle() to fail on any xdst bundle with
2813          * this dst linked in it.
2814          */
2815         if (dst->obsolete < 0 && !stale_bundle(dst))
2816                 return dst;
2817
2818         return NULL;
2819 }
2820
2821 static int stale_bundle(struct dst_entry *dst)
2822 {
2823         return !xfrm_bundle_ok((struct xfrm_dst *)dst);
2824 }
2825
2826 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
2827 {
2828         while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
2829                 dst->dev = dev_net(dev)->loopback_dev;
2830                 dev_hold(dst->dev);
2831                 dev_put(dev);
2832         }
2833 }
2834 EXPORT_SYMBOL(xfrm_dst_ifdown);
2835
2836 static void xfrm_link_failure(struct sk_buff *skb)
2837 {
2838         /* Impossible. Such dst must be popped before reaches point of failure. */
2839 }
2840
2841 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2842 {
2843         if (dst) {
2844                 if (dst->obsolete) {
2845                         dst_release(dst);
2846                         dst = NULL;
2847                 }
2848         }
2849         return dst;
2850 }
2851
2852 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
2853 {
2854         while (nr--) {
2855                 struct xfrm_dst *xdst = bundle[nr];
2856                 u32 pmtu, route_mtu_cached;
2857                 struct dst_entry *dst;
2858
2859                 dst = &xdst->u.dst;
2860                 pmtu = dst_mtu(xfrm_dst_child(dst));
2861                 xdst->child_mtu_cached = pmtu;
2862
2863                 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2864
2865                 route_mtu_cached = dst_mtu(xdst->route);
2866                 xdst->route_mtu_cached = route_mtu_cached;
2867
2868                 if (pmtu > route_mtu_cached)
2869                         pmtu = route_mtu_cached;
2870
2871                 dst_metric_set(dst, RTAX_MTU, pmtu);
2872         }
2873 }
2874
2875 /* Check that the bundle accepts the flow and its components are
2876  * still valid.
2877  */
2878
2879 static int xfrm_bundle_ok(struct xfrm_dst *first)
2880 {
2881         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2882         struct dst_entry *dst = &first->u.dst;
2883         struct xfrm_dst *xdst;
2884         int start_from, nr;
2885         u32 mtu;
2886
2887         if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
2888             (dst->dev && !netif_running(dst->dev)))
2889                 return 0;
2890
2891         if (dst->flags & DST_XFRM_QUEUE)
2892                 return 1;
2893
2894         start_from = nr = 0;
2895         do {
2896                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2897
2898                 if (dst->xfrm->km.state != XFRM_STATE_VALID)
2899                         return 0;
2900                 if (xdst->xfrm_genid != dst->xfrm->genid)
2901                         return 0;
2902                 if (xdst->num_pols > 0 &&
2903                     xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
2904                         return 0;
2905
2906                 bundle[nr++] = xdst;
2907
2908                 mtu = dst_mtu(xfrm_dst_child(dst));
2909                 if (xdst->child_mtu_cached != mtu) {
2910                         start_from = nr;
2911                         xdst->child_mtu_cached = mtu;
2912                 }
2913
2914                 if (!dst_check(xdst->route, xdst->route_cookie))
2915                         return 0;
2916                 mtu = dst_mtu(xdst->route);
2917                 if (xdst->route_mtu_cached != mtu) {
2918                         start_from = nr;
2919                         xdst->route_mtu_cached = mtu;
2920                 }
2921
2922                 dst = xfrm_dst_child(dst);
2923         } while (dst->xfrm);
2924
2925         if (likely(!start_from))
2926                 return 1;
2927
2928         xdst = bundle[start_from - 1];
2929         mtu = xdst->child_mtu_cached;
2930         while (start_from--) {
2931                 dst = &xdst->u.dst;
2932
2933                 mtu = xfrm_state_mtu(dst->xfrm, mtu);
2934                 if (mtu > xdst->route_mtu_cached)
2935                         mtu = xdst->route_mtu_cached;
2936                 dst_metric_set(dst, RTAX_MTU, mtu);
2937                 if (!start_from)
2938                         break;
2939
2940                 xdst = bundle[start_from - 1];
2941                 xdst->child_mtu_cached = mtu;
2942         }
2943
2944         return 1;
2945 }
2946
2947 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2948 {
2949         return dst_metric_advmss(xfrm_dst_path(dst));
2950 }
2951
2952 static unsigned int xfrm_mtu(const struct dst_entry *dst)
2953 {
2954         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2955
2956         return mtu ? : dst_mtu(xfrm_dst_path(dst));
2957 }
2958
2959 static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
2960                                         const void *daddr)
2961 {
2962         while (dst->xfrm) {
2963                 const struct xfrm_state *xfrm = dst->xfrm;
2964
2965                 dst = xfrm_dst_child(dst);
2966
2967                 if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
2968                         continue;
2969                 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
2970                         daddr = xfrm->coaddr;
2971                 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
2972                         daddr = &xfrm->id.daddr;
2973         }
2974         return daddr;
2975 }
2976
2977 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
2978                                            struct sk_buff *skb,
2979                                            const void *daddr)
2980 {
2981         const struct dst_entry *path = xfrm_dst_path(dst);
2982
2983         if (!skb)
2984                 daddr = xfrm_get_dst_nexthop(dst, daddr);
2985         return path->ops->neigh_lookup(path, skb, daddr);
2986 }
2987
2988 static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
2989 {
2990         const struct dst_entry *path = xfrm_dst_path(dst);
2991
2992         daddr = xfrm_get_dst_nexthop(dst, daddr);
2993         path->ops->confirm_neigh(path, daddr);
2994 }
2995
2996 int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
2997 {
2998         int err = 0;
2999
3000         if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
3001                 return -EAFNOSUPPORT;
3002
3003         spin_lock(&xfrm_policy_afinfo_lock);
3004         if (unlikely(xfrm_policy_afinfo[family] != NULL))
3005                 err = -EEXIST;
3006         else {
3007                 struct dst_ops *dst_ops = afinfo->dst_ops;
3008                 if (likely(dst_ops->kmem_cachep == NULL))
3009                         dst_ops->kmem_cachep = xfrm_dst_cache;
3010                 if (likely(dst_ops->check == NULL))
3011                         dst_ops->check = xfrm_dst_check;
3012                 if (likely(dst_ops->default_advmss == NULL))
3013                         dst_ops->default_advmss = xfrm_default_advmss;
3014                 if (likely(dst_ops->mtu == NULL))
3015                         dst_ops->mtu = xfrm_mtu;
3016                 if (likely(dst_ops->negative_advice == NULL))
3017                         dst_ops->negative_advice = xfrm_negative_advice;
3018                 if (likely(dst_ops->link_failure == NULL))
3019                         dst_ops->link_failure = xfrm_link_failure;
3020                 if (likely(dst_ops->neigh_lookup == NULL))
3021                         dst_ops->neigh_lookup = xfrm_neigh_lookup;
3022                 if (likely(!dst_ops->confirm_neigh))
3023                         dst_ops->confirm_neigh = xfrm_confirm_neigh;
3024                 rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
3025         }
3026         spin_unlock(&xfrm_policy_afinfo_lock);
3027
3028         return err;
3029 }
3030 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
3031
3032 void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
3033 {
3034         struct dst_ops *dst_ops = afinfo->dst_ops;
3035         int i;
3036
3037         for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
3038                 if (xfrm_policy_afinfo[i] != afinfo)
3039                         continue;
3040                 RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
3041                 break;
3042         }
3043
3044         synchronize_rcu();
3045
3046         dst_ops->kmem_cachep = NULL;
3047         dst_ops->check = NULL;
3048         dst_ops->negative_advice = NULL;
3049         dst_ops->link_failure = NULL;
3050 }
3051 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
3052
3053 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
3054 {
3055         spin_lock(&xfrm_if_cb_lock);
3056         rcu_assign_pointer(xfrm_if_cb, ifcb);
3057         spin_unlock(&xfrm_if_cb_lock);
3058 }
3059 EXPORT_SYMBOL(xfrm_if_register_cb);
3060
3061 void xfrm_if_unregister_cb(void)
3062 {
3063         RCU_INIT_POINTER(xfrm_if_cb, NULL);
3064         synchronize_rcu();
3065 }
3066 EXPORT_SYMBOL(xfrm_if_unregister_cb);
3067
3068 #ifdef CONFIG_XFRM_STATISTICS
3069 static int __net_init xfrm_statistics_init(struct net *net)
3070 {
3071         int rv;
3072         net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
3073         if (!net->mib.xfrm_statistics)
3074                 return -ENOMEM;
3075         rv = xfrm_proc_init(net);
3076         if (rv < 0)
3077                 free_percpu(net->mib.xfrm_statistics);
3078         return rv;
3079 }
3080
3081 static void xfrm_statistics_fini(struct net *net)
3082 {
3083         xfrm_proc_fini(net);
3084         free_percpu(net->mib.xfrm_statistics);
3085 }
3086 #else
3087 static int __net_init xfrm_statistics_init(struct net *net)
3088 {
3089         return 0;
3090 }
3091
3092 static void xfrm_statistics_fini(struct net *net)
3093 {
3094 }
3095 #endif
3096
3097 static int __net_init xfrm_policy_init(struct net *net)
3098 {
3099         unsigned int hmask, sz;
3100         int dir, err;
3101
3102         if (net_eq(net, &init_net)) {
3103                 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
3104                                            sizeof(struct xfrm_dst),
3105                                            0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
3106                                            NULL);
3107                 err = rhashtable_init(&xfrm_policy_inexact_table,
3108                                       &xfrm_pol_inexact_params);
3109                 BUG_ON(err);
3110         }
3111
3112         hmask = 8 - 1;
3113         sz = (hmask+1) * sizeof(struct hlist_head);
3114
3115         net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
3116         if (!net->xfrm.policy_byidx)
3117                 goto out_byidx;
3118         net->xfrm.policy_idx_hmask = hmask;
3119
3120         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
3121                 struct xfrm_policy_hash *htab;
3122
3123                 net->xfrm.policy_count[dir] = 0;
3124                 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
3125                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
3126
3127                 htab = &net->xfrm.policy_bydst[dir];
3128                 htab->table = xfrm_hash_alloc(sz);
3129                 if (!htab->table)
3130                         goto out_bydst;
3131                 htab->hmask = hmask;
3132                 htab->dbits4 = 32;
3133                 htab->sbits4 = 32;
3134                 htab->dbits6 = 128;
3135                 htab->sbits6 = 128;
3136         }
3137         net->xfrm.policy_hthresh.lbits4 = 32;
3138         net->xfrm.policy_hthresh.rbits4 = 32;
3139         net->xfrm.policy_hthresh.lbits6 = 128;
3140         net->xfrm.policy_hthresh.rbits6 = 128;
3141
3142         seqlock_init(&net->xfrm.policy_hthresh.lock);
3143
3144         INIT_LIST_HEAD(&net->xfrm.policy_all);
3145         INIT_LIST_HEAD(&net->xfrm.inexact_bins);
3146         INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
3147         INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
3148         return 0;
3149
3150 out_bydst:
3151         for (dir--; dir >= 0; dir--) {
3152                 struct xfrm_policy_hash *htab;
3153
3154                 htab = &net->xfrm.policy_bydst[dir];
3155                 xfrm_hash_free(htab->table, sz);
3156         }
3157         xfrm_hash_free(net->xfrm.policy_byidx, sz);
3158 out_byidx:
3159         return -ENOMEM;
3160 }
3161
3162 static void xfrm_policy_fini(struct net *net)
3163 {
3164         struct xfrm_pol_inexact_bin *bin, *tmp;
3165         unsigned int sz;
3166         int dir;
3167
3168         flush_work(&net->xfrm.policy_hash_work);
3169 #ifdef CONFIG_XFRM_SUB_POLICY
3170         xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
3171 #endif
3172         xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
3173
3174         WARN_ON(!list_empty(&net->xfrm.policy_all));
3175
3176         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
3177                 struct xfrm_policy_hash *htab;
3178
3179                 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
3180
3181                 htab = &net->xfrm.policy_bydst[dir];
3182                 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
3183                 WARN_ON(!hlist_empty(htab->table));
3184                 xfrm_hash_free(htab->table, sz);
3185         }
3186
3187         sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
3188         WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
3189         xfrm_hash_free(net->xfrm.policy_byidx, sz);
3190
3191         list_for_each_entry_safe(bin, tmp, &net->xfrm.inexact_bins,
3192                                  inexact_bins) {
3193                 WARN_ON(!hlist_empty(&bin->hhead));
3194                 xfrm_policy_inexact_delete_bin(net, bin);
3195         }
3196 }
3197
3198 static int __net_init xfrm_net_init(struct net *net)
3199 {
3200         int rv;
3201
3202         /* Initialize the per-net locks here */
3203         spin_lock_init(&net->xfrm.xfrm_state_lock);
3204         spin_lock_init(&net->xfrm.xfrm_policy_lock);
3205         mutex_init(&net->xfrm.xfrm_cfg_mutex);
3206
3207         rv = xfrm_statistics_init(net);
3208         if (rv < 0)
3209                 goto out_statistics;
3210         rv = xfrm_state_init(net);
3211         if (rv < 0)
3212                 goto out_state;
3213         rv = xfrm_policy_init(net);
3214         if (rv < 0)
3215                 goto out_policy;
3216         rv = xfrm_sysctl_init(net);
3217         if (rv < 0)
3218                 goto out_sysctl;
3219
3220         return 0;
3221
3222 out_sysctl:
3223         xfrm_policy_fini(net);
3224 out_policy:
3225         xfrm_state_fini(net);
3226 out_state:
3227         xfrm_statistics_fini(net);
3228 out_statistics:
3229         return rv;
3230 }
3231
3232 static void __net_exit xfrm_net_exit(struct net *net)
3233 {
3234         xfrm_sysctl_fini(net);
3235         xfrm_policy_fini(net);
3236         xfrm_state_fini(net);
3237         xfrm_statistics_fini(net);
3238 }
3239
3240 static struct pernet_operations __net_initdata xfrm_net_ops = {
3241         .init = xfrm_net_init,
3242         .exit = xfrm_net_exit,
3243 };
3244
3245 void __init xfrm_init(void)
3246 {
3247         register_pernet_subsys(&xfrm_net_ops);
3248         xfrm_dev_init();
3249         seqcount_init(&xfrm_policy_hash_generation);
3250         xfrm_input_init();
3251
3252         RCU_INIT_POINTER(xfrm_if_cb, NULL);
3253         synchronize_rcu();
3254 }
3255
3256 #ifdef CONFIG_AUDITSYSCALL
3257 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
3258                                          struct audit_buffer *audit_buf)
3259 {
3260         struct xfrm_sec_ctx *ctx = xp->security;
3261         struct xfrm_selector *sel = &xp->selector;
3262
3263         if (ctx)
3264                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
3265                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
3266
3267         switch (sel->family) {
3268         case AF_INET:
3269                 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
3270                 if (sel->prefixlen_s != 32)
3271                         audit_log_format(audit_buf, " src_prefixlen=%d",
3272                                          sel->prefixlen_s);
3273                 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
3274                 if (sel->prefixlen_d != 32)
3275                         audit_log_format(audit_buf, " dst_prefixlen=%d",
3276                                          sel->prefixlen_d);
3277                 break;
3278         case AF_INET6:
3279                 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
3280                 if (sel->prefixlen_s != 128)
3281                         audit_log_format(audit_buf, " src_prefixlen=%d",
3282                                          sel->prefixlen_s);
3283                 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
3284                 if (sel->prefixlen_d != 128)
3285                         audit_log_format(audit_buf, " dst_prefixlen=%d",
3286                                          sel->prefixlen_d);
3287                 break;
3288         }
3289 }
3290
3291 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
3292 {
3293         struct audit_buffer *audit_buf;
3294
3295         audit_buf = xfrm_audit_start("SPD-add");
3296         if (audit_buf == NULL)
3297                 return;
3298         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3299         audit_log_format(audit_buf, " res=%u", result);
3300         xfrm_audit_common_policyinfo(xp, audit_buf);
3301         audit_log_end(audit_buf);
3302 }
3303 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
3304
3305 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
3306                               bool task_valid)
3307 {
3308         struct audit_buffer *audit_buf;
3309
3310         audit_buf = xfrm_audit_start("SPD-delete");
3311         if (audit_buf == NULL)
3312                 return;
3313         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3314         audit_log_format(audit_buf, " res=%u", result);
3315         xfrm_audit_common_policyinfo(xp, audit_buf);
3316         audit_log_end(audit_buf);
3317 }
3318 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
3319 #endif
3320
3321 #ifdef CONFIG_XFRM_MIGRATE
3322 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
3323                                         const struct xfrm_selector *sel_tgt)
3324 {
3325         if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
3326                 if (sel_tgt->family == sel_cmp->family &&
3327                     xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
3328                                     sel_cmp->family) &&
3329                     xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
3330                                     sel_cmp->family) &&
3331                     sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
3332                     sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
3333                         return true;
3334                 }
3335         } else {
3336                 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
3337                         return true;
3338                 }
3339         }
3340         return false;
3341 }
3342
3343 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
3344                                                     u8 dir, u8 type, struct net *net)
3345 {
3346         struct xfrm_policy *pol, *ret = NULL;
3347         struct hlist_head *chain;
3348         u32 priority = ~0U;
3349
3350         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
3351         chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
3352         hlist_for_each_entry(pol, chain, bydst) {
3353                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3354                     pol->type == type) {
3355                         ret = pol;
3356                         priority = ret->priority;
3357                         break;
3358                 }
3359         }
3360         chain = &net->xfrm.policy_inexact[dir];
3361         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
3362                 if ((pol->priority >= priority) && ret)
3363                         break;
3364
3365                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
3366                     pol->type == type) {
3367                         ret = pol;
3368                         break;
3369                 }
3370         }
3371
3372         xfrm_pol_hold(ret);
3373
3374         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
3375
3376         return ret;
3377 }
3378
3379 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
3380 {
3381         int match = 0;
3382
3383         if (t->mode == m->mode && t->id.proto == m->proto &&
3384             (m->reqid == 0 || t->reqid == m->reqid)) {
3385                 switch (t->mode) {
3386                 case XFRM_MODE_TUNNEL:
3387                 case XFRM_MODE_BEET:
3388                         if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
3389                                             m->old_family) &&
3390                             xfrm_addr_equal(&t->saddr, &m->old_saddr,
3391                                             m->old_family)) {
3392                                 match = 1;
3393                         }
3394                         break;
3395                 case XFRM_MODE_TRANSPORT:
3396                         /* in case of transport mode, template does not store
3397                            any IP addresses, hence we just compare mode and
3398                            protocol */
3399                         match = 1;
3400                         break;
3401                 default:
3402                         break;
3403                 }
3404         }
3405         return match;
3406 }
3407
3408 /* update endpoint address(es) of template(s) */
3409 static int xfrm_policy_migrate(struct xfrm_policy *pol,
3410                                struct xfrm_migrate *m, int num_migrate)
3411 {
3412         struct xfrm_migrate *mp;
3413         int i, j, n = 0;
3414
3415         write_lock_bh(&pol->lock);
3416         if (unlikely(pol->walk.dead)) {
3417                 /* target policy has been deleted */
3418                 write_unlock_bh(&pol->lock);
3419                 return -ENOENT;
3420         }
3421
3422         for (i = 0; i < pol->xfrm_nr; i++) {
3423                 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
3424                         if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
3425                                 continue;
3426                         n++;
3427                         if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
3428                             pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
3429                                 continue;
3430                         /* update endpoints */
3431                         memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
3432                                sizeof(pol->xfrm_vec[i].id.daddr));
3433                         memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
3434                                sizeof(pol->xfrm_vec[i].saddr));
3435                         pol->xfrm_vec[i].encap_family = mp->new_family;
3436                         /* flush bundles */
3437                         atomic_inc(&pol->genid);
3438                 }
3439         }
3440
3441         write_unlock_bh(&pol->lock);
3442
3443         if (!n)
3444                 return -ENODATA;
3445
3446         return 0;
3447 }
3448
3449 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
3450 {
3451         int i, j;
3452
3453         if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
3454                 return -EINVAL;
3455
3456         for (i = 0; i < num_migrate; i++) {
3457                 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
3458                     xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
3459                         return -EINVAL;
3460
3461                 /* check if there is any duplicated entry */
3462                 for (j = i + 1; j < num_migrate; j++) {
3463                         if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
3464                                     sizeof(m[i].old_daddr)) &&
3465                             !memcmp(&m[i].old_saddr, &m[j].old_saddr,
3466                                     sizeof(m[i].old_saddr)) &&
3467                             m[i].proto == m[j].proto &&
3468                             m[i].mode == m[j].mode &&
3469                             m[i].reqid == m[j].reqid &&
3470                             m[i].old_family == m[j].old_family)
3471                                 return -EINVAL;
3472                 }
3473         }
3474
3475         return 0;
3476 }
3477
3478 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
3479                  struct xfrm_migrate *m, int num_migrate,
3480                  struct xfrm_kmaddress *k, struct net *net,
3481                  struct xfrm_encap_tmpl *encap)
3482 {
3483         int i, err, nx_cur = 0, nx_new = 0;
3484         struct xfrm_policy *pol = NULL;
3485         struct xfrm_state *x, *xc;
3486         struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
3487         struct xfrm_state *x_new[XFRM_MAX_DEPTH];
3488         struct xfrm_migrate *mp;
3489
3490         /* Stage 0 - sanity checks */
3491         if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
3492                 goto out;
3493
3494         if (dir >= XFRM_POLICY_MAX) {
3495                 err = -EINVAL;
3496                 goto out;
3497         }
3498
3499         /* Stage 1 - find policy */
3500         if ((pol = xfrm_migrate_policy_find(sel, dir, type, net)) == NULL) {
3501                 err = -ENOENT;
3502                 goto out;
3503         }
3504
3505         /* Stage 2 - find and update state(s) */
3506         for (i = 0, mp = m; i < num_migrate; i++, mp++) {
3507                 if ((x = xfrm_migrate_state_find(mp, net))) {
3508                         x_cur[nx_cur] = x;
3509                         nx_cur++;
3510                         xc = xfrm_state_migrate(x, mp, encap);
3511                         if (xc) {
3512                                 x_new[nx_new] = xc;
3513                                 nx_new++;
3514                         } else {
3515                                 err = -ENODATA;
3516                                 goto restore_state;
3517                         }
3518                 }
3519         }
3520
3521         /* Stage 3 - update policy */
3522         if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
3523                 goto restore_state;
3524
3525         /* Stage 4 - delete old state(s) */
3526         if (nx_cur) {
3527                 xfrm_states_put(x_cur, nx_cur);
3528                 xfrm_states_delete(x_cur, nx_cur);
3529         }
3530
3531         /* Stage 5 - announce */
3532         km_migrate(sel, dir, type, m, num_migrate, k, encap);
3533
3534         xfrm_pol_put(pol);
3535
3536         return 0;
3537 out:
3538         return err;
3539
3540 restore_state:
3541         if (pol)
3542                 xfrm_pol_put(pol);
3543         if (nx_cur)
3544                 xfrm_states_put(x_cur, nx_cur);
3545         if (nx_new)
3546                 xfrm_states_delete(x_new, nx_new);
3547
3548         return err;
3549 }
3550 EXPORT_SYMBOL(xfrm_migrate);
3551 #endif