net/ipv4/nexthop.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Generic nexthop implementation
   3  *
   4  * Copyright (c) 2017-19 Cumulus Networks
   5  * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
   6  */
   7
   8 #include <linux/nexthop.h>
   9 #include <linux/rtnetlink.h>
  10 #include <linux/slab.h>
  11 #include <net/arp.h>
  12 #include <net/ipv6_stubs.h>
  13 #include <net/lwtunnel.h>
  14 #include <net/ndisc.h>
  15 #include <net/nexthop.h>
  16 #include <net/route.h>
  17 #include <net/sock.h>
  18
  19 static void remove_nexthop(struct net *net, struct nexthop *nh,
  20                            struct nl_info *nlinfo);
  21
  22 #define NH_DEV_HASHBITS  8
  23 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
  24
  25 static const struct nla_policy rtm_nh_policy_new[] = {
  26         [NHA_ID]                = { .type = NLA_U32 },
  27         [NHA_GROUP]             = { .type = NLA_BINARY },
  28         [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
  29         [NHA_BLACKHOLE]         = { .type = NLA_FLAG },
  30         [NHA_OIF]               = { .type = NLA_U32 },
  31         [NHA_GATEWAY]           = { .type = NLA_BINARY },
  32         [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
  33         [NHA_ENCAP]             = { .type = NLA_NESTED },
  34         [NHA_FDB]               = { .type = NLA_FLAG },
  35 };
  36
  37 static const struct nla_policy rtm_nh_policy_get[] = {
  38         [NHA_ID]                = { .type = NLA_U32 },
  39 };
  40
  41 static const struct nla_policy rtm_nh_policy_dump[] = {
  42         [NHA_OIF]               = { .type = NLA_U32 },
  43         [NHA_GROUPS]            = { .type = NLA_FLAG },
  44         [NHA_MASTER]            = { .type = NLA_U32 },
  45         [NHA_FDB]               = { .type = NLA_FLAG },
  46 };
  47
  48 static bool nexthop_notifiers_is_empty(struct net *net)
  49 {
  50         return !net->nexthop.notifier_chain.head;
  51 }
  52
  53 static void
  54 __nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
  55                                const struct nexthop *nh)
  56 {
  57         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
  58
  59         nh_info->dev = nhi->fib_nhc.nhc_dev;
  60         nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
  61         if (nh_info->gw_family == AF_INET)
  62                 nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
  63         else if (nh_info->gw_family == AF_INET6)
  64                 nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;
  65
  66         nh_info->is_reject = nhi->reject_nh;
  67         nh_info->is_fdb = nhi->fdb_nh;
  68         nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
  69 }
  70
  71 static int nh_notifier_single_info_init(struct nh_notifier_info *info,
  72                                         const struct nexthop *nh)
  73 {
  74         info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
  75         info->nh = kzalloc(sizeof(*info->nh), GFP_KERNEL);
  76         if (!info->nh)
  77                 return -ENOMEM;
  78
  79         __nh_notifier_single_info_init(info->nh, nh);
  80
  81         return 0;
  82 }
  83
  84 static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
  85 {
  86         kfree(info->nh);
  87 }
  88
  89 static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
  90                                      const struct nexthop *nh)
  91 {
  92         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
  93         u16 num_nh = nhg->num_nh;
  94         int i;
  95
  96         info->type = NH_NOTIFIER_INFO_TYPE_GRP;
  97         info->nh_grp = kzalloc(struct_size(info->nh_grp, nh_entries, num_nh),
  98                                GFP_KERNEL);
  99         if (!info->nh_grp)
 100                 return -ENOMEM;
 101
 102         info->nh_grp->num_nh = num_nh;
 103         info->nh_grp->is_fdb = nhg->fdb_nh;
 104
 105         for (i = 0; i < num_nh; i++) {
 106                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 107
 108                 info->nh_grp->nh_entries[i].id = nhge->nh->id;
 109                 info->nh_grp->nh_entries[i].weight = nhge->weight;
 110                 __nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
 111                                                nhge->nh);
 112         }
 113
 114         return 0;
 115 }
 116
 117 static void nh_notifier_grp_info_fini(struct nh_notifier_info *info)
 118 {
 119         kfree(info->nh_grp);
 120 }
 121
 122 static int nh_notifier_info_init(struct nh_notifier_info *info,
 123                                  const struct nexthop *nh)
 124 {
 125         info->id = nh->id;
 126
 127         if (nh->is_group)
 128                 return nh_notifier_grp_info_init(info, nh);
 129         else
 130                 return nh_notifier_single_info_init(info, nh);
 131 }
 132
 133 static void nh_notifier_info_fini(struct nh_notifier_info *info,
 134                                   const struct nexthop *nh)
 135 {
 136         if (nh->is_group)
 137                 nh_notifier_grp_info_fini(info);
 138         else
 139                 nh_notifier_single_info_fini(info);
 140 }
 141
 142 static int call_nexthop_notifiers(struct net *net,
 143                                   enum nexthop_event_type event_type,
 144                                   struct nexthop *nh,
 145                                   struct netlink_ext_ack *extack)
 146 {
 147         struct nh_notifier_info info = {
 148                 .net = net,
 149                 .extack = extack,
 150         };
 151         int err;
 152
 153         ASSERT_RTNL();
 154
 155         if (nexthop_notifiers_is_empty(net))
 156                 return 0;
 157
 158         err = nh_notifier_info_init(&info, nh);
 159         if (err) {
 160                 NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
 161                 return err;
 162         }
 163
 164         err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
 165                                            event_type, &info);
 166         nh_notifier_info_fini(&info, nh);
 167
 168         return notifier_to_errno(err);
 169 }
 170
 171 static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
 172                                  enum nexthop_event_type event_type,
 173                                  struct nexthop *nh,
 174                                  struct netlink_ext_ack *extack)
 175 {
 176         struct nh_notifier_info info = {
 177                 .net = net,
 178                 .extack = extack,
 179         };
 180         int err;
 181
 182         err = nh_notifier_info_init(&info, nh);
 183         if (err)
 184                 return err;
 185
 186         err = nb->notifier_call(nb, event_type, &info);
 187         nh_notifier_info_fini(&info, nh);
 188
 189         return notifier_to_errno(err);
 190 }
 191
 192 static unsigned int nh_dev_hashfn(unsigned int val)
 193 {
 194         unsigned int mask = NH_DEV_HASHSIZE - 1;
 195
 196         return (val ^
 197                 (val >> NH_DEV_HASHBITS) ^
 198                 (val >> (NH_DEV_HASHBITS * 2))) & mask;
 199 }
 200
 201 static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
 202 {
 203         struct net_device *dev = nhi->fib_nhc.nhc_dev;
 204         struct hlist_head *head;
 205         unsigned int hash;
 206
 207         WARN_ON(!dev);
 208
 209         hash = nh_dev_hashfn(dev->ifindex);
 210         head = &net->nexthop.devhash[hash];
 211         hlist_add_head(&nhi->dev_hash, head);
 212 }
 213
 214 static void nexthop_free_group(struct nexthop *nh)
 215 {
 216         struct nh_group *nhg;
 217         int i;
 218
 219         nhg = rcu_dereference_raw(nh->nh_grp);
 220         for (i = 0; i < nhg->num_nh; ++i) {
 221                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 222
 223                 WARN_ON(!list_empty(&nhge->nh_list));
 224                 nexthop_put(nhge->nh);
 225         }
 226
 227         WARN_ON(nhg->spare == nhg);
 228
 229         kfree(nhg->spare);
 230         kfree(nhg);
 231 }
 232
 233 static void nexthop_free_single(struct nexthop *nh)
 234 {
 235         struct nh_info *nhi;
 236
 237         nhi = rcu_dereference_raw(nh->nh_info);
 238         switch (nhi->family) {
 239         case AF_INET:
 240                 fib_nh_release(nh->net, &nhi->fib_nh);
 241                 break;
 242         case AF_INET6:
 243                 ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
 244                 break;
 245         }
 246         kfree(nhi);
 247 }
 248
 249 void nexthop_free_rcu(struct rcu_head *head)
 250 {
 251         struct nexthop *nh = container_of(head, struct nexthop, rcu);
 252
 253         if (nh->is_group)
 254                 nexthop_free_group(nh);
 255         else
 256                 nexthop_free_single(nh);
 257
 258         kfree(nh);
 259 }
 260 EXPORT_SYMBOL_GPL(nexthop_free_rcu);
 261
 262 static struct nexthop *nexthop_alloc(void)
 263 {
 264         struct nexthop *nh;
 265
 266         nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
 267         if (nh) {
 268                 INIT_LIST_HEAD(&nh->fi_list);
 269                 INIT_LIST_HEAD(&nh->f6i_list);
 270                 INIT_LIST_HEAD(&nh->grp_list);
 271                 INIT_LIST_HEAD(&nh->fdb_list);
 272         }
 273         return nh;
 274 }
 275
 276 static struct nh_group *nexthop_grp_alloc(u16 num_nh)
 277 {
 278         struct nh_group *nhg;
 279
 280         nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);
 281         if (nhg)
 282                 nhg->num_nh = num_nh;
 283
 284         return nhg;
 285 }
 286
 287 static void nh_base_seq_inc(struct net *net)
 288 {
 289         while (++net->nexthop.seq == 0)
 290                 ;
 291 }
 292
 293 /* no reference taken; rcu lock or rtnl must be held */
 294 struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
 295 {
 296         struct rb_node **pp, *parent = NULL, *next;
 297
 298         pp = &net->nexthop.rb_root.rb_node;
 299         while (1) {
 300                 struct nexthop *nh;
 301
 302                 next = rcu_dereference_raw(*pp);
 303                 if (!next)
 304                         break;
 305                 parent = next;
 306
 307                 nh = rb_entry(parent, struct nexthop, rb_node);
 308                 if (id < nh->id)
 309                         pp = &next->rb_left;
 310                 else if (id > nh->id)
 311                         pp = &next->rb_right;
 312                 else
 313                         return nh;
 314         }
 315         return NULL;
 316 }
 317 EXPORT_SYMBOL_GPL(nexthop_find_by_id);
 318
 319 /* used for auto id allocation; called with rtnl held */
 320 static u32 nh_find_unused_id(struct net *net)
 321 {
 322         u32 id_start = net->nexthop.last_id_allocated;
 323
 324         while (1) {
 325                 net->nexthop.last_id_allocated++;
 326                 if (net->nexthop.last_id_allocated == id_start)
 327                         break;
 328
 329                 if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
 330                         return net->nexthop.last_id_allocated;
 331         }
 332         return 0;
 333 }
 334
 335 static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)
 336 {
 337         struct nexthop_grp *p;
 338         size_t len = nhg->num_nh * sizeof(*p);
 339         struct nlattr *nla;
 340         u16 group_type = 0;
 341         int i;
 342
 343         if (nhg->mpath)
 344                 group_type = NEXTHOP_GRP_TYPE_MPATH;
 345
 346         if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
 347                 goto nla_put_failure;
 348
 349         nla = nla_reserve(skb, NHA_GROUP, len);
 350         if (!nla)
 351                 goto nla_put_failure;
 352
 353         p = nla_data(nla);
 354         for (i = 0; i < nhg->num_nh; ++i) {
 355                 p->id = nhg->nh_entries[i].nh->id;
 356                 p->weight = nhg->nh_entries[i].weight - 1;
 357                 p += 1;
 358         }
 359
 360         return 0;
 361
 362 nla_put_failure:
 363         return -EMSGSIZE;
 364 }
 365
 366 static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
 367                         int event, u32 portid, u32 seq, unsigned int nlflags)
 368 {
 369         struct fib6_nh *fib6_nh;
 370         struct fib_nh *fib_nh;
 371         struct nlmsghdr *nlh;
 372         struct nh_info *nhi;
 373         struct nhmsg *nhm;
 374
 375         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
 376         if (!nlh)
 377                 return -EMSGSIZE;
 378
 379         nhm = nlmsg_data(nlh);
 380         nhm->nh_family = AF_UNSPEC;
 381         nhm->nh_flags = nh->nh_flags;
 382         nhm->nh_protocol = nh->protocol;
 383         nhm->nh_scope = 0;
 384         nhm->resvd = 0;
 385
 386         if (nla_put_u32(skb, NHA_ID, nh->id))
 387                 goto nla_put_failure;
 388
 389         if (nh->is_group) {
 390                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 391
 392                 if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
 393                         goto nla_put_failure;
 394                 if (nla_put_nh_group(skb, nhg))
 395                         goto nla_put_failure;
 396                 goto out;
 397         }
 398
 399         nhi = rtnl_dereference(nh->nh_info);
 400         nhm->nh_family = nhi->family;
 401         if (nhi->reject_nh) {
 402                 if (nla_put_flag(skb, NHA_BLACKHOLE))
 403                         goto nla_put_failure;
 404                 goto out;
 405         } else if (nhi->fdb_nh) {
 406                 if (nla_put_flag(skb, NHA_FDB))
 407                         goto nla_put_failure;
 408         } else {
 409                 const struct net_device *dev;
 410
 411                 dev = nhi->fib_nhc.nhc_dev;
 412                 if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
 413                         goto nla_put_failure;
 414         }
 415
 416         nhm->nh_scope = nhi->fib_nhc.nhc_scope;
 417         switch (nhi->family) {
 418         case AF_INET:
 419                 fib_nh = &nhi->fib_nh;
 420                 if (fib_nh->fib_nh_gw_family &&
 421                     nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
 422                         goto nla_put_failure;
 423                 break;
 424
 425         case AF_INET6:
 426                 fib6_nh = &nhi->fib6_nh;
 427                 if (fib6_nh->fib_nh_gw_family &&
 428                     nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
 429                         goto nla_put_failure;
 430                 break;
 431         }
 432
 433         if (nhi->fib_nhc.nhc_lwtstate &&
 434             lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
 435                                 NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
 436                 goto nla_put_failure;
 437
 438 out:
 439         nlmsg_end(skb, nlh);
 440         return 0;
 441
 442 nla_put_failure:
 443         nlmsg_cancel(skb, nlh);
 444         return -EMSGSIZE;
 445 }
 446
 447 static size_t nh_nlmsg_size_grp(struct nexthop *nh)
 448 {
 449         struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 450         size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
 451
 452         return nla_total_size(sz) +
 453                nla_total_size(2);  /* NHA_GROUP_TYPE */
 454 }
 455
 456 static size_t nh_nlmsg_size_single(struct nexthop *nh)
 457 {
 458         struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 459         size_t sz;
 460
 461         /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
 462          * are mutually exclusive
 463          */
 464         sz = nla_total_size(4);  /* NHA_OIF */
 465
 466         switch (nhi->family) {
 467         case AF_INET:
 468                 if (nhi->fib_nh.fib_nh_gw_family)
 469                         sz += nla_total_size(4);  /* NHA_GATEWAY */
 470                 break;
 471
 472         case AF_INET6:
 473                 /* NHA_GATEWAY */
 474                 if (nhi->fib6_nh.fib_nh_gw_family)
 475                         sz += nla_total_size(sizeof(const struct in6_addr));
 476                 break;
 477         }
 478
 479         if (nhi->fib_nhc.nhc_lwtstate) {
 480                 sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
 481                 sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
 482         }
 483
 484         return sz;
 485 }
 486
 487 static size_t nh_nlmsg_size(struct nexthop *nh)
 488 {
 489         size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));
 490
 491         sz += nla_total_size(4); /* NHA_ID */
 492
 493         if (nh->is_group)
 494                 sz += nh_nlmsg_size_grp(nh);
 495         else
 496                 sz += nh_nlmsg_size_single(nh);
 497
 498         return sz;
 499 }
 500
 501 static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
 502 {
 503         unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
 504         u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
 505         struct sk_buff *skb;
 506         int err = -ENOBUFS;
 507
 508         skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
 509         if (!skb)
 510                 goto errout;
 511
 512         err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags);
 513         if (err < 0) {
 514                 /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
 515                 WARN_ON(err == -EMSGSIZE);
 516                 kfree_skb(skb);
 517                 goto errout;
 518         }
 519
 520         rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
 521                     info->nlh, gfp_any());
 522         return;
 523 errout:
 524         if (err < 0)
 525                 rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
 526 }
 527
 528 static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
 529                            bool *is_fdb, struct netlink_ext_ack *extack)
 530 {
 531         if (nh->is_group) {
 532                 struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
 533
 534                 /* nested multipath (group within a group) is not
 535                  * supported
 536                  */
 537                 if (nhg->mpath) {
 538                         NL_SET_ERR_MSG(extack,
 539                                        "Multipath group can not be a nexthop within a group");
 540                         return false;
 541                 }
 542                 *is_fdb = nhg->fdb_nh;
 543         } else {
 544                 struct nh_info *nhi = rtnl_dereference(nh->nh_info);
 545
 546                 if (nhi->reject_nh && npaths > 1) {
 547                         NL_SET_ERR_MSG(extack,
 548                                        "Blackhole nexthop can not be used in a group with more than 1 path");
 549                         return false;
 550                 }
 551                 *is_fdb = nhi->fdb_nh;
 552         }
 553
 554         return true;
 555 }
 556
 557 static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
 558                                    struct netlink_ext_ack *extack)
 559 {
 560         struct nh_info *nhi;
 561
 562         nhi = rtnl_dereference(nh->nh_info);
 563
 564         if (!nhi->fdb_nh) {
 565                 NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
 566                 return -EINVAL;
 567         }
 568
 569         if (*nh_family == AF_UNSPEC) {
 570                 *nh_family = nhi->family;
 571         } else if (*nh_family != nhi->family) {
 572                 NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
 573                 return -EINVAL;
 574         }
 575
 576         return 0;
 577 }
 578
 579 static int nh_check_attr_group(struct net *net,
 580                                struct nlattr *tb[], size_t tb_size,
 581                                struct netlink_ext_ack *extack)
 582 {
 583         unsigned int len = nla_len(tb[NHA_GROUP]);
 584         u8 nh_family = AF_UNSPEC;
 585         struct nexthop_grp *nhg;
 586         unsigned int i, j;
 587         u8 nhg_fdb = 0;
 588
 589         if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
 590                 NL_SET_ERR_MSG(extack,
 591                                "Invalid length for nexthop group attribute");
 592                 return -EINVAL;
 593         }
 594
 595         /* convert len to number of nexthop ids */
 596         len /= sizeof(*nhg);
 597
 598         nhg = nla_data(tb[NHA_GROUP]);
 599         for (i = 0; i < len; ++i) {
 600                 if (nhg[i].resvd1 || nhg[i].resvd2) {
 601                         NL_SET_ERR_MSG(extack, "Reserved fields in nexthop_grp must be 0");
 602                         return -EINVAL;
 603                 }
 604                 if (nhg[i].weight > 254) {
 605                         NL_SET_ERR_MSG(extack, "Invalid value for weight");
 606                         return -EINVAL;
 607                 }
 608                 for (j = i + 1; j < len; ++j) {
 609                         if (nhg[i].id == nhg[j].id) {
 610                                 NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
 611                                 return -EINVAL;
 612                         }
 613                 }
 614         }
 615
 616         if (tb[NHA_FDB])
 617                 nhg_fdb = 1;
 618         nhg = nla_data(tb[NHA_GROUP]);
 619         for (i = 0; i < len; ++i) {
 620                 struct nexthop *nh;
 621                 bool is_fdb_nh;
 622
 623                 nh = nexthop_find_by_id(net, nhg[i].id);
 624                 if (!nh) {
 625                         NL_SET_ERR_MSG(extack, "Invalid nexthop id");
 626                         return -EINVAL;
 627                 }
 628                 if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
 629                         return -EINVAL;
 630
 631                 if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
 632                         return -EINVAL;
 633
 634                 if (!nhg_fdb && is_fdb_nh) {
 635                         NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
 636                         return -EINVAL;
 637                 }
 638         }
 639         for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
 640                 if (!tb[i])
 641                         continue;
 642                 if (i == NHA_FDB)
 643                         continue;
 644                 NL_SET_ERR_MSG(extack,
 645                                "No other attributes can be set in nexthop groups");
 646                 return -EINVAL;
 647         }
 648
 649         return 0;
 650 }
 651
 652 static bool ipv6_good_nh(const struct fib6_nh *nh)
 653 {
 654         int state = NUD_REACHABLE;
 655         struct neighbour *n;
 656
 657         rcu_read_lock_bh();
 658
 659         n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
 660         if (n)
 661                 state = n->nud_state;
 662
 663         rcu_read_unlock_bh();
 664
 665         return !!(state & NUD_VALID);
 666 }
 667
 668 static bool ipv4_good_nh(const struct fib_nh *nh)
 669 {
 670         int state = NUD_REACHABLE;
 671         struct neighbour *n;
 672
 673         rcu_read_lock_bh();
 674
 675         n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
 676                                       (__force u32)nh->fib_nh_gw4);
 677         if (n)
 678                 state = n->nud_state;
 679
 680         rcu_read_unlock_bh();
 681
 682         return !!(state & NUD_VALID);
 683 }
 684
 685 static struct nexthop *nexthop_select_path_mp(struct nh_group *nhg, int hash)
 686 {
 687         struct nexthop *rc = NULL;
 688         int i;
 689
 690         for (i = 0; i < nhg->num_nh; ++i) {
 691                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 692                 struct nh_info *nhi;
 693
 694                 if (hash > atomic_read(&nhge->mpath.upper_bound))
 695                         continue;
 696
 697                 nhi = rcu_dereference(nhge->nh->nh_info);
 698                 if (nhi->fdb_nh)
 699                         return nhge->nh;
 700
 701                 /* nexthops always check if it is good and does
 702                  * not rely on a sysctl for this behavior
 703                  */
 704                 switch (nhi->family) {
 705                 case AF_INET:
 706                         if (ipv4_good_nh(&nhi->fib_nh))
 707                                 return nhge->nh;
 708                         break;
 709                 case AF_INET6:
 710                         if (ipv6_good_nh(&nhi->fib6_nh))
 711                                 return nhge->nh;
 712                         break;
 713                 }
 714
 715                 if (!rc)
 716                         rc = nhge->nh;
 717         }
 718
 719         return rc;
 720 }
 721
 722 struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
 723 {
 724         struct nh_group *nhg;
 725
 726         if (!nh->is_group)
 727                 return nh;
 728
 729         nhg = rcu_dereference(nh->nh_grp);
 730         if (nhg->mpath)
 731                 return nexthop_select_path_mp(nhg, hash);
 732
 733         /* Unreachable. */
 734         return NULL;
 735 }
 736 EXPORT_SYMBOL_GPL(nexthop_select_path);
 737
 738 int nexthop_for_each_fib6_nh(struct nexthop *nh,
 739                              int (*cb)(struct fib6_nh *nh, void *arg),
 740                              void *arg)
 741 {
 742         struct nh_info *nhi;
 743         int err;
 744
 745         if (nh->is_group) {
 746                 struct nh_group *nhg;
 747                 int i;
 748
 749                 nhg = rcu_dereference_rtnl(nh->nh_grp);
 750                 for (i = 0; i < nhg->num_nh; i++) {
 751                         struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 752
 753                         nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
 754                         err = cb(&nhi->fib6_nh, arg);
 755                         if (err)
 756                                 return err;
 757                 }
 758         } else {
 759                 nhi = rcu_dereference_rtnl(nh->nh_info);
 760                 err = cb(&nhi->fib6_nh, arg);
 761                 if (err)
 762                         return err;
 763         }
 764
 765         return 0;
 766 }
 767 EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);
 768
 769 static int check_src_addr(const struct in6_addr *saddr,
 770                           struct netlink_ext_ack *extack)
 771 {
 772         if (!ipv6_addr_any(saddr)) {
 773                 NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
 774                 return -EINVAL;
 775         }
 776         return 0;
 777 }
 778
 779 int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
 780                        struct netlink_ext_ack *extack)
 781 {
 782         struct nh_info *nhi;
 783         bool is_fdb_nh;
 784
 785         /* fib6_src is unique to a fib6_info and limits the ability to cache
 786          * routes in fib6_nh within a nexthop that is potentially shared
 787          * across multiple fib entries. If the config wants to use source
 788          * routing it can not use nexthop objects. mlxsw also does not allow
 789          * fib6_src on routes.
 790          */
 791         if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
 792                 return -EINVAL;
 793
 794         if (nh->is_group) {
 795                 struct nh_group *nhg;
 796
 797                 nhg = rtnl_dereference(nh->nh_grp);
 798                 if (nhg->has_v4)
 799                         goto no_v4_nh;
 800                 is_fdb_nh = nhg->fdb_nh;
 801         } else {
 802                 nhi = rtnl_dereference(nh->nh_info);
 803                 if (nhi->family == AF_INET)
 804                         goto no_v4_nh;
 805                 is_fdb_nh = nhi->fdb_nh;
 806         }
 807
 808         if (is_fdb_nh) {
 809                 NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 810                 return -EINVAL;
 811         }
 812
 813         return 0;
 814 no_v4_nh:
 815         NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
 816         return -EINVAL;
 817 }
 818 EXPORT_SYMBOL_GPL(fib6_check_nexthop);
 819
 820 /* if existing nexthop has ipv6 routes linked to it, need
 821  * to verify this new spec works with ipv6
 822  */
 823 static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
 824                               struct netlink_ext_ack *extack)
 825 {
 826         struct fib6_info *f6i;
 827
 828         if (list_empty(&old->f6i_list))
 829                 return 0;
 830
 831         list_for_each_entry(f6i, &old->f6i_list, nh_list) {
 832                 if (check_src_addr(&f6i->fib6_src.addr, extack) < 0)
 833                         return -EINVAL;
 834         }
 835
 836         return fib6_check_nexthop(new, NULL, extack);
 837 }
 838
 839 static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
 840                                struct netlink_ext_ack *extack)
 841 {
 842         if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
 843                 NL_SET_ERR_MSG(extack,
 844                                "Route with host scope can not have a gateway");
 845                 return -EINVAL;
 846         }
 847
 848         if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
 849                 NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
 850                 return -EINVAL;
 851         }
 852
 853         return 0;
 854 }
 855
 856 /* Invoked by fib add code to verify nexthop by id is ok with
 857  * config for prefix; parts of fib_check_nh not done when nexthop
 858  * object is used.
 859  */
 860 int fib_check_nexthop(struct nexthop *nh, u8 scope,
 861                       struct netlink_ext_ack *extack)
 862 {
 863         struct nh_info *nhi;
 864         int err = 0;
 865
 866         if (nh->is_group) {
 867                 struct nh_group *nhg;
 868
 869                 nhg = rtnl_dereference(nh->nh_grp);
 870                 if (nhg->fdb_nh) {
 871                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 872                         err = -EINVAL;
 873                         goto out;
 874                 }
 875
 876                 if (scope == RT_SCOPE_HOST) {
 877                         NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
 878                         err = -EINVAL;
 879                         goto out;
 880                 }
 881
 882                 /* all nexthops in a group have the same scope */
 883                 nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
 884                 err = nexthop_check_scope(nhi, scope, extack);
 885         } else {
 886                 nhi = rtnl_dereference(nh->nh_info);
 887                 if (nhi->fdb_nh) {
 888                         NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
 889                         err = -EINVAL;
 890                         goto out;
 891                 }
 892                 err = nexthop_check_scope(nhi, scope, extack);
 893         }
 894
 895 out:
 896         return err;
 897 }
 898
 899 static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
 900                              struct netlink_ext_ack *extack)
 901 {
 902         struct fib_info *fi;
 903
 904         list_for_each_entry(fi, &old->fi_list, nh_list) {
 905                 int err;
 906
 907                 err = fib_check_nexthop(new, fi->fib_scope, extack);
 908                 if (err)
 909                         return err;
 910         }
 911         return 0;
 912 }
 913
 914 static void nh_group_rebalance(struct nh_group *nhg)
 915 {
 916         int total = 0;
 917         int w = 0;
 918         int i;
 919
 920         for (i = 0; i < nhg->num_nh; ++i)
 921                 total += nhg->nh_entries[i].weight;
 922
 923         for (i = 0; i < nhg->num_nh; ++i) {
 924                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
 925                 int upper_bound;
 926
 927                 w += nhge->weight;
 928                 upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
 929                 atomic_set(&nhge->mpath.upper_bound, upper_bound);
 930         }
 931 }
 932
 933 static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
 934                                 struct nl_info *nlinfo)
 935 {
 936         struct nh_grp_entry *nhges, *new_nhges;
 937         struct nexthop *nhp = nhge->nh_parent;
 938         struct netlink_ext_ack extack;
 939         struct nexthop *nh = nhge->nh;
 940         struct nh_group *nhg, *newg;
 941         int i, j, err;
 942
 943         WARN_ON(!nh);
 944
 945         nhg = rtnl_dereference(nhp->nh_grp);
 946         newg = nhg->spare;
 947
 948         /* last entry, keep it visible and remove the parent */
 949         if (nhg->num_nh == 1) {
 950                 remove_nexthop(net, nhp, nlinfo);
 951                 return;
 952         }
 953
 954         newg->has_v4 = false;
 955         newg->mpath = nhg->mpath;
 956         newg->fdb_nh = nhg->fdb_nh;
 957         newg->num_nh = nhg->num_nh;
 958
 959         /* copy old entries to new except the one getting removed */
 960         nhges = nhg->nh_entries;
 961         new_nhges = newg->nh_entries;
 962         for (i = 0, j = 0; i < nhg->num_nh; ++i) {
 963                 struct nh_info *nhi;
 964
 965                 /* current nexthop getting removed */
 966                 if (nhg->nh_entries[i].nh == nh) {
 967                         newg->num_nh--;
 968                         continue;
 969                 }
 970
 971                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
 972                 if (nhi->family == AF_INET)
 973                         newg->has_v4 = true;
 974
 975                 list_del(&nhges[i].nh_list);
 976                 new_nhges[j].nh_parent = nhges[i].nh_parent;
 977                 new_nhges[j].nh = nhges[i].nh;
 978                 new_nhges[j].weight = nhges[i].weight;
 979                 list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
 980                 j++;
 981         }
 982
 983         nh_group_rebalance(newg);
 984         rcu_assign_pointer(nhp->nh_grp, newg);
 985
 986         list_del(&nhge->nh_list);
 987         nexthop_put(nhge->nh);
 988
 989         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp, &extack);
 990         if (err)
 991                 pr_err("%s\n", extack._msg);
 992
 993         if (nlinfo)
 994                 nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
 995 }
 996
 997 static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
 998                                        struct nl_info *nlinfo)
 999 {
1000         struct nh_grp_entry *nhge, *tmp;
1001
1002         list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
1003                 remove_nh_grp_entry(net, nhge, nlinfo);
1004
1005         /* make sure all see the newly published array before releasing rtnl */
1006         synchronize_net();
1007 }
1008
1009 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
1010 {
1011         struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
1012         int i, num_nh = nhg->num_nh;
1013
1014         for (i = 0; i < num_nh; ++i) {
1015                 struct nh_grp_entry *nhge = &nhg->nh_entries[i];
1016
1017                 if (WARN_ON(!nhge->nh))
1018                         continue;
1019
1020                 list_del_init(&nhge->nh_list);
1021         }
1022 }
1023
1024 /* not called for nexthop replace */
1025 static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
1026 {
1027         struct fib6_info *f6i, *tmp;
1028         bool do_flush = false;
1029         struct fib_info *fi;
1030
1031         list_for_each_entry(fi, &nh->fi_list, nh_list) {
1032                 fi->fib_flags |= RTNH_F_DEAD;
1033                 do_flush = true;
1034         }
1035         if (do_flush)
1036                 fib_flush(net);
1037
1038         /* ip6_del_rt removes the entry from this list hence the _safe */
1039         list_for_each_entry_safe(f6i, tmp, &nh->f6i_list, nh_list) {
1040                 /* __ip6_del_rt does a release, so do a hold here */
1041                 fib6_info_hold(f6i);
1042                 ipv6_stub->ip6_del_rt(net, f6i,
1043                                       !net->ipv4.sysctl_nexthop_compat_mode);
1044         }
1045 }
1046
1047 static void __remove_nexthop(struct net *net, struct nexthop *nh,
1048                              struct nl_info *nlinfo)
1049 {
1050         __remove_nexthop_fib(net, nh);
1051
1052         if (nh->is_group) {
1053                 remove_nexthop_group(nh, nlinfo);
1054         } else {
1055                 struct nh_info *nhi;
1056
1057                 nhi = rtnl_dereference(nh->nh_info);
1058                 if (nhi->fib_nhc.nhc_dev)
1059                         hlist_del(&nhi->dev_hash);
1060
1061                 remove_nexthop_from_groups(net, nh, nlinfo);
1062         }
1063 }
1064
1065 static void remove_nexthop(struct net *net, struct nexthop *nh,
1066                            struct nl_info *nlinfo)
1067 {
1068         call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);
1069
1070         /* remove from the tree */
1071         rb_erase(&nh->rb_node, &net->nexthop.rb_root);
1072
1073         if (nlinfo)
1074                 nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
1075
1076         __remove_nexthop(net, nh, nlinfo);
1077         nh_base_seq_inc(net);
1078
1079         nexthop_put(nh);
1080 }
1081
1082 /* if any FIB entries reference this nexthop, any dst entries
1083  * need to be regenerated
1084  */
1085 static void nh_rt_cache_flush(struct net *net, struct nexthop *nh)
1086 {
1087         struct fib6_info *f6i;
1088
1089         if (!list_empty(&nh->fi_list))
1090                 rt_cache_flush(net);
1091
1092         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
1093                 ipv6_stub->fib6_update_sernum(net, f6i);
1094 }
1095
1096 static int replace_nexthop_grp(struct net *net, struct nexthop *old,
1097                                struct nexthop *new,
1098                                struct netlink_ext_ack *extack)
1099 {
1100         struct nh_group *oldg, *newg;
1101         int i, err;
1102
1103         if (!new->is_group) {
1104                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
1105                 return -EINVAL;
1106         }
1107
1108         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
1109         if (err)
1110                 return err;
1111
1112         oldg = rtnl_dereference(old->nh_grp);
1113         newg = rtnl_dereference(new->nh_grp);
1114
1115         /* update parents - used by nexthop code for cleanup */
1116         for (i = 0; i < newg->num_nh; i++)
1117                 newg->nh_entries[i].nh_parent = old;
1118
1119         rcu_assign_pointer(old->nh_grp, newg);
1120
1121         for (i = 0; i < oldg->num_nh; i++)
1122                 oldg->nh_entries[i].nh_parent = new;
1123
1124         rcu_assign_pointer(new->nh_grp, oldg);
1125
1126         return 0;
1127 }
1128
1129 static void nh_group_v4_update(struct nh_group *nhg)
1130 {
1131         struct nh_grp_entry *nhges;
1132         bool has_v4 = false;
1133         int i;
1134
1135         nhges = nhg->nh_entries;
1136         for (i = 0; i < nhg->num_nh; i++) {
1137                 struct nh_info *nhi;
1138
1139                 nhi = rtnl_dereference(nhges[i].nh->nh_info);
1140                 if (nhi->family == AF_INET)
1141                         has_v4 = true;
1142         }
1143         nhg->has_v4 = has_v4;
1144 }
1145
1146 static int replace_nexthop_single(struct net *net, struct nexthop *old,
1147                                   struct nexthop *new,
1148                                   struct netlink_ext_ack *extack)
1149 {
1150         u8 old_protocol, old_nh_flags;
1151         struct nh_info *oldi, *newi;
1152         struct nh_grp_entry *nhge;
1153         int err;
1154
1155         if (new->is_group) {
1156                 NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
1157                 return -EINVAL;
1158         }
1159
1160         err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
1161         if (err)
1162                 return err;
1163
1164         /* Hardware flags were set on 'old' as 'new' is not in the red-black
1165          * tree. Therefore, inherit the flags from 'old' to 'new'.
1166          */
1167         new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);
1168
1169         oldi = rtnl_dereference(old->nh_info);
1170         newi = rtnl_dereference(new->nh_info);
1171
1172         newi->nh_parent = old;
1173         oldi->nh_parent = new;
1174
1175         old_protocol = old->protocol;
1176         old_nh_flags = old->nh_flags;
1177
1178         old->protocol = new->protocol;
1179         old->nh_flags = new->nh_flags;
1180
1181         rcu_assign_pointer(old->nh_info, newi);
1182         rcu_assign_pointer(new->nh_info, oldi);
1183
1184         /* Send a replace notification for all the groups using the nexthop. */
1185         list_for_each_entry(nhge, &old->grp_list, nh_list) {
1186                 struct nexthop *nhp = nhge->nh_parent;
1187
1188                 err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
1189                                              extack);
1190                 if (err)
1191                         goto err_notify;
1192         }
1193
1194         /* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially
1195          * update IPv4 indication in all the groups using the nexthop.
1196          */
1197         if (oldi->family == AF_INET && newi->family == AF_INET6) {
1198                 list_for_each_entry(nhge, &old->grp_list, nh_list) {
1199                         struct nexthop *nhp = nhge->nh_parent;
1200                         struct nh_group *nhg;
1201
1202                         nhg = rtnl_dereference(nhp->nh_grp);
1203                         nh_group_v4_update(nhg);
1204                 }
1205         }
1206
1207         return 0;
1208
1209 err_notify:
1210         rcu_assign_pointer(new->nh_info, newi);
1211         rcu_assign_pointer(old->nh_info, oldi);
1212         old->nh_flags = old_nh_flags;
1213         old->protocol = old_protocol;
1214         oldi->nh_parent = old;
1215         newi->nh_parent = new;
1216         list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
1217                 struct nexthop *nhp = nhge->nh_parent;
1218
1219                 call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp, extack);
1220         }
1221         call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
1222         return err;
1223 }
1224
1225 static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
1226                                      struct nl_info *info)
1227 {
1228         struct fib6_info *f6i;
1229
1230         if (!list_empty(&nh->fi_list)) {
1231                 struct fib_info *fi;
1232
1233                 /* expectation is a few fib_info per nexthop and then
1234                  * a lot of routes per fib_info. So mark the fib_info
1235                  * and then walk the fib tables once
1236                  */
1237                 list_for_each_entry(fi, &nh->fi_list, nh_list)
1238                         fi->nh_updated = true;
1239
1240                 fib_info_notify_update(net, info);
1241
1242                 list_for_each_entry(fi, &nh->fi_list, nh_list)
1243                         fi->nh_updated = false;
1244         }
1245
1246         list_for_each_entry(f6i, &nh->f6i_list, nh_list)
1247                 ipv6_stub->fib6_rt_update(net, f6i, info);
1248 }
1249
1250 /* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
1251  * linked to this nexthop and for all groups that the nexthop
1252  * is a member of
1253  */
1254 static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
1255                                    struct nl_info *info)
1256 {
1257         struct nh_grp_entry *nhge;
1258
1259         __nexthop_replace_notify(net, nh, info);
1260
1261         list_for_each_entry(nhge, &nh->grp_list, nh_list)
1262                 __nexthop_replace_notify(net, nhge->nh_parent, info);
1263 }
1264
1265 static int replace_nexthop(struct net *net, struct nexthop *old,
1266                            struct nexthop *new, struct netlink_ext_ack *extack)
1267 {
1268         bool new_is_reject = false;
1269         struct nh_grp_entry *nhge;
1270         int err;
1271
1272         /* check that existing FIB entries are ok with the
1273          * new nexthop definition
1274          */
1275         err = fib_check_nh_list(old, new, extack);
1276         if (err)
1277                 return err;
1278
1279         err = fib6_check_nh_list(old, new, extack);
1280         if (err)
1281                 return err;
1282
1283         if (!new->is_group) {
1284                 struct nh_info *nhi = rtnl_dereference(new->nh_info);
1285
1286                 new_is_reject = nhi->reject_nh;
1287         }
1288
1289         list_for_each_entry(nhge, &old->grp_list, nh_list) {
1290                 /* if new nexthop is a blackhole, any groups using this
1291                  * nexthop cannot have more than 1 path
1292                  */
1293                 if (new_is_reject &&
1294                     nexthop_num_path(nhge->nh_parent) > 1) {
1295                         NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
1296                         return -EINVAL;
1297                 }
1298
1299                 err = fib_check_nh_list(nhge->nh_parent, new, extack);
1300                 if (err)
1301                         return err;
1302
1303                 err = fib6_check_nh_list(nhge->nh_parent, new, extack);
1304                 if (err)
1305                         return err;
1306         }
1307
1308         if (old->is_group)
1309                 err = replace_nexthop_grp(net, old, new, extack);
1310         else
1311                 err = replace_nexthop_single(net, old, new, extack);
1312
1313         if (!err) {
1314                 nh_rt_cache_flush(net, old);
1315
1316                 __remove_nexthop(net, new, NULL);
1317                 nexthop_put(new);
1318         }
1319
1320         return err;
1321 }
1322
1323 /* called with rtnl_lock held */
1324 static int insert_nexthop(struct net *net, struct nexthop *new_nh,
1325                           struct nh_config *cfg, struct netlink_ext_ack *extack)
1326 {
1327         struct rb_node **pp, *parent = NULL, *next;
1328         struct rb_root *root = &net->nexthop.rb_root;
1329         bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
1330         bool create = !!(cfg->nlflags & NLM_F_CREATE);
1331         u32 new_id = new_nh->id;
1332         int replace_notify = 0;
1333         int rc = -EEXIST;
1334
1335         pp = &root->rb_node;
1336         while (1) {
1337                 struct nexthop *nh;
1338
1339                 next = *pp;
1340                 if (!next)
1341                         break;
1342
1343                 parent = next;
1344
1345                 nh = rb_entry(parent, struct nexthop, rb_node);
1346                 if (new_id < nh->id) {
1347                         pp = &next->rb_left;
1348                 } else if (new_id > nh->id) {
1349                         pp = &next->rb_right;
1350                 } else if (replace) {
1351                         rc = replace_nexthop(net, nh, new_nh, extack);
1352                         if (!rc) {
1353                                 new_nh = nh; /* send notification with old nh */
1354                                 replace_notify = 1;
1355                         }
1356                         goto out;
1357                 } else {
1358                         /* id already exists and not a replace */
1359                         goto out;
1360                 }
1361         }
1362
1363         if (replace && !create) {
1364                 NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
1365                 rc = -ENOENT;
1366                 goto out;
1367         }
1368
1369         rb_link_node_rcu(&new_nh->rb_node, parent, pp);
1370         rb_insert_color(&new_nh->rb_node, root);
1371
1372         rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
1373         if (rc)
1374                 rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);
1375
1376 out:
1377         if (!rc) {
1378                 nh_base_seq_inc(net);
1379                 nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
1380                 if (replace_notify && net->ipv4.sysctl_nexthop_compat_mode)
1381                         nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
1382         }
1383
1384         return rc;
1385 }
1386
1387 /* rtnl */
1388 /* remove all nexthops tied to a device being deleted */
1389 static void nexthop_flush_dev(struct net_device *dev)
1390 {
1391         unsigned int hash = nh_dev_hashfn(dev->ifindex);
1392         struct net *net = dev_net(dev);
1393         struct hlist_head *head = &net->nexthop.devhash[hash];
1394         struct hlist_node *n;
1395         struct nh_info *nhi;
1396
1397         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
1398                 if (nhi->fib_nhc.nhc_dev != dev)
1399                         continue;
1400
1401                 remove_nexthop(net, nhi->nh_parent, NULL);
1402         }
1403 }
1404
1405 /* rtnl; called when net namespace is deleted */
1406 static void flush_all_nexthops(struct net *net)
1407 {
1408         struct rb_root *root = &net->nexthop.rb_root;
1409         struct rb_node *node;
1410         struct nexthop *nh;
1411
1412         while ((node = rb_first(root))) {
1413                 nh = rb_entry(node, struct nexthop, rb_node);
1414                 remove_nexthop(net, nh, NULL);
1415                 cond_resched();
1416         }
1417 }
1418
1419 static struct nexthop *nexthop_create_group(struct net *net,
1420                                             struct nh_config *cfg)
1421 {
1422         struct nlattr *grps_attr = cfg->nh_grp;
1423         struct nexthop_grp *entry = nla_data(grps_attr);
1424         u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
1425         struct nh_group *nhg;
1426         struct nexthop *nh;
1427         int i;
1428
1429         if (WARN_ON(!num_nh))
1430                 return ERR_PTR(-EINVAL);
1431
1432         nh = nexthop_alloc();
1433         if (!nh)
1434                 return ERR_PTR(-ENOMEM);
1435
1436         nh->is_group = 1;
1437
1438         nhg = nexthop_grp_alloc(num_nh);
1439         if (!nhg) {
1440                 kfree(nh);
1441                 return ERR_PTR(-ENOMEM);
1442         }
1443
1444         /* spare group used for removals */
1445         nhg->spare = nexthop_grp_alloc(num_nh);
1446         if (!nhg->spare) {
1447                 kfree(nhg);
1448                 kfree(nh);
1449                 return ERR_PTR(-ENOMEM);
1450         }
1451         nhg->spare->spare = nhg;
1452
1453         for (i = 0; i < nhg->num_nh; ++i) {
1454                 struct nexthop *nhe;
1455                 struct nh_info *nhi;
1456
1457                 nhe = nexthop_find_by_id(net, entry[i].id);
1458                 if (!nexthop_get(nhe))
1459                         goto out_no_nh;
1460
1461                 nhi = rtnl_dereference(nhe->nh_info);
1462                 if (nhi->family == AF_INET)
1463                         nhg->has_v4 = true;
1464
1465                 nhg->nh_entries[i].nh = nhe;
1466                 nhg->nh_entries[i].weight = entry[i].weight + 1;
1467                 list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
1468                 nhg->nh_entries[i].nh_parent = nh;
1469         }
1470
1471         if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH)
1472                 nhg->mpath = 1;
1473
1474         WARN_ON_ONCE(nhg->mpath != 1);
1475
1476         if (nhg->mpath)
1477                 nh_group_rebalance(nhg);
1478
1479         if (cfg->nh_fdb)
1480                 nhg->fdb_nh = 1;
1481
1482         rcu_assign_pointer(nh->nh_grp, nhg);
1483
1484         return nh;
1485
1486 out_no_nh:
1487         for (i--; i >= 0; --i) {
1488                 list_del(&nhg->nh_entries[i].nh_list);
1489                 nexthop_put(nhg->nh_entries[i].nh);
1490         }
1491
1492         kfree(nhg->spare);
1493         kfree(nhg);
1494         kfree(nh);
1495
1496         return ERR_PTR(-ENOENT);
1497 }
1498
1499 static int nh_create_ipv4(struct net *net, struct nexthop *nh,
1500                           struct nh_info *nhi, struct nh_config *cfg,
1501                           struct netlink_ext_ack *extack)
1502 {
1503         struct fib_nh *fib_nh = &nhi->fib_nh;
1504         struct fib_config fib_cfg = {
1505                 .fc_oif   = cfg->nh_ifindex,
1506                 .fc_gw4   = cfg->gw.ipv4,
1507                 .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
1508                 .fc_flags = cfg->nh_flags,
1509                 .fc_encap = cfg->nh_encap,
1510                 .fc_encap_type = cfg->nh_encap_type,
1511         };
1512         u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
1513         int err;
1514
1515         err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
1516         if (err) {
1517                 fib_nh_release(net, fib_nh);
1518                 goto out;
1519         }
1520
1521         if (nhi->fdb_nh)
1522                 goto out;
1523
1524         /* sets nh_dev if successful */
1525         err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
1526         if (!err) {
1527                 nh->nh_flags = fib_nh->fib_nh_flags;
1528                 fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
1529                                           fib_nh->fib_nh_scope);
1530         } else {
1531                 fib_nh_release(net, fib_nh);
1532         }
1533 out:
1534         return err;
1535 }
1536
1537 static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
1538                           struct nh_info *nhi, struct nh_config *cfg,
1539                           struct netlink_ext_ack *extack)
1540 {
1541         struct fib6_nh *fib6_nh = &nhi->fib6_nh;
1542         struct fib6_config fib6_cfg = {
1543                 .fc_table = l3mdev_fib_table(cfg->dev),
1544                 .fc_ifindex = cfg->nh_ifindex,
1545                 .fc_gateway = cfg->gw.ipv6,
1546                 .fc_flags = cfg->nh_flags,
1547                 .fc_encap = cfg->nh_encap,
1548                 .fc_encap_type = cfg->nh_encap_type,
1549                 .fc_is_fdb = cfg->nh_fdb,
1550         };
1551         int err;
1552
1553         if (!ipv6_addr_any(&cfg->gw.ipv6))
1554                 fib6_cfg.fc_flags |= RTF_GATEWAY;
1555
1556         /* sets nh_dev if successful */
1557         err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
1558                                       extack);
1559         if (err)
1560                 ipv6_stub->fib6_nh_release(fib6_nh);
1561         else
1562                 nh->nh_flags = fib6_nh->fib_nh_flags;
1563
1564         return err;
1565 }
1566
1567 static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
1568                                       struct netlink_ext_ack *extack)
1569 {
1570         struct nh_info *nhi;
1571         struct nexthop *nh;
1572         int err = 0;
1573
1574         nh = nexthop_alloc();
1575         if (!nh)
1576                 return ERR_PTR(-ENOMEM);
1577
1578         nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
1579         if (!nhi) {
1580                 kfree(nh);
1581                 return ERR_PTR(-ENOMEM);
1582         }
1583
1584         nh->nh_flags = cfg->nh_flags;
1585         nh->net = net;
1586
1587         nhi->nh_parent = nh;
1588         nhi->family = cfg->nh_family;
1589         nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;
1590
1591         if (cfg->nh_fdb)
1592                 nhi->fdb_nh = 1;
1593
1594         if (cfg->nh_blackhole) {
1595                 nhi->reject_nh = 1;
1596                 cfg->nh_ifindex = net->loopback_dev->ifindex;
1597         }
1598
1599         switch (cfg->nh_family) {
1600         case AF_INET:
1601                 err = nh_create_ipv4(net, nh, nhi, cfg, extack);
1602                 break;
1603         case AF_INET6:
1604                 err = nh_create_ipv6(net, nh, nhi, cfg, extack);
1605                 break;
1606         }
1607
1608         if (err) {
1609                 kfree(nhi);
1610                 kfree(nh);
1611                 return ERR_PTR(err);
1612         }
1613
1614         /* add the entry to the device based hash */
1615         if (!nhi->fdb_nh)
1616                 nexthop_devhash_add(net, nhi);
1617
1618         rcu_assign_pointer(nh->nh_info, nhi);
1619
1620         return nh;
1621 }
1622
1623 /* called with rtnl lock held */
1624 static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
1625                                    struct netlink_ext_ack *extack)
1626 {
1627         struct nexthop *nh;
1628         int err;
1629
1630         if (cfg->nlflags & NLM_F_REPLACE && !cfg->nh_id) {
1631                 NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
1632                 return ERR_PTR(-EINVAL);
1633         }
1634
1635         if (!cfg->nh_id) {
1636                 cfg->nh_id = nh_find_unused_id(net);
1637                 if (!cfg->nh_id) {
1638                         NL_SET_ERR_MSG(extack, "No unused id");
1639                         return ERR_PTR(-EINVAL);
1640                 }
1641         }
1642
1643         if (cfg->nh_grp)
1644                 nh = nexthop_create_group(net, cfg);
1645         else
1646                 nh = nexthop_create(net, cfg, extack);
1647
1648         if (IS_ERR(nh))
1649                 return nh;
1650
1651         refcount_set(&nh->refcnt, 1);
1652         nh->id = cfg->nh_id;
1653         nh->protocol = cfg->nh_protocol;
1654         nh->net = net;
1655
1656         err = insert_nexthop(net, nh, cfg, extack);
1657         if (err) {
1658                 __remove_nexthop(net, nh, NULL);
1659                 nexthop_put(nh);
1660                 nh = ERR_PTR(err);
1661         }
1662
1663         return nh;
1664 }
1665
1666 static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
1667                             struct nlmsghdr *nlh, struct nh_config *cfg,
1668                             struct netlink_ext_ack *extack)
1669 {
1670         struct nhmsg *nhm = nlmsg_data(nlh);
1671         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
1672         int err;
1673
1674         err = nlmsg_parse(nlh, sizeof(*nhm), tb,
1675                           ARRAY_SIZE(rtm_nh_policy_new) - 1,
1676                           rtm_nh_policy_new, extack);
1677         if (err < 0)
1678                 return err;
1679
1680         err = -EINVAL;
1681         if (nhm->resvd || nhm->nh_scope) {
1682                 NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
1683                 goto out;
1684         }
1685         if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
1686                 NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
1687                 goto out;
1688         }
1689
1690         switch (nhm->nh_family) {
1691         case AF_INET:
1692         case AF_INET6:
1693                 break;
1694         case AF_UNSPEC:
1695                 if (tb[NHA_GROUP])
1696                         break;
1697                 fallthrough;
1698         default:
1699                 NL_SET_ERR_MSG(extack, "Invalid address family");
1700                 goto out;
1701         }
1702
1703         memset(cfg, 0, sizeof(*cfg));
1704         cfg->nlflags = nlh->nlmsg_flags;
1705         cfg->nlinfo.portid = NETLINK_CB(skb).portid;
1706         cfg->nlinfo.nlh = nlh;
1707         cfg->nlinfo.nl_net = net;
1708
1709         cfg->nh_family = nhm->nh_family;
1710         cfg->nh_protocol = nhm->nh_protocol;
1711         cfg->nh_flags = nhm->nh_flags;
1712
1713         if (tb[NHA_ID])
1714                 cfg->nh_id = nla_get_u32(tb[NHA_ID]);
1715
1716         if (tb[NHA_FDB]) {
1717                 if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
1718                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
1719                         NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
1720                         goto out;
1721                 }
1722                 if (nhm->nh_flags) {
1723                         NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
1724                         goto out;
1725                 }
1726                 cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
1727         }
1728
1729         if (tb[NHA_GROUP]) {
1730                 if (nhm->nh_family != AF_UNSPEC) {
1731                         NL_SET_ERR_MSG(extack, "Invalid family for group");
1732                         goto out;
1733                 }
1734                 cfg->nh_grp = tb[NHA_GROUP];
1735
1736                 cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
1737                 if (tb[NHA_GROUP_TYPE])
1738                         cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);
1739
1740                 if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
1741                         NL_SET_ERR_MSG(extack, "Invalid group type");
1742                         goto out;
1743                 }
1744                 err = nh_check_attr_group(net, tb, ARRAY_SIZE(tb), extack);
1745
1746                 /* no other attributes should be set */
1747                 goto out;
1748         }
1749
1750         if (tb[NHA_BLACKHOLE]) {
1751                 if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
1752                     tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
1753                         NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
1754                         goto out;
1755                 }
1756
1757                 cfg->nh_blackhole = 1;
1758                 err = 0;
1759                 goto out;
1760         }
1761
1762         if (!cfg->nh_fdb && !tb[NHA_OIF]) {
1763                 NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
1764                 goto out;
1765         }
1766
1767         if (!cfg->nh_fdb && tb[NHA_OIF]) {
1768                 cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
1769                 if (cfg->nh_ifindex)
1770                         cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);
1771
1772                 if (!cfg->dev) {
1773                         NL_SET_ERR_MSG(extack, "Invalid device index");
1774                         goto out;
1775                 } else if (!(cfg->dev->flags & IFF_UP)) {
1776                         NL_SET_ERR_MSG(extack, "Nexthop device is not up");
1777                         err = -ENETDOWN;
1778                         goto out;
1779                 } else if (!netif_carrier_ok(cfg->dev)) {
1780                         NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
1781                         err = -ENETDOWN;
1782                         goto out;
1783                 }
1784         }
1785
1786         err = -EINVAL;
1787         if (tb[NHA_GATEWAY]) {
1788                 struct nlattr *gwa = tb[NHA_GATEWAY];
1789
1790                 switch (cfg->nh_family) {
1791                 case AF_INET:
1792                         if (nla_len(gwa) != sizeof(u32)) {
1793                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1794                                 goto out;
1795                         }
1796                         cfg->gw.ipv4 = nla_get_be32(gwa);
1797                         break;
1798                 case AF_INET6:
1799                         if (nla_len(gwa) != sizeof(struct in6_addr)) {
1800                                 NL_SET_ERR_MSG(extack, "Invalid gateway");
1801                                 goto out;
1802                         }
1803                         cfg->gw.ipv6 = nla_get_in6_addr(gwa);
1804                         break;
1805                 default:
1806                         NL_SET_ERR_MSG(extack,
1807                                        "Unknown address family for gateway");
1808                         goto out;
1809                 }
1810         } else {
1811                 /* device only nexthop (no gateway) */
1812                 if (cfg->nh_flags & RTNH_F_ONLINK) {
1813                         NL_SET_ERR_MSG(extack,
1814                                        "ONLINK flag can not be set for nexthop without a gateway");
1815                         goto out;
1816                 }
1817         }
1818
1819         if (tb[NHA_ENCAP]) {
1820                 cfg->nh_encap = tb[NHA_ENCAP];
1821
1822                 if (!tb[NHA_ENCAP_TYPE]) {
1823                         NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
1824                         goto out;
1825                 }
1826
1827                 cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
1828                 err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
1829                 if (err < 0)
1830                         goto out;
1831
1832         } else if (tb[NHA_ENCAP_TYPE]) {
1833                 NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
1834                 goto out;
1835         }
1836
1837
1838         err = 0;
1839 out:
1840         return err;
1841 }
1842
1843 /* rtnl */
1844 static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1845                            struct netlink_ext_ack *extack)
1846 {
1847         struct net *net = sock_net(skb->sk);
1848         struct nh_config cfg;
1849         struct nexthop *nh;
1850         int err;
1851
1852         err = rtm_to_nh_config(net, skb, nlh, &cfg, extack);
1853         if (!err) {
1854                 nh = nexthop_add(net, &cfg, extack);
1855                 if (IS_ERR(nh))
1856                         err = PTR_ERR(nh);
1857         }
1858
1859         return err;
1860 }
1861
1862 static int nh_valid_get_del_req(struct nlmsghdr *nlh, u32 *id,
1863                                 struct netlink_ext_ack *extack)
1864 {
1865         struct nhmsg *nhm = nlmsg_data(nlh);
1866         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
1867         int err;
1868
1869         err = nlmsg_parse(nlh, sizeof(*nhm), tb,
1870                           ARRAY_SIZE(rtm_nh_policy_get) - 1,
1871                           rtm_nh_policy_get, extack);
1872         if (err < 0)
1873                 return err;
1874
1875         err = -EINVAL;
1876         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
1877                 NL_SET_ERR_MSG(extack, "Invalid values in header");
1878                 goto out;
1879         }
1880
1881         if (!tb[NHA_ID]) {
1882                 NL_SET_ERR_MSG(extack, "Nexthop id is missing");
1883                 goto out;
1884         }
1885
1886         *id = nla_get_u32(tb[NHA_ID]);
1887         if (!(*id))
1888                 NL_SET_ERR_MSG(extack, "Invalid nexthop id");
1889         else
1890                 err = 0;
1891 out:
1892         return err;
1893 }
1894
1895 /* rtnl */
1896 static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
1897                            struct netlink_ext_ack *extack)
1898 {
1899         struct net *net = sock_net(skb->sk);
1900         struct nl_info nlinfo = {
1901                 .nlh = nlh,
1902                 .nl_net = net,
1903                 .portid = NETLINK_CB(skb).portid,
1904         };
1905         struct nexthop *nh;
1906         int err;
1907         u32 id;
1908
1909         err = nh_valid_get_del_req(nlh, &id, extack);
1910         if (err)
1911                 return err;
1912
1913         nh = nexthop_find_by_id(net, id);
1914         if (!nh)
1915                 return -ENOENT;
1916
1917         remove_nexthop(net, nh, &nlinfo);
1918
1919         return 0;
1920 }
1921
1922 /* rtnl */
1923 static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
1924                            struct netlink_ext_ack *extack)
1925 {
1926         struct net *net = sock_net(in_skb->sk);
1927         struct sk_buff *skb = NULL;
1928         struct nexthop *nh;
1929         int err;
1930         u32 id;
1931
1932         err = nh_valid_get_del_req(nlh, &id, extack);
1933         if (err)
1934                 return err;
1935
1936         err = -ENOBUFS;
1937         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1938         if (!skb)
1939                 goto out;
1940
1941         err = -ENOENT;
1942         nh = nexthop_find_by_id(net, id);
1943         if (!nh)
1944                 goto errout_free;
1945
1946         err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
1947                            nlh->nlmsg_seq, 0);
1948         if (err < 0) {
1949                 WARN_ON(err == -EMSGSIZE);
1950                 goto errout_free;
1951         }
1952
1953         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
1954 out:
1955         return err;
1956 errout_free:
1957         kfree_skb(skb);
1958         goto out;
1959 }
1960
1961 static bool nh_dump_filtered(struct nexthop *nh, int dev_idx, int master_idx,
1962                              bool group_filter, u8 family)
1963 {
1964         const struct net_device *dev;
1965         const struct nh_info *nhi;
1966
1967         if (group_filter && !nh->is_group)
1968                 return true;
1969
1970         if (!dev_idx && !master_idx && !family)
1971                 return false;
1972
1973         if (nh->is_group)
1974                 return true;
1975
1976         nhi = rtnl_dereference(nh->nh_info);
1977         if (family && nhi->family != family)
1978                 return true;
1979
1980         dev = nhi->fib_nhc.nhc_dev;
1981         if (dev_idx && (!dev || dev->ifindex != dev_idx))
1982                 return true;
1983
1984         if (master_idx) {
1985                 struct net_device *master;
1986
1987                 if (!dev)
1988                         return true;
1989
1990                 master = netdev_master_upper_dev_get((struct net_device *)dev);
1991                 if (!master || master->ifindex != master_idx)
1992                         return true;
1993         }
1994
1995         return false;
1996 }
1997
1998 static int nh_valid_dump_req(const struct nlmsghdr *nlh, int *dev_idx,
1999                              int *master_idx, bool *group_filter,
2000                              bool *fdb_filter, struct netlink_callback *cb)
2001 {
2002         struct netlink_ext_ack *extack = cb->extack;
2003         struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
2004         struct nhmsg *nhm;
2005         int err;
2006         u32 idx;
2007
2008         err = nlmsg_parse(nlh, sizeof(*nhm), tb,
2009                           ARRAY_SIZE(rtm_nh_policy_dump) - 1,
2010                           rtm_nh_policy_dump, NULL);
2011         if (err < 0)
2012                 return err;
2013
2014         if (tb[NHA_OIF]) {
2015                 idx = nla_get_u32(tb[NHA_OIF]);
2016                 if (idx > INT_MAX) {
2017                         NL_SET_ERR_MSG(extack, "Invalid device index");
2018                         return -EINVAL;
2019                 }
2020                 *dev_idx = idx;
2021         }
2022         if (tb[NHA_MASTER]) {
2023                 idx = nla_get_u32(tb[NHA_MASTER]);
2024                 if (idx > INT_MAX) {
2025                         NL_SET_ERR_MSG(extack, "Invalid master device index");
2026                         return -EINVAL;
2027                 }
2028                 *master_idx = idx;
2029         }
2030         *group_filter = nla_get_flag(tb[NHA_GROUPS]);
2031         *fdb_filter = nla_get_flag(tb[NHA_FDB]);
2032
2033         nhm = nlmsg_data(nlh);
2034         if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
2035                 NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
2036                 return -EINVAL;
2037         }
2038
2039         return 0;
2040 }
2041
2042 /* rtnl */
2043 static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
2044 {
2045         bool group_filter = false, fdb_filter = false;
2046         struct nhmsg *nhm = nlmsg_data(cb->nlh);
2047         int dev_filter_idx = 0, master_idx = 0;
2048         struct net *net = sock_net(skb->sk);
2049         struct rb_root *root = &net->nexthop.rb_root;
2050         struct rb_node *node;
2051         int idx = 0, s_idx;
2052         int err;
2053
2054         err = nh_valid_dump_req(cb->nlh, &dev_filter_idx, &master_idx,
2055                                 &group_filter, &fdb_filter, cb);
2056         if (err < 0)
2057                 return err;
2058
2059         s_idx = cb->args[0];
2060         for (node = rb_first(root); node; node = rb_next(node)) {
2061                 struct nexthop *nh;
2062
2063                 if (idx < s_idx)
2064                         goto cont;
2065
2066                 nh = rb_entry(node, struct nexthop, rb_node);
2067                 if (nh_dump_filtered(nh, dev_filter_idx, master_idx,
2068                                      group_filter, nhm->nh_family))
2069                         goto cont;
2070
2071                 err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
2072                                    NETLINK_CB(cb->skb).portid,
2073                                    cb->nlh->nlmsg_seq, NLM_F_MULTI);
2074                 if (err < 0) {
2075                         if (likely(skb->len))
2076                                 goto out;
2077
2078                         goto out_err;
2079                 }
2080 cont:
2081                 idx++;
2082         }
2083
2084 out:
2085         err = skb->len;
2086 out_err:
2087         cb->args[0] = idx;
2088         cb->seq = net->nexthop.seq;
2089         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
2090
2091         return err;
2092 }
2093
2094 static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
2095 {
2096         unsigned int hash = nh_dev_hashfn(dev->ifindex);
2097         struct net *net = dev_net(dev);
2098         struct hlist_head *head = &net->nexthop.devhash[hash];
2099         struct hlist_node *n;
2100         struct nh_info *nhi;
2101
2102         hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
2103                 if (nhi->fib_nhc.nhc_dev == dev) {
2104                         if (nhi->family == AF_INET)
2105                                 fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
2106                                                    orig_mtu);
2107                 }
2108         }
2109 }
2110
2111 /* rtnl */
2112 static int nh_netdev_event(struct notifier_block *this,
2113                            unsigned long event, void *ptr)
2114 {
2115         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2116         struct netdev_notifier_info_ext *info_ext;
2117
2118         switch (event) {
2119         case NETDEV_DOWN:
2120         case NETDEV_UNREGISTER:
2121                 nexthop_flush_dev(dev);
2122                 break;
2123         case NETDEV_CHANGE:
2124                 if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
2125                         nexthop_flush_dev(dev);
2126                 break;
2127         case NETDEV_CHANGEMTU:
2128                 info_ext = ptr;
2129                 nexthop_sync_mtu(dev, info_ext->ext.mtu);
2130                 rt_cache_flush(dev_net(dev));
2131                 break;
2132         }
2133         return NOTIFY_DONE;
2134 }
2135
2136 static struct notifier_block nh_netdev_notifier = {
2137         .notifier_call = nh_netdev_event,
2138 };
2139
2140 static int nexthops_dump(struct net *net, struct notifier_block *nb,
2141                          struct netlink_ext_ack *extack)
2142 {
2143         struct rb_root *root = &net->nexthop.rb_root;
2144         struct rb_node *node;
2145         int err = 0;
2146
2147         for (node = rb_first(root); node; node = rb_next(node)) {
2148                 struct nexthop *nh;
2149
2150                 nh = rb_entry(node, struct nexthop, rb_node);
2151                 err = call_nexthop_notifier(nb, net, NEXTHOP_EVENT_REPLACE, nh,
2152                                             extack);
2153                 if (err)
2154                         break;
2155         }
2156
2157         return err;
2158 }
2159
2160 int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
2161                               struct netlink_ext_ack *extack)
2162 {
2163         int err;
2164
2165         rtnl_lock();
2166         err = nexthops_dump(net, nb, extack);
2167         if (err)
2168                 goto unlock;
2169         err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
2170                                                nb);
2171 unlock:
2172         rtnl_unlock();
2173         return err;
2174 }
2175 EXPORT_SYMBOL(register_nexthop_notifier);
2176
2177 int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
2178 {
2179         return blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
2180                                                   nb);
2181 }
2182 EXPORT_SYMBOL(unregister_nexthop_notifier);
2183
2184 void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
2185 {
2186         struct nexthop *nexthop;
2187
2188         rcu_read_lock();
2189
2190         nexthop = nexthop_find_by_id(net, id);
2191         if (!nexthop)
2192                 goto out;
2193
2194         nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
2195         if (offload)
2196                 nexthop->nh_flags |= RTNH_F_OFFLOAD;
2197         if (trap)
2198                 nexthop->nh_flags |= RTNH_F_TRAP;
2199
2200 out:
2201         rcu_read_unlock();
2202 }
2203 EXPORT_SYMBOL(nexthop_set_hw_flags);
2204
2205 static void __net_exit nexthop_net_exit(struct net *net)
2206 {
2207         rtnl_lock();
2208         flush_all_nexthops(net);
2209         rtnl_unlock();
2210         kfree(net->nexthop.devhash);
2211 }
2212
2213 static int __net_init nexthop_net_init(struct net *net)
2214 {
2215         size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;
2216
2217         net->nexthop.rb_root = RB_ROOT;
2218         net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
2219         if (!net->nexthop.devhash)
2220                 return -ENOMEM;
2221         BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);
2222
2223         return 0;
2224 }
2225
2226 static struct pernet_operations nexthop_net_ops = {
2227         .init = nexthop_net_init,
2228         .exit = nexthop_net_exit,
2229 };
2230
2231 static int __init nexthop_init(void)
2232 {
2233         register_pernet_subsys(&nexthop_net_ops);
2234
2235         register_netdevice_notifier(&nh_netdev_notifier);
2236
2237         rtnl_register(PF_UNSPEC, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2238         rtnl_register(PF_UNSPEC, RTM_DELNEXTHOP, rtm_del_nexthop, NULL, 0);
2239         rtnl_register(PF_UNSPEC, RTM_GETNEXTHOP, rtm_get_nexthop,
2240                       rtm_dump_nexthop, 0);
2241
2242         rtnl_register(PF_INET, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2243         rtnl_register(PF_INET, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
2244
2245         rtnl_register(PF_INET6, RTM_NEWNEXTHOP, rtm_new_nexthop, NULL, 0);
2246         rtnl_register(PF_INET6, RTM_GETNEXTHOP, NULL, rtm_dump_nexthop, 0);
2247
2248         return 0;
2249 }
2250 subsys_initcall(nexthop_init);