projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
f2fs: Provide a splice-read wrapper
[linux-block.git] / net / ipv6 / route.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * Linux INET6 implementation
4 * FIB front-end.
5 *
6 * Authors:
1ab1457c 7 * Pedro Roque <roque@di.fc.ul.pt>
1da177e4
LT		 8 */
9
10/* Changes:
11 *
12 * YOSHIFUJI Hideaki @USAGI
13 * reworked default router selection.
14 * - respect outgoing interface
15 * - select from (probably) reachable routers (i.e.
16 * routers in REACHABLE, STALE, DELAY or PROBE states).
17 * - always select the same router if it is (probably)
18 * reachable. otherwise, round-robin the list.
c0bece9f
YH		 19 * Ville Nuorvala
20 * Fixed routing subtrees.
1da177e4
LT		 21 */
22
f3213831
JP		 23#define pr_fmt(fmt) "IPv6: " fmt
24
4fc268d2 25#include <linux/capability.h>
1da177e4 26#include <linux/errno.h>
bc3b2d7f 27#include <linux/export.h>
1da177e4
LT		 28#include <linux/types.h>
29#include <linux/times.h>
30#include <linux/socket.h>
31#include <linux/sockios.h>
32#include <linux/net.h>
33#include <linux/route.h>
34#include <linux/netdevice.h>
35#include <linux/in6.h>
7bc570c8 36#include <linux/mroute6.h>
1da177e4 37#include <linux/init.h>
1da177e4 38#include <linux/if_arp.h>
1da177e4
LT		 39#include <linux/proc_fs.h>
40#include <linux/seq_file.h>
5b7c931d 41#include <linux/nsproxy.h>
5a0e3ad6 42#include <linux/slab.h>
35732d01 43#include <linux/jhash.h>
4785305c 44#include <linux/siphash.h>
457c4cbc 45#include <net/net_namespace.h>
1da177e4
LT		 46#include <net/snmp.h>
47#include <net/ipv6.h>
48#include <net/ip6_fib.h>
49#include <net/ip6_route.h>
50#include <net/ndisc.h>
51#include <net/addrconf.h>
52#include <net/tcp.h>
53#include <linux/rtnetlink.h>
54#include <net/dst.h>
904af04d 55#include <net/dst_metadata.h>
1da177e4 56#include <net/xfrm.h>
8d71740c 57#include <net/netevent.h>
21713ebc 58#include <net/netlink.h>
3c618c1d 59#include <net/rtnh.h>
19e42e45 60#include <net/lwtunnel.h>
904af04d 61#include <net/ip_tunnels.h>
ca254490 62#include <net/l3mdev.h>
eacb9384 63#include <net/ip.h>
7c0f6ba6 64#include <linux/uaccess.h>
951cf368 65#include <linux/btf_ids.h>
1da177e4
LT		 66
67#ifdef CONFIG_SYSCTL
68#include <linux/sysctl.h>
69#endif
70
30d444d3
DA		 71static int ip6_rt_type_to_error(u8 fib6_type);
72
73#define CREATE_TRACE_POINTS
74#include <trace/events/fib6.h>
75EXPORT_TRACEPOINT_SYMBOL_GPL(fib6_table_lookup);
76#undef CREATE_TRACE_POINTS
77
afc154e9 78enum rt6_nud_state {
7e980569
JB		 79 RT6_NUD_FAIL_HARD = -3,
80 RT6_NUD_FAIL_PROBE = -2,
81 RT6_NUD_FAIL_DO_RR = -1,
afc154e9
HFS		 82 RT6_NUD_SUCCEED = 1
83};
84
bbd807df
BV		 85INDIRECT_CALLABLE_SCOPE
86struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
0dbaee3b 87static unsigned int ip6_default_advmss(const struct dst_entry *dst);
f67fbeae
BV		 88INDIRECT_CALLABLE_SCOPE
89unsigned int ip6_mtu(const struct dst_entry *dst);
1da177e4
LT		 90static struct dst_entry *ip6_negative_advice(struct dst_entry *);
91static void ip6_dst_destroy(struct dst_entry *);
92static void ip6_dst_ifdown(struct dst_entry *,
93 struct net_device *dev, int how);
af6d1034 94static void ip6_dst_gc(struct dst_ops *ops);
1da177e4
LT		 95
96static int ip6_pkt_discard(struct sk_buff *skb);
ede2059d 97static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
7150aede 98static int ip6_pkt_prohibit(struct sk_buff *skb);
ede2059d 99static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb);
1da177e4 100static void ip6_link_failure(struct sk_buff *skb);
6700c270 101static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
bd085ef6
HL		 102 struct sk_buff *skb, u32 mtu,
103 bool confirm_neigh);
6700c270
DM		 104static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk,
105 struct sk_buff *skb);
702cea56
DA		 106static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
107 int strict);
a1b7a1f0 108static size_t rt6_nlmsg_size(struct fib6_info *f6i);
d4ead6b3 109static int rt6_fill_node(struct net *net, struct sk_buff *skb,
8d1c802b 110 struct fib6_info *rt, struct dst_entry *dst,
d4ead6b3 111 struct in6_addr *dest, struct in6_addr *src,
16a16cd3
DA		 112 int iif, int type, u32 portid, u32 seq,
113 unsigned int flags);
7e4b5128 114static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
510e2ced
WW		 115 const struct in6_addr *daddr,
116 const struct in6_addr *saddr);
1da177e4 117
70ceb4f5 118#ifdef CONFIG_IPV6_ROUTE_INFO
8d1c802b 119static struct fib6_info *rt6_add_route_info(struct net *net,
b71d1d42 120 const struct in6_addr *prefix, int prefixlen,
830218c1
DA		 121 const struct in6_addr *gwaddr,
122 struct net_device *dev,
95c96174 123 unsigned int pref);
8d1c802b 124static struct fib6_info *rt6_get_route_info(struct net *net,
b71d1d42 125 const struct in6_addr *prefix, int prefixlen,
830218c1
DA		 126 const struct in6_addr *gwaddr,
127 struct net_device *dev);
70ceb4f5
YH		 128#endif
129
8d0b94af
MKL		 130struct uncached_list {
131 spinlock_t lock;
132 struct list_head head;
ba55ef81 133 struct list_head quarantine;
8d0b94af
MKL		 134};
135
136static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list);
137
510c321b 138void rt6_uncached_list_add(struct rt6_info *rt)
8d0b94af
MKL		 139{
140 struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list);
141
d288a162 142 rt->dst.rt_uncached_list = ul;
8d0b94af
MKL		 143
144 spin_lock_bh(&ul->lock);
d288a162 145 list_add_tail(&rt->dst.rt_uncached, &ul->head);
8d0b94af
MKL		 146 spin_unlock_bh(&ul->lock);
147}
148
510c321b 149void rt6_uncached_list_del(struct rt6_info *rt)
8d0b94af 150{
d288a162
WG		 151 if (!list_empty(&rt->dst.rt_uncached)) {
152 struct uncached_list *ul = rt->dst.rt_uncached_list;
8d0b94af
MKL		 153
154 spin_lock_bh(&ul->lock);
d288a162 155 list_del_init(&rt->dst.rt_uncached);
8d0b94af
MKL		 156 spin_unlock_bh(&ul->lock);
157 }
158}
159
e5f80fcf 160static void rt6_uncached_list_flush_dev(struct net_device *dev)
8d0b94af 161{
8d0b94af
MKL		 162 int cpu;
163
164 for_each_possible_cpu(cpu) {
165 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
ba55ef81
ED		 166 struct rt6_info *rt, *safe;
167
168 if (list_empty(&ul->head))
169 continue;
8d0b94af
MKL		 170
171 spin_lock_bh(&ul->lock);
d288a162 172 list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) {
8d0b94af
MKL		 173 struct inet6_dev *rt_idev = rt->rt6i_idev;
174 struct net_device *rt_dev = rt->dst.dev;
ba55ef81 175 bool handled = false;
8d0b94af 176
e332bc67 177 if (rt_idev->dev == dev) {
e5f80fcf 178 rt->rt6i_idev = in6_dev_get(blackhole_netdev);
8d0b94af 179 in6_dev_put(rt_idev);
ba55ef81 180 handled = true;
8d0b94af
MKL		 181 }
182
e332bc67 183 if (rt_dev == dev) {
8d7017fd 184 rt->dst.dev = blackhole_netdev;
d62607c3
JK		 185 netdev_ref_replace(rt_dev, blackhole_netdev,
186 &rt->dst.dev_tracker,
187 GFP_ATOMIC);
ba55ef81 188 handled = true;
8d0b94af 189 }
ba55ef81 190 if (handled)
d288a162 191 list_move(&rt->dst.rt_uncached,
ba55ef81 192 &ul->quarantine);
8d0b94af
MKL		 193 }
194 spin_unlock_bh(&ul->lock);
195 }
196}
197
f8a1b43b 198static inline const void *choose_neigh_daddr(const struct in6_addr *p,
f894cbf8
DM		 199 struct sk_buff *skb,
200 const void *daddr)
39232973 201{
a7563f34 202 if (!ipv6_addr_any(p))
39232973 203 return (const void *) p;
f894cbf8
DM		 204 else if (skb)
205 return &ipv6_hdr(skb)->daddr;
39232973
DM		 206 return daddr;
207}
208
f8a1b43b
DA		 209struct neighbour *ip6_neigh_lookup(const struct in6_addr *gw,
210 struct net_device *dev,
211 struct sk_buff *skb,
212 const void *daddr)
d3aaeb38 213{
39232973
DM		 214 struct neighbour *n;
215
f8a1b43b
DA		 216 daddr = choose_neigh_daddr(gw, skb, daddr);
217 n = __ipv6_neigh_lookup(dev, daddr);
f83c7790
DM		 218 if (n)
219 return n;
7adf3246
SB		 220
221 n = neigh_create(&nd_tbl, daddr, dev);
222 return IS_ERR(n) ? NULL : n;
f8a1b43b
DA		 223}
224
225static struct neighbour *ip6_dst_neigh_lookup(const struct dst_entry *dst,
226 struct sk_buff *skb,
227 const void *daddr)
228{
229 const struct rt6_info *rt = container_of(dst, struct rt6_info, dst);
230
2c6b55f4
ND		 231 return ip6_neigh_lookup(rt6_nexthop(rt, &in6addr_any),
232 dst->dev, skb, daddr);
f83c7790
DM		 233}
234
63fca65d
JA		 235static void ip6_confirm_neigh(const struct dst_entry *dst, const void *daddr)
236{
237 struct net_device *dev = dst->dev;
238 struct rt6_info *rt = (struct rt6_info *)dst;
239
cbfd6891 240 daddr = choose_neigh_daddr(rt6_nexthop(rt, &in6addr_any), NULL, daddr);
63fca65d
JA		 241 if (!daddr)
242 return;
243 if (dev->flags & (IFF_NOARP | IFF_LOOPBACK))
244 return;
245 if (ipv6_addr_is_multicast((const struct in6_addr *)daddr))
246 return;
247 __ipv6_confirm_neigh(dev, daddr);
248}
249
9a7ec3a9 250static struct dst_ops ip6_dst_ops_template = {
1da177e4 251 .family = AF_INET6,
1da177e4
LT		 252 .gc = ip6_dst_gc,
253 .gc_thresh = 1024,
254 .check = ip6_dst_check,
0dbaee3b 255 .default_advmss = ip6_default_advmss,
ebb762f2 256 .mtu = ip6_mtu,
d4ead6b3 257 .cow_metrics = dst_cow_metrics_generic,
1da177e4
LT		 258 .destroy = ip6_dst_destroy,
259 .ifdown = ip6_dst_ifdown,
260 .negative_advice = ip6_negative_advice,
261 .link_failure = ip6_link_failure,
262 .update_pmtu = ip6_rt_update_pmtu,
6e157b6a 263 .redirect = rt6_do_redirect,
9f8955cc 264 .local_out = __ip6_local_out,
f8a1b43b 265 .neigh_lookup = ip6_dst_neigh_lookup,
63fca65d 266 .confirm_neigh = ip6_confirm_neigh,
1da177e4
LT		 267};
268
14e50e57 269static struct dst_ops ip6_dst_blackhole_ops = {
c4c877b2
DB		 270 .family = AF_INET6,
271 .default_advmss = ip6_default_advmss,
272 .neigh_lookup = ip6_dst_neigh_lookup,
273 .check = ip6_dst_check,
274 .destroy = ip6_dst_destroy,
275 .cow_metrics = dst_cow_metrics_generic,
276 .update_pmtu = dst_blackhole_update_pmtu,
277 .redirect = dst_blackhole_redirect,
278 .mtu = dst_blackhole_mtu,
14e50e57
DM		 279};
280
62fa8a84 281static const u32 ip6_template_metrics[RTAX_MAX] = {
14edd87d 282 [RTAX_HOPLIMIT - 1] = 0,
62fa8a84
DM		 283};
284
8d1c802b 285static const struct fib6_info fib6_null_entry_template = {
93c2fb25
DA		 286 .fib6_flags = (RTF_REJECT | RTF_NONEXTHOP),
287 .fib6_protocol = RTPROT_KERNEL,
288 .fib6_metric = ~(u32)0,
f05713e0 289 .fib6_ref = REFCOUNT_INIT(1),
421842ed
DA		 290 .fib6_type = RTN_UNREACHABLE,
291 .fib6_metrics = (struct dst_metrics *)&dst_default_metrics,
292};
293
fb0af4c7 294static const struct rt6_info ip6_null_entry_template = {
d8d1f30b 295 .dst = {
bc9d3a9f 296 .__rcuref = RCUREF_INIT(1),
d8d1f30b 297 .__use = 1,
2c20cbd7 298 .obsolete = DST_OBSOLETE_FORCE_CHK,
d8d1f30b 299 .error = -ENETUNREACH,
d8d1f30b
CG		 300 .input = ip6_pkt_discard,
301 .output = ip6_pkt_discard_out,
1da177e4
LT		 302 },
303 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
1da177e4
LT		 304};
305
101367c2
TG		 306#ifdef CONFIG_IPV6_MULTIPLE_TABLES
307
fb0af4c7 308static const struct rt6_info ip6_prohibit_entry_template = {
d8d1f30b 309 .dst = {
bc9d3a9f 310 .__rcuref = RCUREF_INIT(1),
d8d1f30b 311 .__use = 1,
2c20cbd7 312 .obsolete = DST_OBSOLETE_FORCE_CHK,
d8d1f30b 313 .error = -EACCES,
d8d1f30b
CG		 314 .input = ip6_pkt_prohibit,
315 .output = ip6_pkt_prohibit_out,
101367c2
TG		 316 },
317 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
101367c2
TG		 318};
319
fb0af4c7 320static const struct rt6_info ip6_blk_hole_entry_template = {
d8d1f30b 321 .dst = {
bc9d3a9f 322 .__rcuref = RCUREF_INIT(1),
d8d1f30b 323 .__use = 1,
2c20cbd7 324 .obsolete = DST_OBSOLETE_FORCE_CHK,
d8d1f30b 325 .error = -EINVAL,
d8d1f30b 326 .input = dst_discard,
ede2059d 327 .output = dst_discard_out,
101367c2
TG		 328 },
329 .rt6i_flags = (RTF_REJECT | RTF_NONEXTHOP),
101367c2
TG		 330};
331
332#endif
333
ebfa45f0
MKL		 334static void rt6_info_init(struct rt6_info *rt)
335{
8f2a83b4 336 memset_after(rt, 0, dst);
ebfa45f0
MKL		 337}
338
1da177e4 339/* allocate dst with ip6_dst_ops */
93531c67
DA		 340struct rt6_info *ip6_dst_alloc(struct net *net, struct net_device *dev,
341 int flags)
1da177e4 342{
97bab73f 343 struct rt6_info *rt = dst_alloc(&net->ipv6.ip6_dst_ops, dev,
b2a9c0ed 344 1, DST_OBSOLETE_FORCE_CHK, flags);
cf911662 345
81eb8447 346 if (rt) {
ebfa45f0 347 rt6_info_init(rt);
81eb8447
WW		 348 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
349 }
8104891b 350
cf911662 351 return rt;
1da177e4 352}
9ab179d8 353EXPORT_SYMBOL(ip6_dst_alloc);
d52d3997 354
1da177e4
LT		 355static void ip6_dst_destroy(struct dst_entry *dst)
356{
357 struct rt6_info *rt = (struct rt6_info *)dst;
a68886a6 358 struct fib6_info *from;
8d0b94af 359 struct inet6_dev *idev;
1da177e4 360
1620a336 361 ip_dst_metrics_put(dst);
8d0b94af
MKL		 362 rt6_uncached_list_del(rt);
363
364 idev = rt->rt6i_idev;
38308473 365 if (idev) {
1da177e4
LT		 366 rt->rt6i_idev = NULL;
367 in6_dev_put(idev);
1ab1457c 368 }
1716a961 369
0e233874 370 from = xchg((__force struct fib6_info **)&rt->from, NULL);
93531c67 371 fib6_info_release(from);
b3419363
DM		 372}
373
1da177e4
LT		 374static void ip6_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
375 int how)
376{
377 struct rt6_info *rt = (struct rt6_info *)dst;
378 struct inet6_dev *idev = rt->rt6i_idev;
379
e5f80fcf
ED		 380 if (idev && idev->dev != blackhole_netdev) {
381 struct inet6_dev *blackhole_idev = in6_dev_get(blackhole_netdev);
382
383 if (blackhole_idev) {
384 rt->rt6i_idev = blackhole_idev;
e5645f51 385 in6_dev_put(idev);
97cac082 386 }
1da177e4
LT		 387 }
388}
389
5973fb1e
MKL		 390static bool __rt6_check_expired(const struct rt6_info *rt)
391{
392 if (rt->rt6i_flags & RTF_EXPIRES)
393 return time_after(jiffies, rt->dst.expires);
394 else
395 return false;
396}
397
a50feda5 398static bool rt6_check_expired(const struct rt6_info *rt)
1da177e4 399{
a68886a6
DA		 400 struct fib6_info *from;
401
402 from = rcu_dereference(rt->from);
403
1716a961
G		 404 if (rt->rt6i_flags & RTF_EXPIRES) {
405 if (time_after(jiffies, rt->dst.expires))
a50feda5 406 return true;
a68886a6 407 } else if (from) {
1e2ea8ad 408 return rt->dst.obsolete != DST_OBSOLETE_FORCE_CHK ||
a68886a6 409 fib6_check_expired(from);
1716a961 410 }
a50feda5 411 return false;
1da177e4
LT		 412}
413
b1d40991
DA		 414void fib6_select_path(const struct net *net, struct fib6_result *res,
415 struct flowi6 *fl6, int oif, bool have_oif_match,
416 const struct sk_buff *skb, int strict)
51ebd318 417{
8d1c802b 418 struct fib6_info *sibling, *next_sibling;
b1d40991
DA		 419 struct fib6_info *match = res->f6i;
420
34fe5a1c 421 if (!match->nh && (!match->fib6_nsiblings || have_oif_match))
b1d40991 422 goto out;
51ebd318 423
34fe5a1c
DA		 424 if (match->nh && have_oif_match && res->nh)
425 return;
426
b673d6cc
JS		 427 /* We might have already computed the hash for ICMPv6 errors. In such
428 * case it will always be non-zero. Otherwise now is the time to do it.
429 */
f88d8ea6
DA		 430 if (!fl6->mp_hash &&
431 (!match->nh || nexthop_is_multipath(match->nh)))
b4bac172 432 fl6->mp_hash = rt6_multipath_hash(net, fl6, skb, NULL);
b673d6cc 433
f88d8ea6
DA		 434 if (unlikely(match->nh)) {
435 nexthop_path_fib6_result(res, fl6->mp_hash);
436 return;
437 }
438
1cf844c7 439 if (fl6->mp_hash <= atomic_read(&match->fib6_nh->fib_nh_upper_bound))
b1d40991 440 goto out;
3d709f69 441
93c2fb25
DA		 442 list_for_each_entry_safe(sibling, next_sibling, &match->fib6_siblings,
443 fib6_siblings) {
1cf844c7 444 const struct fib6_nh *nh = sibling->fib6_nh;
5e670d84
DA		 445 int nh_upper_bound;
446
702cea56 447 nh_upper_bound = atomic_read(&nh->fib_nh_upper_bound);
5e670d84 448 if (fl6->mp_hash > nh_upper_bound)
3d709f69 449 continue;
702cea56 450 if (rt6_score_route(nh, sibling->fib6_flags, oif, strict) < 0)
3d709f69
IS		 451 break;
452 match = sibling;
453 break;
454 }
455
b1d40991
DA		 456out:
457 res->f6i = match;
1cf844c7 458 res->nh = match->fib6_nh;
51ebd318
ND		 459}
460
1da177e4 461/*
66f5d6ce 462 * Route lookup. rcu_read_lock() should be held.
1da177e4
LT		 463 */
464
0c59d006
DA		 465static bool __rt6_device_match(struct net *net, const struct fib6_nh *nh,
466 const struct in6_addr *saddr, int oif, int flags)
467{
468 const struct net_device *dev;
469
470 if (nh->fib_nh_flags & RTNH_F_DEAD)
471 return false;
472
473 dev = nh->fib_nh_dev;
474 if (oif) {
475 if (dev->ifindex == oif)
476 return true;
477 } else {
478 if (ipv6_chk_addr(net, saddr, dev,
479 flags & RT6_LOOKUP_F_IFACE))
480 return true;
481 }
482
483 return false;
484}
485
962b6803
DA		 486struct fib6_nh_dm_arg {
487 struct net *net;
488 const struct in6_addr *saddr;
489 int oif;
490 int flags;
491 struct fib6_nh *nh;
492};
493
494static int __rt6_nh_dev_match(struct fib6_nh *nh, void *_arg)
495{
496 struct fib6_nh_dm_arg *arg = _arg;
497
498 arg->nh = nh;
499 return __rt6_device_match(arg->net, nh, arg->saddr, arg->oif,
500 arg->flags);
501}
502
503/* returns fib6_nh from nexthop or NULL */
504static struct fib6_nh *rt6_nh_dev_match(struct net *net, struct nexthop *nh,
505 struct fib6_result *res,
506 const struct in6_addr *saddr,
507 int oif, int flags)
508{
509 struct fib6_nh_dm_arg arg = {
510 .net = net,
511 .saddr = saddr,
512 .oif = oif,
513 .flags = flags,
514 };
515
516 if (nexthop_is_blackhole(nh))
517 return NULL;
518
519 if (nexthop_for_each_fib6_nh(nh, __rt6_nh_dev_match, &arg))
520 return arg.nh;
521
522 return NULL;
523}
524
75ef7389
DA		 525static void rt6_device_match(struct net *net, struct fib6_result *res,
526 const struct in6_addr *saddr, int oif, int flags)
1da177e4 527{
75ef7389
DA		 528 struct fib6_info *f6i = res->f6i;
529 struct fib6_info *spf6i;
530 struct fib6_nh *nh;
1da177e4 531
75ef7389 532 if (!oif && ipv6_addr_any(saddr)) {
f88d8ea6
DA		 533 if (unlikely(f6i->nh)) {
534 nh = nexthop_fib6_nh(f6i->nh);
535 if (nexthop_is_blackhole(f6i->nh))
536 goto out_blackhole;
537 } else {
538 nh = f6i->fib6_nh;
539 }
7d21fec9
DA		 540 if (!(nh->fib_nh_flags & RTNH_F_DEAD))
541 goto out;
75ef7389 542 }
dd3abc4e 543
75ef7389 544 for (spf6i = f6i; spf6i; spf6i = rcu_dereference(spf6i->fib6_next)) {
962b6803
DA		 545 bool matched = false;
546
547 if (unlikely(spf6i->nh)) {
548 nh = rt6_nh_dev_match(net, spf6i->nh, res, saddr,
549 oif, flags);
550 if (nh)
551 matched = true;
552 } else {
553 nh = spf6i->fib6_nh;
554 if (__rt6_device_match(net, nh, saddr, oif, flags))
555 matched = true;
556 }
557 if (matched) {
75ef7389 558 res->f6i = spf6i;
7d21fec9 559 goto out;
75ef7389 560 }
dd3abc4e 561 }
1da177e4 562
75ef7389
DA		 563 if (oif && flags & RT6_LOOKUP_F_IFACE) {
564 res->f6i = net->ipv6.fib6_null_entry;
1cf844c7 565 nh = res->f6i->fib6_nh;
7d21fec9 566 goto out;
75ef7389 567 }
8067bb8c 568
f88d8ea6
DA		 569 if (unlikely(f6i->nh)) {
570 nh = nexthop_fib6_nh(f6i->nh);
571 if (nexthop_is_blackhole(f6i->nh))
572 goto out_blackhole;
573 } else {
574 nh = f6i->fib6_nh;
575 }
576
7d21fec9 577 if (nh->fib_nh_flags & RTNH_F_DEAD) {
75ef7389 578 res->f6i = net->ipv6.fib6_null_entry;
1cf844c7 579 nh = res->f6i->fib6_nh;
75ef7389 580 }
7d21fec9
DA		 581out:
582 res->nh = nh;
583 res->fib6_type = res->f6i->fib6_type;
584 res->fib6_flags = res->f6i->fib6_flags;
f88d8ea6
DA		 585 return;
586
587out_blackhole:
588 res->fib6_flags |= RTF_REJECT;
589 res->fib6_type = RTN_BLACKHOLE;
590 res->nh = nh;
1da177e4
LT		 591}
592
27097255 593#ifdef CONFIG_IPV6_ROUTER_PREF
c2f17e82
HFS		 594struct __rt6_probe_work {
595 struct work_struct work;
596 struct in6_addr target;
597 struct net_device *dev;
fb67510b 598 netdevice_tracker dev_tracker;
c2f17e82
HFS		 599};
600
601static void rt6_probe_deferred(struct work_struct *w)
602{
603 struct in6_addr mcaddr;
604 struct __rt6_probe_work *work =
605 container_of(w, struct __rt6_probe_work, work);
606
607 addrconf_addr_solict_mult(&work->target, &mcaddr);
adc176c5 608 ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0);
d62607c3 609 netdev_put(work->dev, &work->dev_tracker);
662f5533 610 kfree(work);
c2f17e82
HFS		 611}
612
cc3a86c8 613static void rt6_probe(struct fib6_nh *fib6_nh)
27097255 614{
f547fac6 615 struct __rt6_probe_work *work = NULL;
5e670d84 616 const struct in6_addr *nh_gw;
1bef4c22 617 unsigned long last_probe;
f2c31e32 618 struct neighbour *neigh;
5e670d84 619 struct net_device *dev;
f547fac6 620 struct inet6_dev *idev;
5e670d84 621
27097255
YH		 622 /*
623 * Okay, this does not seem to be appropriate
624 * for now, however, we need to check if it
625 * is really so; aka Router Reachability Probing.
626 *
627 * Router Reachability Probe MUST be rate-limited
628 * to no more than one per minute.
629 */
004b3942 630 if (!fib6_nh->fib_nh_gw_family)
7ff74a59 631 return;
5e670d84 632
cc3a86c8
DA		 633 nh_gw = &fib6_nh->fib_nh_gw6;
634 dev = fib6_nh->fib_nh_dev;
09eed119 635 rcu_read_lock();
1bef4c22 636 last_probe = READ_ONCE(fib6_nh->last_probe);
f547fac6 637 idev = __in6_dev_get(dev);
5e670d84 638 neigh = __ipv6_neigh_lookup_noref(dev, nh_gw);
2152caea 639 if (neigh) {
b071af52 640 if (READ_ONCE(neigh->nud_state) & NUD_VALID)
8d6c31bf
MKL		 641 goto out;
642
09eed119 643 write_lock_bh(&neigh->lock);
990edb42
MKL		 644 if (!(neigh->nud_state & NUD_VALID) &&
645 time_after(jiffies,
dcd1f572 646 neigh->updated + idev->cnf.rtr_probe_interval)) {
990edb42
MKL		 647 work = kmalloc(sizeof(*work), GFP_ATOMIC);
648 if (work)
649 __neigh_set_probe_once(neigh);
c2f17e82 650 }
09eed119 651 write_unlock_bh(&neigh->lock);
1bef4c22 652 } else if (time_after(jiffies, last_probe +
f547fac6 653 idev->cnf.rtr_probe_interval)) {
990edb42 654 work = kmalloc(sizeof(*work), GFP_ATOMIC);
f2c31e32 655 }
990edb42 656
1bef4c22
ED		 657 if (!work || cmpxchg(&fib6_nh->last_probe,
658 last_probe, jiffies) != last_probe) {
659 kfree(work);
660 } else {
990edb42 661 INIT_WORK(&work->work, rt6_probe_deferred);
5e670d84 662 work->target = *nh_gw;
d62607c3 663 netdev_hold(dev, &work->dev_tracker, GFP_ATOMIC);
5e670d84 664 work->dev = dev;
990edb42
MKL		 665 schedule_work(&work->work);
666 }
667
8d6c31bf 668out:
09eed119 669 rcu_read_unlock();
27097255
YH		 670}
671#else
cc3a86c8 672static inline void rt6_probe(struct fib6_nh *fib6_nh)
27097255 673{
27097255
YH		 674}
675#endif
676
1da177e4 677/*
554cfb7e 678 * Default Router Selection (RFC 2461 6.3.6)
1da177e4 679 */
1ba9a895 680static enum rt6_nud_state rt6_check_neigh(const struct fib6_nh *fib6_nh)
1da177e4 681{
afc154e9 682 enum rt6_nud_state ret = RT6_NUD_FAIL_HARD;
5e670d84 683 struct neighbour *neigh;
f2c31e32 684
09eed119 685 rcu_read_lock();
1ba9a895
DA		 686 neigh = __ipv6_neigh_lookup_noref(fib6_nh->fib_nh_dev,
687 &fib6_nh->fib_nh_gw6);
145a3621 688 if (neigh) {
c486640a
ED		 689 u8 nud_state = READ_ONCE(neigh->nud_state);
690
691 if (nud_state & NUD_VALID)
afc154e9 692 ret = RT6_NUD_SUCCEED;
398bcbeb 693#ifdef CONFIG_IPV6_ROUTER_PREF
c486640a 694 else if (!(nud_state & NUD_FAILED))
afc154e9 695 ret = RT6_NUD_SUCCEED;
7e980569
JB		 696 else
697 ret = RT6_NUD_FAIL_PROBE;
398bcbeb 698#endif
afc154e9
HFS		 699 } else {
700 ret = IS_ENABLED(CONFIG_IPV6_ROUTER_PREF) ?
7e980569 701 RT6_NUD_SUCCEED : RT6_NUD_FAIL_DO_RR;
a5a81f0b 702 }
09eed119 703 rcu_read_unlock();
145a3621 704
a5a81f0b 705 return ret;
1da177e4
LT		 706}
707
702cea56
DA		 708static int rt6_score_route(const struct fib6_nh *nh, u32 fib6_flags, int oif,
709 int strict)
1da177e4 710{
6e1809a5
DA		 711 int m = 0;
712
713 if (!oif || nh->fib_nh_dev->ifindex == oif)
714 m = 2;
1ab1457c 715
77d16f45 716 if (!m && (strict & RT6_LOOKUP_F_IFACE))
afc154e9 717 return RT6_NUD_FAIL_HARD;
ebacaaa0 718#ifdef CONFIG_IPV6_ROUTER_PREF
702cea56 719 m |= IPV6_DECODE_PREF(IPV6_EXTRACT_PREF(fib6_flags)) << 2;
ebacaaa0 720#endif
1ba9a895 721 if ((strict & RT6_LOOKUP_F_REACHABLE) &&
702cea56 722 !(fib6_flags & RTF_NONEXTHOP) && nh->fib_nh_gw_family) {
1ba9a895 723 int n = rt6_check_neigh(nh);
afc154e9
HFS		 724 if (n < 0)
725 return n;
726 }
554cfb7e
YH		 727 return m;
728}
729
28679ed1
DA		 730static bool find_match(struct fib6_nh *nh, u32 fib6_flags,
731 int oif, int strict, int *mpri, bool *do_rr)
554cfb7e 732{
afc154e9 733 bool match_do_rr = false;
28679ed1
DA		 734 bool rc = false;
735 int m;
35103d11 736
28679ed1 737 if (nh->fib_nh_flags & RTNH_F_DEAD)
8067bb8c
IS		 738 goto out;
739
28679ed1
DA		 740 if (ip6_ignore_linkdown(nh->fib_nh_dev) &&
741 nh->fib_nh_flags & RTNH_F_LINKDOWN &&
d5d32e4b 742 !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE))
35103d11 743 goto out;
f11e6659 744
28679ed1 745 m = rt6_score_route(nh, fib6_flags, oif, strict);
7e980569 746 if (m == RT6_NUD_FAIL_DO_RR) {
afc154e9
HFS		 747 match_do_rr = true;
748 m = 0; /* lowest valid score */
7e980569 749 } else if (m == RT6_NUD_FAIL_HARD) {
f11e6659 750 goto out;
afc154e9
HFS		 751 }
752
753 if (strict & RT6_LOOKUP_F_REACHABLE)
28679ed1 754 rt6_probe(nh);
f11e6659 755
7e980569 756 /* note that m can be RT6_NUD_FAIL_PROBE at this point */
f11e6659 757 if (m > *mpri) {
afc154e9 758 *do_rr = match_do_rr;
f11e6659 759 *mpri = m;
28679ed1 760 rc = true;
f11e6659 761 }
f11e6659 762out:
28679ed1 763 return rc;
f11e6659
DM		 764}
765
17a5984e
DA		 766struct fib6_nh_frl_arg {
767 u32 flags;
768 int oif;
769 int strict;
770 int *mpri;
771 bool *do_rr;
772 struct fib6_nh *nh;
773};
774
775static int rt6_nh_find_match(struct fib6_nh *nh, void *_arg)
776{
777 struct fib6_nh_frl_arg *arg = _arg;
778
779 arg->nh = nh;
780 return find_match(nh, arg->flags, arg->oif, arg->strict,
781 arg->mpri, arg->do_rr);
782}
783
b7bc4b6a 784static void __find_rr_leaf(struct fib6_info *f6i_start,
30c15f03 785 struct fib6_info *nomatch, u32 metric,
b7bc4b6a 786 struct fib6_result *res, struct fib6_info **cont,
30c15f03 787 int oif, int strict, bool *do_rr, int *mpri)
f11e6659 788{
b7bc4b6a 789 struct fib6_info *f6i;
1da177e4 790
b7bc4b6a
DA		 791 for (f6i = f6i_start;
792 f6i && f6i != nomatch;
793 f6i = rcu_dereference(f6i->fib6_next)) {
17a5984e 794 bool matched = false;
30c15f03
DA		 795 struct fib6_nh *nh;
796
b7bc4b6a
DA		 797 if (cont && f6i->fib6_metric != metric) {
798 *cont = f6i;
30c15f03 799 return;
9fbdcfaf
SK		 800 }
801
b7bc4b6a 802 if (fib6_check_expired(f6i))
28679ed1
DA		 803 continue;
804
17a5984e
DA		 805 if (unlikely(f6i->nh)) {
806 struct fib6_nh_frl_arg arg = {
807 .flags = f6i->fib6_flags,
808 .oif = oif,
809 .strict = strict,
810 .mpri = mpri,
811 .do_rr = do_rr
812 };
813
814 if (nexthop_is_blackhole(f6i->nh)) {
815 res->fib6_flags = RTF_REJECT;
816 res->fib6_type = RTN_BLACKHOLE;
817 res->f6i = f6i;
818 res->nh = nexthop_fib6_nh(f6i->nh);
819 return;
820 }
821 if (nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_find_match,
822 &arg)) {
823 matched = true;
824 nh = arg.nh;
825 }
826 } else {
827 nh = f6i->fib6_nh;
828 if (find_match(nh, f6i->fib6_flags, oif, strict,
829 mpri, do_rr))
830 matched = true;
831 }
832 if (matched) {
b7bc4b6a
DA		 833 res->f6i = f6i;
834 res->nh = nh;
7d21fec9
DA		 835 res->fib6_flags = f6i->fib6_flags;
836 res->fib6_type = f6i->fib6_type;
b7bc4b6a 837 }
9fbdcfaf 838 }
30c15f03 839}
9fbdcfaf 840
b7bc4b6a
DA		 841static void find_rr_leaf(struct fib6_node *fn, struct fib6_info *leaf,
842 struct fib6_info *rr_head, int oif, int strict,
843 bool *do_rr, struct fib6_result *res)
30c15f03 844{
b7bc4b6a
DA		 845 u32 metric = rr_head->fib6_metric;
846 struct fib6_info *cont = NULL;
30c15f03 847 int mpri = -1;
9fbdcfaf 848
b7bc4b6a 849 __find_rr_leaf(rr_head, NULL, metric, res, &cont,
30c15f03 850 oif, strict, do_rr, &mpri);
28679ed1 851
b7bc4b6a 852 __find_rr_leaf(leaf, rr_head, metric, res, &cont,
30c15f03 853 oif, strict, do_rr, &mpri);
9fbdcfaf 854
b7bc4b6a
DA		 855 if (res->f6i || !cont)
856 return;
9fbdcfaf 857
b7bc4b6a 858 __find_rr_leaf(cont, NULL, metric, res, NULL,
30c15f03 859 oif, strict, do_rr, &mpri);
f11e6659 860}
1da177e4 861
b7bc4b6a
DA		 862static void rt6_select(struct net *net, struct fib6_node *fn, int oif,
863 struct fib6_result *res, int strict)
f11e6659 864{
8d1c802b 865 struct fib6_info *leaf = rcu_dereference(fn->leaf);
b7bc4b6a 866 struct fib6_info *rt0;
afc154e9 867 bool do_rr = false;
17ecf590 868 int key_plen;
1da177e4 869
b7bc4b6a
DA		 870 /* make sure this function or its helpers sets f6i */
871 res->f6i = NULL;
872
421842ed 873 if (!leaf || leaf == net->ipv6.fib6_null_entry)
b7bc4b6a 874 goto out;
8d1040e8 875
66f5d6ce 876 rt0 = rcu_dereference(fn->rr_ptr);
f11e6659 877 if (!rt0)
66f5d6ce 878 rt0 = leaf;
1da177e4 879
17ecf590
WW		 880 /* Double check to make sure fn is not an intermediate node
881 * and fn->leaf does not points to its child's leaf
882 * (This might happen if all routes under fn are deleted from
883 * the tree and fib6_repair_tree() is called on the node.)
884 */
93c2fb25 885 key_plen = rt0->fib6_dst.plen;
17ecf590 886#ifdef CONFIG_IPV6_SUBTREES
93c2fb25
DA		 887 if (rt0->fib6_src.plen)
888 key_plen = rt0->fib6_src.plen;
17ecf590
WW		 889#endif
890 if (fn->fn_bit != key_plen)
b7bc4b6a 891 goto out;
1da177e4 892
b7bc4b6a 893 find_rr_leaf(fn, leaf, rt0, oif, strict, &do_rr, res);
afc154e9 894 if (do_rr) {
8fb11a9a 895 struct fib6_info *next = rcu_dereference(rt0->fib6_next);
f11e6659 896
554cfb7e 897 /* no entries matched; do round-robin */
93c2fb25 898 if (!next || next->fib6_metric != rt0->fib6_metric)
8d1040e8 899 next = leaf;
f11e6659 900
66f5d6ce 901 if (next != rt0) {
93c2fb25 902 spin_lock_bh(&leaf->fib6_table->tb6_lock);
66f5d6ce 903 /* make sure next is not being deleted from the tree */
93c2fb25 904 if (next->fib6_node)
66f5d6ce 905 rcu_assign_pointer(fn->rr_ptr, next);
93c2fb25 906 spin_unlock_bh(&leaf->fib6_table->tb6_lock);
66f5d6ce 907 }
1da177e4 908 }
1da177e4 909
b7bc4b6a
DA		 910out:
911 if (!res->f6i) {
912 res->f6i = net->ipv6.fib6_null_entry;
1cf844c7 913 res->nh = res->f6i->fib6_nh;
7d21fec9
DA		 914 res->fib6_flags = res->f6i->fib6_flags;
915 res->fib6_type = res->f6i->fib6_type;
b7bc4b6a 916 }
1da177e4
LT		 917}
918
85bd05de 919static bool rt6_is_gw_or_nonexthop(const struct fib6_result *res)
8b9df265 920{
85bd05de
DA		 921 return (res->f6i->fib6_flags & RTF_NONEXTHOP) ||
922 res->nh->fib_nh_gw_family;
8b9df265
MKL		 923}
924
70ceb4f5
YH		 925#ifdef CONFIG_IPV6_ROUTE_INFO
926int rt6_route_rcv(struct net_device *dev, u8 *opt, int len,
b71d1d42 927 const struct in6_addr *gwaddr)
70ceb4f5 928{
c346dca1 929 struct net *net = dev_net(dev);
70ceb4f5
YH		 930 struct route_info *rinfo = (struct route_info *) opt;
931 struct in6_addr prefix_buf, *prefix;
932 unsigned int pref;
4bed72e4 933 unsigned long lifetime;
8d1c802b 934 struct fib6_info *rt;
70ceb4f5
YH		 935
936 if (len < sizeof(struct route_info)) {
937 return -EINVAL;
938 }
939
940 /* Sanity check for prefix_len and length */
941 if (rinfo->length > 3) {
942 return -EINVAL;
943 } else if (rinfo->prefix_len > 128) {
944 return -EINVAL;
945 } else if (rinfo->prefix_len > 64) {
946 if (rinfo->length < 2) {
947 return -EINVAL;
948 }
949 } else if (rinfo->prefix_len > 0) {
950 if (rinfo->length < 1) {
951 return -EINVAL;
952 }
953 }
954
955 pref = rinfo->route_pref;
956 if (pref == ICMPV6_ROUTER_PREF_INVALID)
3933fc95 957 return -EINVAL;
70ceb4f5 958
4bed72e4 959 lifetime = addrconf_timeout_fixup(ntohl(rinfo->lifetime), HZ);
70ceb4f5
YH		 960
961 if (rinfo->length == 3)
962 prefix = (struct in6_addr *)rinfo->prefix;
963 else {
964 /* this function is safe */
965 ipv6_addr_prefix(&prefix_buf,
966 (struct in6_addr *)rinfo->prefix,
967 rinfo->prefix_len);
968 prefix = &prefix_buf;
969 }
970
f104a567 971 if (rinfo->prefix_len == 0)
afb1d4b5 972 rt = rt6_get_dflt_router(net, gwaddr, dev);
f104a567
DJ		 973 else
974 rt = rt6_get_route_info(net, prefix, rinfo->prefix_len,
830218c1 975 gwaddr, dev);
70ceb4f5
YH		 976
977 if (rt && !lifetime) {
11dd74b3 978 ip6_del_rt(net, rt, false);
70ceb4f5
YH		 979 rt = NULL;
980 }
981
982 if (!rt && lifetime)
830218c1
DA		 983 rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr,
984 dev, pref);
70ceb4f5 985 else if (rt)
93c2fb25
DA		 986 rt->fib6_flags = RTF_ROUTEINFO |
987 (rt->fib6_flags & ~RTF_PREF_MASK) | RTF_PREF(pref);
70ceb4f5
YH		 988
989 if (rt) {
1716a961 990 if (!addrconf_finite_timeout(lifetime))
14895687 991 fib6_clean_expires(rt);
1716a961 992 else
14895687 993 fib6_set_expires(rt, jiffies + HZ * lifetime);
1716a961 994
93531c67 995 fib6_info_release(rt);
70ceb4f5
YH		 996 }
997 return 0;
998}
999#endif
1000
ae90d867
DA		 1001/*
1002 * Misc support functions
1003 */
1004
1005/* called with rcu_lock held */
0d161581 1006static struct net_device *ip6_rt_get_dev_rcu(const struct fib6_result *res)
ae90d867 1007{
0d161581 1008 struct net_device *dev = res->nh->fib_nh_dev;
ae90d867 1009
7d21fec9 1010 if (res->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) {
ae90d867
DA		 1011 /* for copies of local routes, dst->dev needs to be the
1012 * device if it is a master device, the master device if
1013 * device is enslaved, and the loopback as the default
1014 */
1015 if (netif_is_l3_slave(dev) &&
7d21fec9 1016 !rt6_need_strict(&res->f6i->fib6_dst.addr))
ae90d867
DA		 1017 dev = l3mdev_master_dev_rcu(dev);
1018 else if (!netif_is_l3_master(dev))
1019 dev = dev_net(dev)->loopback_dev;
1020 /* last case is netif_is_l3_master(dev) is true in which
1021 * case we want dev returned to be dev
1022 */
1023 }
1024
1025 return dev;
1026}
1027
6edb3c96
DA		 1028static const int fib6_prop[RTN_MAX + 1] = {
1029 [RTN_UNSPEC] = 0,
1030 [RTN_UNICAST] = 0,
1031 [RTN_LOCAL] = 0,
1032 [RTN_BROADCAST] = 0,
1033 [RTN_ANYCAST] = 0,
1034 [RTN_MULTICAST] = 0,
1035 [RTN_BLACKHOLE] = -EINVAL,
1036 [RTN_UNREACHABLE] = -EHOSTUNREACH,
1037 [RTN_PROHIBIT] = -EACCES,
1038 [RTN_THROW] = -EAGAIN,
1039 [RTN_NAT] = -EINVAL,
1040 [RTN_XRESOLVE] = -EINVAL,
1041};
1042
1043static int ip6_rt_type_to_error(u8 fib6_type)
1044{
1045 return fib6_prop[fib6_type];
1046}
1047
8d1c802b 1048static unsigned short fib6_info_dst_flags(struct fib6_info *rt)
3b6761d1
DA		 1049{
1050 unsigned short flags = 0;
1051
1052 if (rt->dst_nocount)
1053 flags |= DST_NOCOUNT;
1054 if (rt->dst_nopolicy)
1055 flags |= DST_NOPOLICY;
3b6761d1
DA		 1056
1057 return flags;
1058}
1059
7d21fec9 1060static void ip6_rt_init_dst_reject(struct rt6_info *rt, u8 fib6_type)
6edb3c96 1061{
7d21fec9 1062 rt->dst.error = ip6_rt_type_to_error(fib6_type);
6edb3c96 1063
7d21fec9 1064 switch (fib6_type) {
6edb3c96
DA		 1065 case RTN_BLACKHOLE:
1066 rt->dst.output = dst_discard_out;
1067 rt->dst.input = dst_discard;
1068 break;
1069 case RTN_PROHIBIT:
1070 rt->dst.output = ip6_pkt_prohibit_out;
1071 rt->dst.input = ip6_pkt_prohibit;
1072 break;
1073 case RTN_THROW:
1074 case RTN_UNREACHABLE:
1075 default:
1076 rt->dst.output = ip6_pkt_discard_out;
1077 rt->dst.input = ip6_pkt_discard;
1078 break;
1079 }
1080}
1081
0d161581 1082static void ip6_rt_init_dst(struct rt6_info *rt, const struct fib6_result *res)
6edb3c96 1083{
7d21fec9 1084 struct fib6_info *f6i = res->f6i;
0d161581 1085
7d21fec9
DA		 1086 if (res->fib6_flags & RTF_REJECT) {
1087 ip6_rt_init_dst_reject(rt, res->fib6_type);
6edb3c96
DA		 1088 return;
1089 }
1090
1091 rt->dst.error = 0;
1092 rt->dst.output = ip6_output;
1093
7d21fec9 1094 if (res->fib6_type == RTN_LOCAL || res->fib6_type == RTN_ANYCAST) {
6edb3c96 1095 rt->dst.input = ip6_input;
7d21fec9 1096 } else if (ipv6_addr_type(&f6i->fib6_dst.addr) & IPV6_ADDR_MULTICAST) {
6edb3c96
DA		 1097 rt->dst.input = ip6_mc_input;
1098 } else {
1099 rt->dst.input = ip6_forward;
1100 }
1101
0d161581
DA		 1102 if (res->nh->fib_nh_lws) {
1103 rt->dst.lwtstate = lwtstate_get(res->nh->fib_nh_lws);
6edb3c96
DA		 1104 lwtunnel_set_redirect(&rt->dst);
1105 }
1106
1107 rt->dst.lastuse = jiffies;
1108}
1109
e873e4b9 1110/* Caller must already hold reference to @from */
8d1c802b 1111static void rt6_set_from(struct rt6_info *rt, struct fib6_info *from)
ae90d867 1112{
ae90d867 1113 rt->rt6i_flags &= ~RTF_EXPIRES;
a68886a6 1114 rcu_assign_pointer(rt->from, from);
e1255ed4 1115 ip_dst_init_metrics(&rt->dst, from->fib6_metrics);
ae90d867
DA		 1116}
1117
0d161581
DA		 1118/* Caller must already hold reference to f6i in result */
1119static void ip6_rt_copy_init(struct rt6_info *rt, const struct fib6_result *res)
ae90d867 1120{
0d161581
DA		 1121 const struct fib6_nh *nh = res->nh;
1122 const struct net_device *dev = nh->fib_nh_dev;
1123 struct fib6_info *f6i = res->f6i;
dcd1f572 1124
0d161581 1125 ip6_rt_init_dst(rt, res);
6edb3c96 1126
0d161581 1127 rt->rt6i_dst = f6i->fib6_dst;
dcd1f572 1128 rt->rt6i_idev = dev ? in6_dev_get(dev) : NULL;
7d21fec9 1129 rt->rt6i_flags = res->fib6_flags;
0d161581
DA		 1130 if (nh->fib_nh_gw_family) {
1131 rt->rt6i_gateway = nh->fib_nh_gw6;
2b2450ca
DA		 1132 rt->rt6i_flags |= RTF_GATEWAY;
1133 }
0d161581 1134 rt6_set_from(rt, f6i);
ae90d867 1135#ifdef CONFIG_IPV6_SUBTREES
0d161581 1136 rt->rt6i_src = f6i->fib6_src;
ae90d867 1137#endif
ae90d867
DA		 1138}
1139
a3c00e46
MKL		 1140static struct fib6_node* fib6_backtrack(struct fib6_node *fn,
1141 struct in6_addr *saddr)
1142{
66f5d6ce 1143 struct fib6_node *pn, *sn;
a3c00e46
MKL		 1144 while (1) {
1145 if (fn->fn_flags & RTN_TL_ROOT)
1146 return NULL;
66f5d6ce
WW		 1147 pn = rcu_dereference(fn->parent);
1148 sn = FIB6_SUBTREE(pn);
1149 if (sn && sn != fn)
6454743b 1150 fn = fib6_node_lookup(sn, NULL, saddr);
a3c00e46
MKL		 1151 else
1152 fn = pn;
1153 if (fn->fn_flags & RTN_RTINFO)
1154 return fn;
1155 }
1156}
c71099ac 1157
10585b43 1158static bool ip6_hold_safe(struct net *net, struct rt6_info **prt)
d3843fe5
WW		 1159{
1160 struct rt6_info *rt = *prt;
1161
1162 if (dst_hold_safe(&rt->dst))
1163 return true;
10585b43 1164 if (net) {
d3843fe5
WW		 1165 rt = net->ipv6.ip6_null_entry;
1166 dst_hold(&rt->dst);
1167 } else {
1168 rt = NULL;
1169 }
1170 *prt = rt;
1171 return false;
1172}
1173
dec9b0e2 1174/* called with rcu_lock held */
9b6b35ab 1175static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)
dec9b0e2 1176{
9b6b35ab
DA		 1177 struct net_device *dev = res->nh->fib_nh_dev;
1178 struct fib6_info *f6i = res->f6i;
1179 unsigned short flags;
dec9b0e2
DA		 1180 struct rt6_info *nrt;
1181
9b6b35ab 1182 if (!fib6_info_hold_safe(f6i))
1c87e79a 1183 goto fallback;
e873e4b9 1184
9b6b35ab 1185 flags = fib6_info_dst_flags(f6i);
93531c67 1186 nrt = ip6_dst_alloc(dev_net(dev), dev, flags);
1c87e79a 1187 if (!nrt) {
9b6b35ab 1188 fib6_info_release(f6i);
1c87e79a
XL		 1189 goto fallback;
1190 }
dec9b0e2 1191
0d161581 1192 ip6_rt_copy_init(nrt, res);
1c87e79a
XL		 1193 return nrt;
1194
1195fallback:
1196 nrt = dev_net(dev)->ipv6.ip6_null_entry;
1197 dst_hold(&nrt->dst);
dec9b0e2
DA		 1198 return nrt;
1199}
1200
55cced4f 1201INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net,
8ed67789 1202 struct fib6_table *table,
b75cc8f9
DA		 1203 struct flowi6 *fl6,
1204 const struct sk_buff *skb,
1205 int flags)
1da177e4 1206{
b1d40991 1207 struct fib6_result res = {};
1da177e4 1208 struct fib6_node *fn;
23fb93a4 1209 struct rt6_info *rt;
1da177e4 1210
66f5d6ce 1211 rcu_read_lock();
6454743b 1212 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
c71099ac 1213restart:
b1d40991
DA		 1214 res.f6i = rcu_dereference(fn->leaf);
1215 if (!res.f6i)
1216 res.f6i = net->ipv6.fib6_null_entry;
af52a52c 1217 else
75ef7389
DA		 1218 rt6_device_match(net, &res, &fl6->saddr, fl6->flowi6_oif,
1219 flags);
af52a52c 1220
b1d40991 1221 if (res.f6i == net->ipv6.fib6_null_entry) {
a3c00e46
MKL		 1222 fn = fib6_backtrack(fn, &fl6->saddr);
1223 if (fn)
1224 goto restart;
2b760fcf 1225
af52a52c
DA		 1226 rt = net->ipv6.ip6_null_entry;
1227 dst_hold(&rt->dst);
1228 goto out;
f88d8ea6
DA		 1229 } else if (res.fib6_flags & RTF_REJECT) {
1230 goto do_create;
af52a52c 1231 }
d3843fe5 1232
b1d40991
DA		 1233 fib6_select_path(net, &res, fl6, fl6->flowi6_oif,
1234 fl6->flowi6_oif != 0, skb, flags);
1235
2b760fcf 1236 /* Search through exception table */
7e4b5128 1237 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
23fb93a4 1238 if (rt) {
10585b43 1239 if (ip6_hold_safe(net, &rt))
dec9b0e2 1240 dst_use_noref(&rt->dst, jiffies);
23fb93a4 1241 } else {
f88d8ea6 1242do_create:
9b6b35ab 1243 rt = ip6_create_rt_rcu(&res);
dec9b0e2 1244 }
b811580d 1245
af52a52c 1246out:
8ff2e5b2 1247 trace_fib6_table_lookup(net, &res, table, fl6);
af52a52c 1248
66f5d6ce 1249 rcu_read_unlock();
b811580d 1250
c71099ac 1251 return rt;
c71099ac
TG		 1252}
1253
67ba4152 1254struct dst_entry *ip6_route_lookup(struct net *net, struct flowi6 *fl6,
b75cc8f9 1255 const struct sk_buff *skb, int flags)
ea6e574e 1256{
b75cc8f9 1257 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_lookup);
ea6e574e
FW		 1258}
1259EXPORT_SYMBOL_GPL(ip6_route_lookup);
1260
9acd9f3a 1261struct rt6_info *rt6_lookup(struct net *net, const struct in6_addr *daddr,
b75cc8f9
DA		 1262 const struct in6_addr *saddr, int oif,
1263 const struct sk_buff *skb, int strict)
c71099ac 1264{
4c9483b2
DM		 1265 struct flowi6 fl6 = {
1266 .flowi6_oif = oif,
1267 .daddr = *daddr,
c71099ac
TG		 1268 };
1269 struct dst_entry *dst;
77d16f45 1270 int flags = strict ? RT6_LOOKUP_F_IFACE : 0;
c71099ac 1271
adaa70bb 1272 if (saddr) {
4c9483b2 1273 memcpy(&fl6.saddr, saddr, sizeof(*saddr));
adaa70bb
TG		 1274 flags |= RT6_LOOKUP_F_HAS_SADDR;
1275 }
1276
b75cc8f9 1277 dst = fib6_rule_lookup(net, &fl6, skb, flags, ip6_pol_route_lookup);
c71099ac
TG		 1278 if (dst->error == 0)
1279 return (struct rt6_info *) dst;
1280
1281 dst_release(dst);
1282
1da177e4
LT		 1283 return NULL;
1284}
7159039a
YH		 1285EXPORT_SYMBOL(rt6_lookup);
1286
c71099ac 1287/* ip6_ins_rt is called with FREE table->tb6_lock.
1cfb71ee
WW		 1288 * It takes new route entry, the addition fails by any reason the
1289 * route is released.
1290 * Caller must hold dst before calling it.
1da177e4
LT		 1291 */
1292
8d1c802b 1293static int __ip6_ins_rt(struct fib6_info *rt, struct nl_info *info,
333c4301 1294 struct netlink_ext_ack *extack)
1da177e4
LT		 1295{
1296 int err;
c71099ac 1297 struct fib6_table *table;
1da177e4 1298
93c2fb25 1299 table = rt->fib6_table;
66f5d6ce 1300 spin_lock_bh(&table->tb6_lock);
d4ead6b3 1301 err = fib6_add(&table->tb6_root, rt, info, extack);
66f5d6ce 1302 spin_unlock_bh(&table->tb6_lock);
1da177e4
LT		 1303
1304 return err;
1305}
1306
8d1c802b 1307int ip6_ins_rt(struct net *net, struct fib6_info *rt)
40e22e8f 1308{
afb1d4b5 1309 struct nl_info info = { .nl_net = net, };
e715b6d3 1310
d4ead6b3 1311 return __ip6_ins_rt(rt, &info, NULL);
40e22e8f
TG		 1312}
1313
85bd05de 1314static struct rt6_info *ip6_rt_cache_alloc(const struct fib6_result *res,
8b9df265
MKL		 1315 const struct in6_addr *daddr,
1316 const struct in6_addr *saddr)
1da177e4 1317{
85bd05de 1318 struct fib6_info *f6i = res->f6i;
4832c30d 1319 struct net_device *dev;
1da177e4
LT		 1320 struct rt6_info *rt;
1321
1322 /*
1323 * Clone the route.
1324 */
1325
85bd05de 1326 if (!fib6_info_hold_safe(f6i))
e873e4b9
WW		 1327 return NULL;
1328
0d161581 1329 dev = ip6_rt_get_dev_rcu(res);
93531c67 1330 rt = ip6_dst_alloc(dev_net(dev), dev, 0);
e873e4b9 1331 if (!rt) {
85bd05de 1332 fib6_info_release(f6i);
83a09abd 1333 return NULL;
e873e4b9 1334 }
83a09abd 1335
0d161581 1336 ip6_rt_copy_init(rt, res);
83a09abd 1337 rt->rt6i_flags |= RTF_CACHE;
83a09abd
MKL		 1338 rt->rt6i_dst.addr = *daddr;
1339 rt->rt6i_dst.plen = 128;
1da177e4 1340
85bd05de
DA		 1341 if (!rt6_is_gw_or_nonexthop(res)) {
1342 if (f6i->fib6_dst.plen != 128 &&
1343 ipv6_addr_equal(&f6i->fib6_dst.addr, daddr))
83a09abd 1344 rt->rt6i_flags |= RTF_ANYCAST;
1da177e4 1345#ifdef CONFIG_IPV6_SUBTREES
83a09abd
MKL		 1346 if (rt->rt6i_src.plen && saddr) {
1347 rt->rt6i_src.addr = *saddr;
1348 rt->rt6i_src.plen = 128;
8b9df265 1349 }
83a09abd 1350#endif
95a9a5ba 1351 }
1da177e4 1352
95a9a5ba
YH		 1353 return rt;
1354}
1da177e4 1355
db3fedee 1356static struct rt6_info *ip6_rt_pcpu_alloc(const struct fib6_result *res)
d52d3997 1357{
db3fedee
DA		 1358 struct fib6_info *f6i = res->f6i;
1359 unsigned short flags = fib6_info_dst_flags(f6i);
4832c30d 1360 struct net_device *dev;
d52d3997
MKL		 1361 struct rt6_info *pcpu_rt;
1362
db3fedee 1363 if (!fib6_info_hold_safe(f6i))
e873e4b9
WW		 1364 return NULL;
1365
4832c30d 1366 rcu_read_lock();
0d161581 1367 dev = ip6_rt_get_dev_rcu(res);
d8882935 1368 pcpu_rt = ip6_dst_alloc(dev_net(dev), dev, flags | DST_NOCOUNT);
4832c30d 1369 rcu_read_unlock();
e873e4b9 1370 if (!pcpu_rt) {
db3fedee 1371 fib6_info_release(f6i);
d52d3997 1372 return NULL;
e873e4b9 1373 }
0d161581 1374 ip6_rt_copy_init(pcpu_rt, res);
d52d3997 1375 pcpu_rt->rt6i_flags |= RTF_PCPU;
8f34e53b
DA		 1376
1377 if (f6i->nh)
1378 pcpu_rt->sernum = rt_genid_ipv6(dev_net(dev));
1379
d52d3997
MKL		 1380 return pcpu_rt;
1381}
1382
8f34e53b
DA		 1383static bool rt6_is_valid(const struct rt6_info *rt6)
1384{
1385 return rt6->sernum == rt_genid_ipv6(dev_net(rt6->dst.dev));
1386}
1387
66f5d6ce 1388/* It should be called with rcu_read_lock() acquired */
db3fedee 1389static struct rt6_info *rt6_get_pcpu_route(const struct fib6_result *res)
d52d3997 1390{
c353071a 1391 struct rt6_info *pcpu_rt;
d52d3997 1392
c353071a 1393 pcpu_rt = this_cpu_read(*res->nh->rt6i_pcpu);
d52d3997 1394
8f34e53b
DA		 1395 if (pcpu_rt && pcpu_rt->sernum && !rt6_is_valid(pcpu_rt)) {
1396 struct rt6_info *prev, **p;
1397
1398 p = this_cpu_ptr(res->nh->rt6i_pcpu);
1399 prev = xchg(p, NULL);
1400 if (prev) {
1401 dst_dev_put(&prev->dst);
1402 dst_release(&prev->dst);
1403 }
1404
1405 pcpu_rt = NULL;
1406 }
1407
a73e4195
MKL		 1408 return pcpu_rt;
1409}
1410
afb1d4b5 1411static struct rt6_info *rt6_make_pcpu_route(struct net *net,
db3fedee 1412 const struct fib6_result *res)
a73e4195
MKL		 1413{
1414 struct rt6_info *pcpu_rt, *prev, **p;
d52d3997 1415
db3fedee 1416 pcpu_rt = ip6_rt_pcpu_alloc(res);
0e09edcc
WW		 1417 if (!pcpu_rt)
1418 return NULL;
d52d3997 1419
f40b6ae2 1420 p = this_cpu_ptr(res->nh->rt6i_pcpu);
a94b9367 1421 prev = cmpxchg(p, NULL, pcpu_rt);
951f788a 1422 BUG_ON(prev);
a94b9367 1423
61fb0d01
ED		 1424 if (res->f6i->fib6_destroying) {
1425 struct fib6_info *from;
1426
1427 from = xchg((__force struct fib6_info **)&pcpu_rt->from, NULL);
1428 fib6_info_release(from);
1429 }
1430
d52d3997
MKL		 1431 return pcpu_rt;
1432}
1433
35732d01
WW		 1434/* exception hash table implementation
1435 */
1436static DEFINE_SPINLOCK(rt6_exception_lock);
1437
1438/* Remove rt6_ex from hash table and free the memory
1439 * Caller must hold rt6_exception_lock
1440 */
1441static void rt6_remove_exception(struct rt6_exception_bucket *bucket,
1442 struct rt6_exception *rt6_ex)
1443{
f5b51fe8 1444 struct fib6_info *from;
b2427e67 1445 struct net *net;
81eb8447 1446
35732d01
WW		 1447 if (!bucket || !rt6_ex)
1448 return;
b2427e67
CIK		 1449
1450 net = dev_net(rt6_ex->rt6i->dst.dev);
f5b51fe8
PA		 1451 net->ipv6.rt6_stats->fib_rt_cache--;
1452
1453 /* purge completely the exception to allow releasing the held resources:
1454 * some [sk] cache may keep the dst around for unlimited time
1455 */
0e233874 1456 from = xchg((__force struct fib6_info **)&rt6_ex->rt6i->from, NULL);
f5b51fe8
PA		 1457 fib6_info_release(from);
1458 dst_dev_put(&rt6_ex->rt6i->dst);
1459
35732d01 1460 hlist_del_rcu(&rt6_ex->hlist);
77634cc6 1461 dst_release(&rt6_ex->rt6i->dst);
35732d01
WW		 1462 kfree_rcu(rt6_ex, rcu);
1463 WARN_ON_ONCE(!bucket->depth);
1464 bucket->depth--;
1465}
1466
1467/* Remove oldest rt6_ex in bucket and free the memory
1468 * Caller must hold rt6_exception_lock
1469 */
1470static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket)
1471{
1472 struct rt6_exception *rt6_ex, *oldest = NULL;
1473
1474 if (!bucket)
1475 return;
1476
1477 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
1478 if (!oldest || time_before(rt6_ex->stamp, oldest->stamp))
1479 oldest = rt6_ex;
1480 }
1481 rt6_remove_exception(bucket, oldest);
1482}
1483
1484static u32 rt6_exception_hash(const struct in6_addr *dst,
1485 const struct in6_addr *src)
1486{
49ecc2e9 1487 static siphash_aligned_key_t rt6_exception_key;
4785305c
ED		 1488 struct {
1489 struct in6_addr dst;
1490 struct in6_addr src;
1491 } __aligned(SIPHASH_ALIGNMENT) combined = {
1492 .dst = *dst,
1493 };
1494 u64 val;
35732d01 1495
4785305c 1496 net_get_random_once(&rt6_exception_key, sizeof(rt6_exception_key));
35732d01
WW		 1497
1498#ifdef CONFIG_IPV6_SUBTREES
1499 if (src)
4785305c 1500 combined.src = *src;
35732d01 1501#endif
4785305c
ED		 1502 val = siphash(&combined, sizeof(combined), &rt6_exception_key);
1503
1504 return hash_64(val, FIB6_EXCEPTION_BUCKET_SIZE_SHIFT);
35732d01
WW		 1505}
1506
1507/* Helper function to find the cached rt in the hash table
1508 * and update bucket pointer to point to the bucket for this
1509 * (daddr, saddr) pair
1510 * Caller must hold rt6_exception_lock
1511 */
1512static struct rt6_exception *
1513__rt6_find_exception_spinlock(struct rt6_exception_bucket **bucket,
1514 const struct in6_addr *daddr,
1515 const struct in6_addr *saddr)
1516{
1517 struct rt6_exception *rt6_ex;
1518 u32 hval;
1519
1520 if (!(*bucket) || !daddr)
1521 return NULL;
1522
1523 hval = rt6_exception_hash(daddr, saddr);
1524 *bucket += hval;
1525
1526 hlist_for_each_entry(rt6_ex, &(*bucket)->chain, hlist) {
1527 struct rt6_info *rt6 = rt6_ex->rt6i;
1528 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1529
1530#ifdef CONFIG_IPV6_SUBTREES
1531 if (matched && saddr)
1532 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1533#endif
1534 if (matched)
1535 return rt6_ex;
1536 }
1537 return NULL;
1538}
1539
1540/* Helper function to find the cached rt in the hash table
1541 * and update bucket pointer to point to the bucket for this
1542 * (daddr, saddr) pair
1543 * Caller must hold rcu_read_lock()
1544 */
1545static struct rt6_exception *
1546__rt6_find_exception_rcu(struct rt6_exception_bucket **bucket,
1547 const struct in6_addr *daddr,
1548 const struct in6_addr *saddr)
1549{
1550 struct rt6_exception *rt6_ex;
1551 u32 hval;
1552
1553 WARN_ON_ONCE(!rcu_read_lock_held());
1554
1555 if (!(*bucket) || !daddr)
1556 return NULL;
1557
1558 hval = rt6_exception_hash(daddr, saddr);
1559 *bucket += hval;
1560
1561 hlist_for_each_entry_rcu(rt6_ex, &(*bucket)->chain, hlist) {
1562 struct rt6_info *rt6 = rt6_ex->rt6i;
1563 bool matched = ipv6_addr_equal(daddr, &rt6->rt6i_dst.addr);
1564
1565#ifdef CONFIG_IPV6_SUBTREES
1566 if (matched && saddr)
1567 matched = ipv6_addr_equal(saddr, &rt6->rt6i_src.addr);
1568#endif
1569 if (matched)
1570 return rt6_ex;
1571 }
1572 return NULL;
1573}
1574
b748f260 1575static unsigned int fib6_mtu(const struct fib6_result *res)
d4ead6b3 1576{
b748f260 1577 const struct fib6_nh *nh = res->nh;
d4ead6b3
DA		 1578 unsigned int mtu;
1579
b748f260
DA		 1580 if (res->f6i->fib6_pmtu) {
1581 mtu = res->f6i->fib6_pmtu;
dcd1f572 1582 } else {
b748f260 1583 struct net_device *dev = nh->fib_nh_dev;
dcd1f572
DA		 1584 struct inet6_dev *idev;
1585
1586 rcu_read_lock();
1587 idev = __in6_dev_get(dev);
1588 mtu = idev->cnf.mtu6;
1589 rcu_read_unlock();
1590 }
1591
d4ead6b3
DA		 1592 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
1593
b748f260 1594 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
d4ead6b3
DA		 1595}
1596
cc5c073a
DA		 1597#define FIB6_EXCEPTION_BUCKET_FLUSHED 0x1UL
1598
1599/* used when the flushed bit is not relevant, only access to the bucket
1600 * (ie., all bucket users except rt6_insert_exception);
1601 *
1602 * called under rcu lock; sometimes called with rt6_exception_lock held
1603 */
1604static
1605struct rt6_exception_bucket *fib6_nh_get_excptn_bucket(const struct fib6_nh *nh,
1606 spinlock_t *lock)
1607{
1608 struct rt6_exception_bucket *bucket;
1609
1610 if (lock)
1611 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1612 lockdep_is_held(lock));
1613 else
1614 bucket = rcu_dereference(nh->rt6i_exception_bucket);
1615
1616 /* remove bucket flushed bit if set */
1617 if (bucket) {
1618 unsigned long p = (unsigned long)bucket;
1619
1620 p &= ~FIB6_EXCEPTION_BUCKET_FLUSHED;
1621 bucket = (struct rt6_exception_bucket *)p;
1622 }
1623
1624 return bucket;
1625}
1626
1627static bool fib6_nh_excptn_bucket_flushed(struct rt6_exception_bucket *bucket)
1628{
1629 unsigned long p = (unsigned long)bucket;
1630
1631 return !!(p & FIB6_EXCEPTION_BUCKET_FLUSHED);
1632}
1633
1634/* called with rt6_exception_lock held */
1635static void fib6_nh_excptn_bucket_set_flushed(struct fib6_nh *nh,
1636 spinlock_t *lock)
1637{
1638 struct rt6_exception_bucket *bucket;
1639 unsigned long p;
1640
1641 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1642 lockdep_is_held(lock));
1643
1644 p = (unsigned long)bucket;
1645 p |= FIB6_EXCEPTION_BUCKET_FLUSHED;
1646 bucket = (struct rt6_exception_bucket *)p;
1647 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1648}
1649
35732d01 1650static int rt6_insert_exception(struct rt6_info *nrt,
5012f0a5 1651 const struct fib6_result *res)
35732d01 1652{
5e670d84 1653 struct net *net = dev_net(nrt->dst.dev);
35732d01 1654 struct rt6_exception_bucket *bucket;
cc5c073a 1655 struct fib6_info *f6i = res->f6i;
35732d01
WW		 1656 struct in6_addr *src_key = NULL;
1657 struct rt6_exception *rt6_ex;
cc5c073a 1658 struct fib6_nh *nh = res->nh;
a00df2ca 1659 int max_depth;
35732d01
WW		 1660 int err = 0;
1661
35732d01
WW		 1662 spin_lock_bh(&rt6_exception_lock);
1663
cc5c073a
DA		 1664 bucket = rcu_dereference_protected(nh->rt6i_exception_bucket,
1665 lockdep_is_held(&rt6_exception_lock));
35732d01
WW		 1666 if (!bucket) {
1667 bucket = kcalloc(FIB6_EXCEPTION_BUCKET_SIZE, sizeof(*bucket),
1668 GFP_ATOMIC);
1669 if (!bucket) {
1670 err = -ENOMEM;
1671 goto out;
1672 }
cc5c073a
DA		 1673 rcu_assign_pointer(nh->rt6i_exception_bucket, bucket);
1674 } else if (fib6_nh_excptn_bucket_flushed(bucket)) {
1675 err = -EINVAL;
1676 goto out;
35732d01
WW		 1677 }
1678
1679#ifdef CONFIG_IPV6_SUBTREES
5012f0a5 1680 /* fib6_src.plen != 0 indicates f6i is in subtree
35732d01 1681 * and exception table is indexed by a hash of
5012f0a5 1682 * both fib6_dst and fib6_src.
35732d01 1683 * Otherwise, the exception table is indexed by
5012f0a5 1684 * a hash of only fib6_dst.
35732d01 1685 */
5012f0a5 1686 if (f6i->fib6_src.plen)
35732d01
WW		 1687 src_key = &nrt->rt6i_src.addr;
1688#endif
5012f0a5 1689 /* rt6_mtu_change() might lower mtu on f6i.
f5bbe7ee 1690 * Only insert this exception route if its mtu
5012f0a5 1691 * is less than f6i's mtu value.
f5bbe7ee 1692 */
b748f260 1693 if (dst_metric_raw(&nrt->dst, RTAX_MTU) >= fib6_mtu(res)) {
f5bbe7ee
WW		 1694 err = -EINVAL;
1695 goto out;
1696 }
60006a48 1697
35732d01
WW		 1698 rt6_ex = __rt6_find_exception_spinlock(&bucket, &nrt->rt6i_dst.addr,
1699 src_key);
1700 if (rt6_ex)
1701 rt6_remove_exception(bucket, rt6_ex);
1702
1703 rt6_ex = kzalloc(sizeof(*rt6_ex), GFP_ATOMIC);
1704 if (!rt6_ex) {
1705 err = -ENOMEM;
1706 goto out;
1707 }
1708 rt6_ex->rt6i = nrt;
1709 rt6_ex->stamp = jiffies;
35732d01
WW		 1710 hlist_add_head_rcu(&rt6_ex->hlist, &bucket->chain);
1711 bucket->depth++;
81eb8447 1712 net->ipv6.rt6_stats->fib_rt_cache++;
35732d01 1713
a00df2ca 1714 /* Randomize max depth to avoid some side channels attacks. */
8032bf12 1715 max_depth = FIB6_MAX_DEPTH + get_random_u32_below(FIB6_MAX_DEPTH);
a00df2ca 1716 while (bucket->depth > max_depth)
35732d01
WW		 1717 rt6_exception_remove_oldest(bucket);
1718
1719out:
1720 spin_unlock_bh(&rt6_exception_lock);
1721
1722 /* Update fn->fn_sernum to invalidate all cached dst */
b886d5f2 1723 if (!err) {
5012f0a5
DA		 1724 spin_lock_bh(&f6i->fib6_table->tb6_lock);
1725 fib6_update_sernum(net, f6i);
1726 spin_unlock_bh(&f6i->fib6_table->tb6_lock);
b886d5f2
PA		 1727 fib6_force_start_gc(net);
1728 }
35732d01
WW		 1729
1730 return err;
1731}
1732
c0b220cf 1733static void fib6_nh_flush_exceptions(struct fib6_nh *nh, struct fib6_info *from)
35732d01
WW		 1734{
1735 struct rt6_exception_bucket *bucket;
1736 struct rt6_exception *rt6_ex;
1737 struct hlist_node *tmp;
1738 int i;
1739
1740 spin_lock_bh(&rt6_exception_lock);
35732d01 1741
cc5c073a 1742 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
35732d01
WW		 1743 if (!bucket)
1744 goto out;
1745
cc5c073a
DA		 1746 /* Prevent rt6_insert_exception() to recreate the bucket list */
1747 if (!from)
1748 fib6_nh_excptn_bucket_set_flushed(nh, &rt6_exception_lock);
1749
35732d01 1750 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
cc5c073a
DA		 1751 hlist_for_each_entry_safe(rt6_ex, tmp, &bucket->chain, hlist) {
1752 if (!from ||
1753 rcu_access_pointer(rt6_ex->rt6i->from) == from)
1754 rt6_remove_exception(bucket, rt6_ex);
1755 }
1756 WARN_ON_ONCE(!from && bucket->depth);
35732d01
WW		 1757 bucket++;
1758 }
35732d01
WW		 1759out:
1760 spin_unlock_bh(&rt6_exception_lock);
1761}
1762
e659ba31
DA		 1763static int rt6_nh_flush_exceptions(struct fib6_nh *nh, void *arg)
1764{
1765 struct fib6_info *f6i = arg;
1766
1767 fib6_nh_flush_exceptions(nh, f6i);
1768
1769 return 0;
1770}
1771
c0b220cf
DA		 1772void rt6_flush_exceptions(struct fib6_info *f6i)
1773{
e659ba31
DA		 1774 if (f6i->nh)
1775 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_flush_exceptions,
1776 f6i);
1777 else
1778 fib6_nh_flush_exceptions(f6i->fib6_nh, f6i);
c0b220cf
DA		 1779}
1780
35732d01
WW		 1781/* Find cached rt in the hash table inside passed in rt
1782 * Caller has to hold rcu_read_lock()
1783 */
7e4b5128 1784static struct rt6_info *rt6_find_cached_rt(const struct fib6_result *res,
510e2ced
WW		 1785 const struct in6_addr *daddr,
1786 const struct in6_addr *saddr)
35732d01 1787{
510e2ced 1788 const struct in6_addr *src_key = NULL;
35732d01 1789 struct rt6_exception_bucket *bucket;
35732d01 1790 struct rt6_exception *rt6_ex;
7e4b5128 1791 struct rt6_info *ret = NULL;
35732d01 1792
35732d01 1793#ifdef CONFIG_IPV6_SUBTREES
7e4b5128 1794 /* fib6i_src.plen != 0 indicates f6i is in subtree
35732d01 1795 * and exception table is indexed by a hash of
7e4b5128 1796 * both fib6_dst and fib6_src.
510e2ced
WW		 1797 * However, the src addr used to create the hash
1798 * might not be exactly the passed in saddr which
1799 * is a /128 addr from the flow.
1800 * So we need to use f6i->fib6_src to redo lookup
1801 * if the passed in saddr does not find anything.
1802 * (See the logic in ip6_rt_cache_alloc() on how
1803 * rt->rt6i_src is updated.)
35732d01 1804 */
7e4b5128 1805 if (res->f6i->fib6_src.plen)
35732d01 1806 src_key = saddr;
510e2ced 1807find_ex:
35732d01 1808#endif
cc5c073a 1809 bucket = fib6_nh_get_excptn_bucket(res->nh, NULL);
35732d01
WW		 1810 rt6_ex = __rt6_find_exception_rcu(&bucket, daddr, src_key);
1811
1812 if (rt6_ex && !rt6_check_expired(rt6_ex->rt6i))
7e4b5128 1813 ret = rt6_ex->rt6i;
35732d01 1814
510e2ced
WW		 1815#ifdef CONFIG_IPV6_SUBTREES
1816 /* Use fib6_src as src_key and redo lookup */
1817 if (!ret && src_key && src_key != &res->f6i->fib6_src.addr) {
1818 src_key = &res->f6i->fib6_src.addr;
1819 goto find_ex;
1820 }
1821#endif
1822
7e4b5128 1823 return ret;
35732d01
WW		 1824}
1825
1826/* Remove the passed in cached rt from the hash table that contains it */
cc5c073a 1827static int fib6_nh_remove_exception(const struct fib6_nh *nh, int plen,
c0b220cf 1828 const struct rt6_info *rt)
35732d01 1829{
c0b220cf 1830 const struct in6_addr *src_key = NULL;
35732d01 1831 struct rt6_exception_bucket *bucket;
35732d01
WW		 1832 struct rt6_exception *rt6_ex;
1833 int err;
1834
cc5c073a 1835 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
35732d01
WW		 1836 return -ENOENT;
1837
1838 spin_lock_bh(&rt6_exception_lock);
cc5c073a
DA		 1839 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
1840
35732d01 1841#ifdef CONFIG_IPV6_SUBTREES
cc5c073a
DA		 1842 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1843 * and exception table is indexed by a hash of
1844 * both rt6i_dst and rt6i_src.
35732d01
WW		 1845 * Otherwise, the exception table is indexed by
1846 * a hash of only rt6i_dst.
1847 */
c0b220cf 1848 if (plen)
35732d01
WW		 1849 src_key = &rt->rt6i_src.addr;
1850#endif
1851 rt6_ex = __rt6_find_exception_spinlock(&bucket,
1852 &rt->rt6i_dst.addr,
1853 src_key);
1854 if (rt6_ex) {
1855 rt6_remove_exception(bucket, rt6_ex);
1856 err = 0;
1857 } else {
1858 err = -ENOENT;
1859 }
1860
1861 spin_unlock_bh(&rt6_exception_lock);
1862 return err;
1863}
1864
e659ba31
DA		 1865struct fib6_nh_excptn_arg {
1866 struct rt6_info *rt;
1867 int plen;
1868};
1869
1870static int rt6_nh_remove_exception_rt(struct fib6_nh *nh, void *_arg)
1871{
1872 struct fib6_nh_excptn_arg *arg = _arg;
1873 int err;
1874
1875 err = fib6_nh_remove_exception(nh, arg->plen, arg->rt);
1876 if (err == 0)
1877 return 1;
1878
1879 return 0;
1880}
1881
c0b220cf
DA		 1882static int rt6_remove_exception_rt(struct rt6_info *rt)
1883{
1884 struct fib6_info *from;
1885
1886 from = rcu_dereference(rt->from);
cc5c073a 1887 if (!from || !(rt->rt6i_flags & RTF_CACHE))
c0b220cf
DA		 1888 return -EINVAL;
1889
e659ba31
DA		 1890 if (from->nh) {
1891 struct fib6_nh_excptn_arg arg = {
1892 .rt = rt,
1893 .plen = from->fib6_src.plen
1894 };
1895 int rc;
1896
1897 /* rc = 1 means an entry was found */
1898 rc = nexthop_for_each_fib6_nh(from->nh,
1899 rt6_nh_remove_exception_rt,
1900 &arg);
1901 return rc ? 0 : -ENOENT;
1902 }
1903
1cf844c7 1904 return fib6_nh_remove_exception(from->fib6_nh,
cc5c073a 1905 from->fib6_src.plen, rt);
c0b220cf
DA		 1906}
1907
35732d01
WW		 1908/* Find rt6_ex which contains the passed in rt cache and
1909 * refresh its stamp
1910 */
cc5c073a 1911static void fib6_nh_update_exception(const struct fib6_nh *nh, int plen,
c0b220cf 1912 const struct rt6_info *rt)
35732d01 1913{
c0b220cf 1914 const struct in6_addr *src_key = NULL;
35732d01 1915 struct rt6_exception_bucket *bucket;
35732d01 1916 struct rt6_exception *rt6_ex;
193f3685 1917
cc5c073a 1918 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
35732d01 1919#ifdef CONFIG_IPV6_SUBTREES
cc5c073a
DA		 1920 /* rt6i_src.plen != 0 indicates 'from' is in subtree
1921 * and exception table is indexed by a hash of
1922 * both rt6i_dst and rt6i_src.
35732d01
WW		 1923 * Otherwise, the exception table is indexed by
1924 * a hash of only rt6i_dst.
1925 */
c0b220cf 1926 if (plen)
35732d01
WW		 1927 src_key = &rt->rt6i_src.addr;
1928#endif
cc5c073a 1929 rt6_ex = __rt6_find_exception_rcu(&bucket, &rt->rt6i_dst.addr, src_key);
35732d01
WW		 1930 if (rt6_ex)
1931 rt6_ex->stamp = jiffies;
c0b220cf
DA		 1932}
1933
e659ba31
DA		 1934struct fib6_nh_match_arg {
1935 const struct net_device *dev;
1936 const struct in6_addr *gw;
1937 struct fib6_nh *match;
1938};
1939
1940/* determine if fib6_nh has given device and gateway */
1941static int fib6_nh_find_match(struct fib6_nh *nh, void *_arg)
1942{
1943 struct fib6_nh_match_arg *arg = _arg;
1944
1945 if (arg->dev != nh->fib_nh_dev ||
1946 (arg->gw && !nh->fib_nh_gw_family) ||
1947 (!arg->gw && nh->fib_nh_gw_family) ||
1948 (arg->gw && !ipv6_addr_equal(arg->gw, &nh->fib_nh_gw6)))
1949 return 0;
1950
1951 arg->match = nh;
1952
1953 /* found a match, break the loop */
1954 return 1;
1955}
1956
c0b220cf
DA		 1957static void rt6_update_exception_stamp_rt(struct rt6_info *rt)
1958{
1959 struct fib6_info *from;
e659ba31 1960 struct fib6_nh *fib6_nh;
35732d01 1961
c0b220cf
DA		 1962 rcu_read_lock();
1963
1964 from = rcu_dereference(rt->from);
1965 if (!from || !(rt->rt6i_flags & RTF_CACHE))
1966 goto unlock;
1967
e659ba31
DA		 1968 if (from->nh) {
1969 struct fib6_nh_match_arg arg = {
1970 .dev = rt->dst.dev,
1971 .gw = &rt->rt6i_gateway,
1972 };
1973
1974 nexthop_for_each_fib6_nh(from->nh, fib6_nh_find_match, &arg);
1975
1976 if (!arg.match)
cff6a327 1977 goto unlock;
e659ba31
DA		 1978 fib6_nh = arg.match;
1979 } else {
1980 fib6_nh = from->fib6_nh;
1981 }
1982 fib6_nh_update_exception(fib6_nh, from->fib6_src.plen, rt);
193f3685 1983unlock:
35732d01
WW		 1984 rcu_read_unlock();
1985}
1986
e9fa1495
SB		 1987static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev,
1988 struct rt6_info *rt, int mtu)
1989{
1990 /* If the new MTU is lower than the route PMTU, this new MTU will be the
1991 * lowest MTU in the path: always allow updating the route PMTU to
1992 * reflect PMTU decreases.
1993 *
1994 * If the new MTU is higher, and the route PMTU is equal to the local
1995 * MTU, this means the old MTU is the lowest in the path, so allow
1996 * updating it: if other nodes now have lower MTUs, PMTU discovery will
1997 * handle this.
1998 */
1999
2000 if (dst_mtu(&rt->dst) >= mtu)
2001 return true;
2002
2003 if (dst_mtu(&rt->dst) == idev->cnf.mtu6)
2004 return true;
2005
2006 return false;
2007}
2008
2009static void rt6_exceptions_update_pmtu(struct inet6_dev *idev,
cc5c073a 2010 const struct fib6_nh *nh, int mtu)
f5bbe7ee
WW		 2011{
2012 struct rt6_exception_bucket *bucket;
2013 struct rt6_exception *rt6_ex;
2014 int i;
2015
cc5c073a 2016 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
e9fa1495
SB		 2017 if (!bucket)
2018 return;
2019
2020 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2021 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
2022 struct rt6_info *entry = rt6_ex->rt6i;
2023
2024 /* For RTF_CACHE with rt6i_pmtu == 0 (i.e. a redirected
d4ead6b3 2025 * route), the metrics of its rt->from have already
e9fa1495
SB		 2026 * been updated.
2027 */
d4ead6b3 2028 if (dst_metric_raw(&entry->dst, RTAX_MTU) &&
e9fa1495 2029 rt6_mtu_change_route_allowed(idev, entry, mtu))
d4ead6b3 2030 dst_metric_set(&entry->dst, RTAX_MTU, mtu);
f5bbe7ee 2031 }
e9fa1495 2032 bucket++;
f5bbe7ee
WW		 2033 }
2034}
2035
b16cb459
WW		 2036#define RTF_CACHE_GATEWAY (RTF_GATEWAY | RTF_CACHE)
2037
cc5c073a
DA		 2038static void fib6_nh_exceptions_clean_tohost(const struct fib6_nh *nh,
2039 const struct in6_addr *gateway)
b16cb459
WW		 2040{
2041 struct rt6_exception_bucket *bucket;
2042 struct rt6_exception *rt6_ex;
2043 struct hlist_node *tmp;
2044 int i;
2045
cc5c073a 2046 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
b16cb459
WW		 2047 return;
2048
2049 spin_lock_bh(&rt6_exception_lock);
cc5c073a 2050 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
b16cb459
WW		 2051 if (bucket) {
2052 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2053 hlist_for_each_entry_safe(rt6_ex, tmp,
2054 &bucket->chain, hlist) {
2055 struct rt6_info *entry = rt6_ex->rt6i;
2056
2057 if ((entry->rt6i_flags & RTF_CACHE_GATEWAY) ==
2058 RTF_CACHE_GATEWAY &&
2059 ipv6_addr_equal(gateway,
2060 &entry->rt6i_gateway)) {
2061 rt6_remove_exception(bucket, rt6_ex);
2062 }
2063 }
2064 bucket++;
2065 }
2066 }
2067
2068 spin_unlock_bh(&rt6_exception_lock);
2069}
2070
c757faa8
WW		 2071static void rt6_age_examine_exception(struct rt6_exception_bucket *bucket,
2072 struct rt6_exception *rt6_ex,
2073 struct fib6_gc_args *gc_args,
2074 unsigned long now)
2075{
2076 struct rt6_info *rt = rt6_ex->rt6i;
2077
1859bac0
PA		 2078 /* we are pruning and obsoleting aged-out and non gateway exceptions
2079 * even if others have still references to them, so that on next
2080 * dst_check() such references can be dropped.
2081 * EXPIRES exceptions - e.g. pmtu-generated ones are pruned when
2082 * expired, independently from their aging, as per RFC 8201 section 4
2083 */
31afeb42
WW		 2084 if (!(rt->rt6i_flags & RTF_EXPIRES)) {
2085 if (time_after_eq(now, rt->dst.lastuse + gc_args->timeout)) {
2086 RT6_TRACE("aging clone %p\n", rt);
2087 rt6_remove_exception(bucket, rt6_ex);
2088 return;
2089 }
2090 } else if (time_after(jiffies, rt->dst.expires)) {
2091 RT6_TRACE("purging expired route %p\n", rt);
c757faa8
WW		 2092 rt6_remove_exception(bucket, rt6_ex);
2093 return;
31afeb42
WW		 2094 }
2095
2096 if (rt->rt6i_flags & RTF_GATEWAY) {
c757faa8 2097 struct neighbour *neigh;
c757faa8 2098
1bfa26ff 2099 neigh = __ipv6_neigh_lookup_noref(rt->dst.dev, &rt->rt6i_gateway);
1bfa26ff 2100
b7a320c3 2101 if (!(neigh && (neigh->flags & NTF_ROUTER))) {
c757faa8
WW		 2102 RT6_TRACE("purging route %p via non-router but gateway\n",
2103 rt);
2104 rt6_remove_exception(bucket, rt6_ex);
2105 return;
2106 }
2107 }
31afeb42 2108
c757faa8
WW		 2109 gc_args->more++;
2110}
2111
cc5c073a 2112static void fib6_nh_age_exceptions(const struct fib6_nh *nh,
c0b220cf
DA		 2113 struct fib6_gc_args *gc_args,
2114 unsigned long now)
c757faa8
WW		 2115{
2116 struct rt6_exception_bucket *bucket;
2117 struct rt6_exception *rt6_ex;
2118 struct hlist_node *tmp;
2119 int i;
2120
cc5c073a 2121 if (!rcu_access_pointer(nh->rt6i_exception_bucket))
c757faa8
WW		 2122 return;
2123
1bfa26ff
ED		 2124 rcu_read_lock_bh();
2125 spin_lock(&rt6_exception_lock);
cc5c073a 2126 bucket = fib6_nh_get_excptn_bucket(nh, &rt6_exception_lock);
c757faa8
WW		 2127 if (bucket) {
2128 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
2129 hlist_for_each_entry_safe(rt6_ex, tmp,
2130 &bucket->chain, hlist) {
2131 rt6_age_examine_exception(bucket, rt6_ex,
2132 gc_args, now);
2133 }
2134 bucket++;
2135 }
2136 }
1bfa26ff
ED		 2137 spin_unlock(&rt6_exception_lock);
2138 rcu_read_unlock_bh();
c757faa8
WW		 2139}
2140
e659ba31
DA		 2141struct fib6_nh_age_excptn_arg {
2142 struct fib6_gc_args *gc_args;
2143 unsigned long now;
2144};
2145
2146static int rt6_nh_age_exceptions(struct fib6_nh *nh, void *_arg)
2147{
2148 struct fib6_nh_age_excptn_arg *arg = _arg;
2149
2150 fib6_nh_age_exceptions(nh, arg->gc_args, arg->now);
2151 return 0;
2152}
2153
cc5c073a 2154void rt6_age_exceptions(struct fib6_info *f6i,
c0b220cf
DA		 2155 struct fib6_gc_args *gc_args,
2156 unsigned long now)
2157{
e659ba31
DA		 2158 if (f6i->nh) {
2159 struct fib6_nh_age_excptn_arg arg = {
2160 .gc_args = gc_args,
2161 .now = now
2162 };
2163
2164 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_age_exceptions,
2165 &arg);
2166 } else {
2167 fib6_nh_age_exceptions(f6i->fib6_nh, gc_args, now);
2168 }
c0b220cf
DA		 2169}
2170
1d053da9 2171/* must be called with rcu lock held */
effda4dd
DA		 2172int fib6_table_lookup(struct net *net, struct fib6_table *table, int oif,
2173 struct flowi6 *fl6, struct fib6_result *res, int strict)
1da177e4 2174{
367efcb9 2175 struct fib6_node *fn, *saved_fn;
1da177e4 2176
6454743b 2177 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
367efcb9 2178 saved_fn = fn;
1da177e4 2179
a3c00e46 2180redo_rt6_select:
effda4dd
DA		 2181 rt6_select(net, fn, oif, res, strict);
2182 if (res->f6i == net->ipv6.fib6_null_entry) {
a3c00e46
MKL		 2183 fn = fib6_backtrack(fn, &fl6->saddr);
2184 if (fn)
2185 goto redo_rt6_select;
367efcb9
MKL		 2186 else if (strict & RT6_LOOKUP_F_REACHABLE) {
2187 /* also consider unreachable route */
2188 strict &= ~RT6_LOOKUP_F_REACHABLE;
2189 fn = saved_fn;
2190 goto redo_rt6_select;
367efcb9 2191 }
a3c00e46
MKL		 2192 }
2193
effda4dd 2194 trace_fib6_table_lookup(net, res, table, fl6);
fb9de91e 2195
effda4dd 2196 return 0;
1d053da9
DA		 2197}
2198
2199struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table,
2200 int oif, struct flowi6 *fl6,
2201 const struct sk_buff *skb, int flags)
2202{
b1d40991 2203 struct fib6_result res = {};
0e09edcc 2204 struct rt6_info *rt = NULL;
1d053da9
DA		 2205 int strict = 0;
2206
0e09edcc
WW		 2207 WARN_ON_ONCE((flags & RT6_LOOKUP_F_DST_NOREF) &&
2208 !rcu_read_lock_held());
2209
1d053da9
DA		 2210 strict |= flags & RT6_LOOKUP_F_IFACE;
2211 strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE;
2212 if (net->ipv6.devconf_all->forwarding == 0)
2213 strict |= RT6_LOOKUP_F_REACHABLE;
2214
2215 rcu_read_lock();
2216
effda4dd 2217 fib6_table_lookup(net, table, oif, fl6, &res, strict);
0e09edcc
WW		 2218 if (res.f6i == net->ipv6.fib6_null_entry)
2219 goto out;
23fb93a4 2220
b1d40991 2221 fib6_select_path(net, &res, fl6, oif, false, skb, strict);
d83009d4 2222
23fb93a4 2223 /*Search through exception table */
7e4b5128 2224 rt = rt6_find_cached_rt(&res, &fl6->daddr, &fl6->saddr);
23fb93a4 2225 if (rt) {
0e09edcc 2226 goto out;
3da59bd9 2227 } else if (unlikely((fl6->flowi6_flags & FLOWI_FLAG_KNOWN_NH) &&
b1d40991 2228 !res.nh->fib_nh_gw_family)) {
3da59bd9
MKL		 2229 /* Create a RTF_CACHE clone which will not be
2230 * owned by the fib6 tree. It is for the special case where
2231 * the daddr in the skb during the neighbor look-up is different
2232 * from the fl6->daddr used to look-up route here.
2233 */
0e09edcc 2234 rt = ip6_rt_cache_alloc(&res, &fl6->daddr, NULL);
c71099ac 2235
0e09edcc
WW		 2236 if (rt) {
2237 /* 1 refcnt is taken during ip6_rt_cache_alloc().
2238 * As rt6_uncached_list_add() does not consume refcnt,
2239 * this refcnt is always returned to the caller even
2240 * if caller sets RT6_LOOKUP_F_DST_NOREF flag.
1cfb71ee 2241 */
0e09edcc 2242 rt6_uncached_list_add(rt);
0e09edcc 2243 rcu_read_unlock();
b811580d 2244
0e09edcc
WW		 2245 return rt;
2246 }
d52d3997
MKL		 2247 } else {
2248 /* Get a percpu copy */
951f788a 2249 local_bh_disable();
0e09edcc 2250 rt = rt6_get_pcpu_route(&res);
d52d3997 2251
0e09edcc
WW		 2252 if (!rt)
2253 rt = rt6_make_pcpu_route(net, &res);
93531c67 2254
951f788a 2255 local_bh_enable();
d52d3997 2256 }
0e09edcc
WW		 2257out:
2258 if (!rt)
2259 rt = net->ipv6.ip6_null_entry;
2260 if (!(flags & RT6_LOOKUP_F_DST_NOREF))
2261 ip6_hold_safe(net, &rt);
2262 rcu_read_unlock();
2263
2264 return rt;
1da177e4 2265}
9ff74384 2266EXPORT_SYMBOL_GPL(ip6_pol_route);
1da177e4 2267
55cced4f 2268INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net,
b75cc8f9
DA		 2269 struct fib6_table *table,
2270 struct flowi6 *fl6,
2271 const struct sk_buff *skb,
2272 int flags)
4acad72d 2273{
b75cc8f9 2274 return ip6_pol_route(net, table, fl6->flowi6_iif, fl6, skb, flags);
4acad72d
PE		 2275}
2276
d409b847
MB		 2277struct dst_entry *ip6_route_input_lookup(struct net *net,
2278 struct net_device *dev,
b75cc8f9
DA		 2279 struct flowi6 *fl6,
2280 const struct sk_buff *skb,
2281 int flags)
72331bc0
SL		 2282{
2283 if (rt6_need_strict(&fl6->daddr) && dev->type != ARPHRD_PIMREG)
2284 flags |= RT6_LOOKUP_F_IFACE;
2285
b75cc8f9 2286 return fib6_rule_lookup(net, fl6, skb, flags, ip6_pol_route_input);
72331bc0 2287}
d409b847 2288EXPORT_SYMBOL_GPL(ip6_route_input_lookup);
72331bc0 2289
23aebdac 2290static void ip6_multipath_l3_keys(const struct sk_buff *skb,
5e5d6fed
RP		 2291 struct flow_keys *keys,
2292 struct flow_keys *flkeys)
23aebdac
JS		 2293{
2294 const struct ipv6hdr *outer_iph = ipv6_hdr(skb);
2295 const struct ipv6hdr *key_iph = outer_iph;
5e5d6fed 2296 struct flow_keys *_flkeys = flkeys;
23aebdac
JS		 2297 const struct ipv6hdr *inner_iph;
2298 const struct icmp6hdr *icmph;
2299 struct ipv6hdr _inner_iph;
cea67a2d 2300 struct icmp6hdr _icmph;
23aebdac
JS		 2301
2302 if (likely(outer_iph->nexthdr != IPPROTO_ICMPV6))
2303 goto out;
2304
cea67a2d
ED		 2305 icmph = skb_header_pointer(skb, skb_transport_offset(skb),
2306 sizeof(_icmph), &_icmph);
2307 if (!icmph)
2308 goto out;
2309
54074f1d 2310 if (!icmpv6_is_err(icmph->icmp6_type))
23aebdac
JS		 2311 goto out;
2312
2313 inner_iph = skb_header_pointer(skb,
2314 skb_transport_offset(skb) + sizeof(*icmph),
2315 sizeof(_inner_iph), &_inner_iph);
2316 if (!inner_iph)
2317 goto out;
2318
2319 key_iph = inner_iph;
5e5d6fed 2320 _flkeys = NULL;
23aebdac 2321out:
5e5d6fed
RP		 2322 if (_flkeys) {
2323 keys->addrs.v6addrs.src = _flkeys->addrs.v6addrs.src;
2324 keys->addrs.v6addrs.dst = _flkeys->addrs.v6addrs.dst;
2325 keys->tags.flow_label = _flkeys->tags.flow_label;
2326 keys->basic.ip_proto = _flkeys->basic.ip_proto;
2327 } else {
2328 keys->addrs.v6addrs.src = key_iph->saddr;
2329 keys->addrs.v6addrs.dst = key_iph->daddr;
fa1be7e0 2330 keys->tags.flow_label = ip6_flowlabel(key_iph);
5e5d6fed
RP		 2331 keys->basic.ip_proto = key_iph->nexthdr;
2332 }
23aebdac
JS		 2333}
2334
73c2c5cb
IS		 2335static u32 rt6_multipath_custom_hash_outer(const struct net *net,
2336 const struct sk_buff *skb,
2337 bool *p_has_inner)
2338{
2339 u32 hash_fields = ip6_multipath_hash_fields(net);
2340 struct flow_keys keys, hash_keys;
2341
2342 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2343 return 0;
2344
2345 memset(&hash_keys, 0, sizeof(hash_keys));
2346 skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
2347
2348 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2349 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2350 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2351 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2352 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2353 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2354 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2355 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2356 hash_keys.tags.flow_label = keys.tags.flow_label;
2357 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
2358 hash_keys.ports.src = keys.ports.src;
2359 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2360 hash_keys.ports.dst = keys.ports.dst;
2361
2362 *p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION);
2363 return flow_hash_from_keys(&hash_keys);
2364}
2365
2366static u32 rt6_multipath_custom_hash_inner(const struct net *net,
2367 const struct sk_buff *skb,
2368 bool has_inner)
2369{
2370 u32 hash_fields = ip6_multipath_hash_fields(net);
2371 struct flow_keys keys, hash_keys;
2372
2373 /* We assume the packet carries an encapsulation, but if none was
2374 * encountered during dissection of the outer flow, then there is no
2375 * point in calling the flow dissector again.
2376 */
2377 if (!has_inner)
2378 return 0;
2379
2380 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK))
2381 return 0;
2382
2383 memset(&hash_keys, 0, sizeof(hash_keys));
2384 skb_flow_dissect_flow_keys(skb, &keys, 0);
2385
2386 if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION))
2387 return 0;
2388
2389 if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2390 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2391 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2392 hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
2393 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2394 hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
2395 } else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2396 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2397 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
2398 hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
2399 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
2400 hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
2401 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
2402 hash_keys.tags.flow_label = keys.tags.flow_label;
2403 }
2404
2405 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
2406 hash_keys.basic.ip_proto = keys.basic.ip_proto;
2407 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
2408 hash_keys.ports.src = keys.ports.src;
2409 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
2410 hash_keys.ports.dst = keys.ports.dst;
2411
2412 return flow_hash_from_keys(&hash_keys);
2413}
2414
2415static u32 rt6_multipath_custom_hash_skb(const struct net *net,
2416 const struct sk_buff *skb)
2417{
2418 u32 mhash, mhash_inner;
2419 bool has_inner = true;
2420
2421 mhash = rt6_multipath_custom_hash_outer(net, skb, &has_inner);
2422 mhash_inner = rt6_multipath_custom_hash_inner(net, skb, has_inner);
2423
2424 return jhash_2words(mhash, mhash_inner, 0);
2425}
2426
2427static u32 rt6_multipath_custom_hash_fl6(const struct net *net,
2428 const struct flowi6 *fl6)
2429{
2430 u32 hash_fields = ip6_multipath_hash_fields(net);
2431 struct flow_keys hash_keys;
2432
2433 if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
2434 return 0;
2435
2436 memset(&hash_keys, 0, sizeof(hash_keys));
2437 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2438 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
2439 hash_keys.addrs.v6addrs.src = fl6->saddr;
2440 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
2441 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2442 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
2443 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2444 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
2445 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2446 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
2447 hash_keys.ports.src = fl6->fl6_sport;
2448 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
2449 hash_keys.ports.dst = fl6->fl6_dport;
2450
2451 return flow_hash_from_keys(&hash_keys);
2452}
2453
23aebdac 2454/* if skb is set it will be used and fl6 can be NULL */
b4bac172
DA		 2455u32 rt6_multipath_hash(const struct net *net, const struct flowi6 *fl6,
2456 const struct sk_buff *skb, struct flow_keys *flkeys)
23aebdac
JS		 2457{
2458 struct flow_keys hash_keys;
b95b6e07 2459 u32 mhash = 0;
23aebdac 2460
bbfa047a 2461 switch (ip6_multipath_hash_policy(net)) {
b4bac172
DA		 2462 case 0:
2463 memset(&hash_keys, 0, sizeof(hash_keys));
2464 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2465 if (skb) {
2466 ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
2467 } else {
2468 hash_keys.addrs.v6addrs.src = fl6->saddr;
2469 hash_keys.addrs.v6addrs.dst = fl6->daddr;
fa1be7e0 2470 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
b4bac172
DA		 2471 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2472 }
b95b6e07 2473 mhash = flow_hash_from_keys(&hash_keys);
b4bac172
DA		 2474 break;
2475 case 1:
2476 if (skb) {
2477 unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
2478 struct flow_keys keys;
2479
2480 /* short-circuit if we already have L4 hash present */
2481 if (skb->l4_hash)
2482 return skb_get_hash_raw(skb) >> 1;
2483
2484 memset(&hash_keys, 0, sizeof(hash_keys));
2485
13fdb940 2486 if (!flkeys) {
b4bac172
DA		 2487 skb_flow_dissect_flow_keys(skb, &keys, flag);
2488 flkeys = &keys;
2489 }
2490 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2491 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2492 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2493 hash_keys.ports.src = flkeys->ports.src;
2494 hash_keys.ports.dst = flkeys->ports.dst;
2495 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2496 } else {
2497 memset(&hash_keys, 0, sizeof(hash_keys));
2498 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2499 hash_keys.addrs.v6addrs.src = fl6->saddr;
2500 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2501 hash_keys.ports.src = fl6->fl6_sport;
2502 hash_keys.ports.dst = fl6->fl6_dport;
2503 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2504 }
b95b6e07 2505 mhash = flow_hash_from_keys(&hash_keys);
b4bac172 2506 break;
d8f74f09
SS		 2507 case 2:
2508 memset(&hash_keys, 0, sizeof(hash_keys));
2509 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2510 if (skb) {
2511 struct flow_keys keys;
2512
2513 if (!flkeys) {
2514 skb_flow_dissect_flow_keys(skb, &keys, 0);
2515 flkeys = &keys;
2516 }
2517
2518 /* Inner can be v4 or v6 */
2519 if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2520 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
2521 hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
2522 hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
2523 } else if (flkeys->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2524 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2525 hash_keys.addrs.v6addrs.src = flkeys->addrs.v6addrs.src;
2526 hash_keys.addrs.v6addrs.dst = flkeys->addrs.v6addrs.dst;
2527 hash_keys.tags.flow_label = flkeys->tags.flow_label;
2528 hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
2529 } else {
2530 /* Same as case 0 */
2531 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2532 ip6_multipath_l3_keys(skb, &hash_keys, flkeys);
2533 }
2534 } else {
2535 /* Same as case 0 */
2536 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
2537 hash_keys.addrs.v6addrs.src = fl6->saddr;
2538 hash_keys.addrs.v6addrs.dst = fl6->daddr;
2539 hash_keys.tags.flow_label = (__force u32)flowi6_get_flowlabel(fl6);
2540 hash_keys.basic.ip_proto = fl6->flowi6_proto;
2541 }
b95b6e07 2542 mhash = flow_hash_from_keys(&hash_keys);
d8f74f09 2543 break;
73c2c5cb
IS		 2544 case 3:
2545 if (skb)
2546 mhash = rt6_multipath_custom_hash_skb(net, skb);
2547 else
2548 mhash = rt6_multipath_custom_hash_fl6(net, fl6);
2549 break;
23aebdac
JS		 2550 }
2551
9a2a537a 2552 return mhash >> 1;
23aebdac
JS		 2553}
2554
67f415dd 2555/* Called with rcu held */
c71099ac
TG		 2556void ip6_route_input(struct sk_buff *skb)
2557{
b71d1d42 2558 const struct ipv6hdr *iph = ipv6_hdr(skb);
c346dca1 2559 struct net *net = dev_net(skb->dev);
67f415dd 2560 int flags = RT6_LOOKUP_F_HAS_SADDR | RT6_LOOKUP_F_DST_NOREF;
904af04d 2561 struct ip_tunnel_info *tun_info;
4c9483b2 2562 struct flowi6 fl6 = {
e0d56fdd 2563 .flowi6_iif = skb->dev->ifindex,
4c9483b2
DM		 2564 .daddr = iph->daddr,
2565 .saddr = iph->saddr,
6502ca52 2566 .flowlabel = ip6_flowinfo(iph),
4c9483b2
DM		 2567 .flowi6_mark = skb->mark,
2568 .flowi6_proto = iph->nexthdr,
c71099ac 2569 };
5e5d6fed 2570 struct flow_keys *flkeys = NULL, _flkeys;
adaa70bb 2571
904af04d 2572 tun_info = skb_tunnel_info(skb);
46fa062a 2573 if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
904af04d 2574 fl6.flowi6_tun_key.tun_id = tun_info->key.tun_id;
5e5d6fed
RP		 2575
2576 if (fib6_rules_early_flow_dissect(net, skb, &fl6, &_flkeys))
2577 flkeys = &_flkeys;
2578
23aebdac 2579 if (unlikely(fl6.flowi6_proto == IPPROTO_ICMPV6))
b4bac172 2580 fl6.mp_hash = rt6_multipath_hash(net, &fl6, skb, flkeys);
06e9d040 2581 skb_dst_drop(skb);
67f415dd
WW		 2582 skb_dst_set_noref(skb, ip6_route_input_lookup(net, skb->dev,
2583 &fl6, skb, flags));
c71099ac
TG		 2584}
2585
55cced4f 2586INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net,
b75cc8f9
DA		 2587 struct fib6_table *table,
2588 struct flowi6 *fl6,
2589 const struct sk_buff *skb,
2590 int flags)
1da177e4 2591{
b75cc8f9 2592 return ip6_pol_route(net, table, fl6->flowi6_oif, fl6, skb, flags);
c71099ac
TG		 2593}
2594
90317bcd
GN		 2595static struct dst_entry *ip6_route_output_flags_noref(struct net *net,
2596 const struct sock *sk,
2597 struct flowi6 *fl6,
2598 int flags)
c71099ac 2599{
d46a9d67 2600 bool any_src;
c71099ac 2601
3ede0bbc
RS		 2602 if (ipv6_addr_type(&fl6->daddr) &
2603 (IPV6_ADDR_MULTICAST | IPV6_ADDR_LINKLOCAL)) {
4c1feac5
DA		 2604 struct dst_entry *dst;
2605
7d9e5f42 2606 /* This function does not take refcnt on the dst */
4c1feac5
DA		 2607 dst = l3mdev_link_scope_lookup(net, fl6);
2608 if (dst)
2609 return dst;
2610 }
ca254490 2611
1fb9489b 2612 fl6->flowi6_iif = LOOPBACK_IFINDEX;
4dc27d1c 2613
7d9e5f42 2614 flags |= RT6_LOOKUP_F_DST_NOREF;
d46a9d67 2615 any_src = ipv6_addr_any(&fl6->saddr);
741a11d9 2616 if ((sk && sk->sk_bound_dev_if) || rt6_need_strict(&fl6->daddr) ||
d46a9d67 2617 (fl6->flowi6_oif && any_src))
77d16f45 2618 flags |= RT6_LOOKUP_F_IFACE;
c71099ac 2619
d46a9d67 2620 if (!any_src)
adaa70bb 2621 flags |= RT6_LOOKUP_F_HAS_SADDR;
0c9a2ac1
YH		 2622 else if (sk)
2623 flags |= rt6_srcprefs2flags(inet6_sk(sk)->srcprefs);
adaa70bb 2624
b75cc8f9 2625 return fib6_rule_lookup(net, fl6, NULL, flags, ip6_pol_route_output);
1da177e4 2626}
7d9e5f42
WW		 2627
2628struct dst_entry *ip6_route_output_flags(struct net *net,
2629 const struct sock *sk,
2630 struct flowi6 *fl6,
2631 int flags)
2632{
13fdb940
SK		 2633 struct dst_entry *dst;
2634 struct rt6_info *rt6;
7d9e5f42 2635
13fdb940
SK		 2636 rcu_read_lock();
2637 dst = ip6_route_output_flags_noref(net, sk, fl6, flags);
2638 rt6 = (struct rt6_info *)dst;
2639 /* For dst cached in uncached_list, refcnt is already taken. */
d288a162 2640 if (list_empty(&rt6->dst.rt_uncached) && !dst_hold_safe(dst)) {
13fdb940
SK		 2641 dst = &net->ipv6.ip6_null_entry->dst;
2642 dst_hold(dst);
2643 }
2644 rcu_read_unlock();
7d9e5f42 2645
13fdb940 2646 return dst;
7d9e5f42 2647}
6f21c96a 2648EXPORT_SYMBOL_GPL(ip6_route_output_flags);
1da177e4 2649
2774c131 2650struct dst_entry *ip6_blackhole_route(struct net *net, struct dst_entry *dst_orig)
14e50e57 2651{
5c1e6aa3 2652 struct rt6_info *rt, *ort = (struct rt6_info *) dst_orig;
1dbe3252 2653 struct net_device *loopback_dev = net->loopback_dev;
14e50e57
DM		 2654 struct dst_entry *new = NULL;
2655
1dbe3252 2656 rt = dst_alloc(&ip6_dst_blackhole_ops, loopback_dev, 1,
62cf27e5 2657 DST_OBSOLETE_DEAD, 0);
14e50e57 2658 if (rt) {
0a1f5962 2659 rt6_info_init(rt);
81eb8447 2660 atomic_inc(&net->ipv6.rt6_stats->fib_rt_alloc);
8104891b 2661
0a1f5962 2662 new = &rt->dst;
14e50e57 2663 new->__use = 1;
352e512c 2664 new->input = dst_discard;
ede2059d 2665 new->output = dst_discard_out;
14e50e57 2666
0a1f5962 2667 dst_copy_metrics(new, &ort->dst);
14e50e57 2668
1dbe3252 2669 rt->rt6i_idev = in6_dev_get(loopback_dev);
4e3fd7a0 2670 rt->rt6i_gateway = ort->rt6i_gateway;
0a1f5962 2671 rt->rt6i_flags = ort->rt6i_flags & ~RTF_PCPU;
14e50e57
DM		 2672
2673 memcpy(&rt->rt6i_dst, &ort->rt6i_dst, sizeof(struct rt6key));
2674#ifdef CONFIG_IPV6_SUBTREES
2675 memcpy(&rt->rt6i_src, &ort->rt6i_src, sizeof(struct rt6key));
2676#endif
14e50e57
DM		 2677 }
2678
69ead7af
DM		 2679 dst_release(dst_orig);
2680 return new ? new : ERR_PTR(-ENOMEM);
14e50e57 2681}
14e50e57 2682
1da177e4
LT		 2683/*
2684 * Destination cache support functions
2685 */
2686
8d1c802b 2687static bool fib6_check(struct fib6_info *f6i, u32 cookie)
4b32b5ad 2688{
93531c67
DA		 2689 u32 rt_cookie = 0;
2690
8ae86971 2691 if (!fib6_get_cookie_safe(f6i, &rt_cookie) || rt_cookie != cookie)
93531c67
DA		 2692 return false;
2693
2694 if (fib6_check_expired(f6i))
2695 return false;
2696
2697 return true;
4b32b5ad
MKL		 2698}
2699
a68886a6
DA		 2700static struct dst_entry *rt6_check(struct rt6_info *rt,
2701 struct fib6_info *from,
2702 u32 cookie)
3da59bd9 2703{
36143645 2704 u32 rt_cookie = 0;
c5cff856 2705
49d05fe2 2706 if (!from || !fib6_get_cookie_safe(from, &rt_cookie) ||
93531c67 2707 rt_cookie != cookie)
3da59bd9
MKL		 2708 return NULL;
2709
2710 if (rt6_check_expired(rt))
2711 return NULL;
2712
2713 return &rt->dst;
2714}
2715
a68886a6
DA		 2716static struct dst_entry *rt6_dst_from_check(struct rt6_info *rt,
2717 struct fib6_info *from,
2718 u32 cookie)
3da59bd9 2719{
5973fb1e
MKL		 2720 if (!__rt6_check_expired(rt) &&
2721 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
a68886a6 2722 fib6_check(from, cookie))
3da59bd9
MKL		 2723 return &rt->dst;
2724 else
2725 return NULL;
2726}
2727
bbd807df
BV		 2728INDIRECT_CALLABLE_SCOPE struct dst_entry *ip6_dst_check(struct dst_entry *dst,
2729 u32 cookie)
1da177e4 2730{
a87b7dc9 2731 struct dst_entry *dst_ret;
a68886a6 2732 struct fib6_info *from;
1da177e4
LT		 2733 struct rt6_info *rt;
2734
a87b7dc9
DA		 2735 rt = container_of(dst, struct rt6_info, dst);
2736
8f34e53b
DA		 2737 if (rt->sernum)
2738 return rt6_is_valid(rt) ? dst : NULL;
2739
a87b7dc9 2740 rcu_read_lock();
1da177e4 2741
6f3118b5
ND		 2742 /* All IPV6 dsts are created with ->obsolete set to the value
2743 * DST_OBSOLETE_FORCE_CHK which forces validation calls down
2744 * into this function always.
2745 */
e3bc10bd 2746
a68886a6 2747 from = rcu_dereference(rt->from);
4b32b5ad 2748
a68886a6 2749 if (from && (rt->rt6i_flags & RTF_PCPU ||
d288a162 2750 unlikely(!list_empty(&rt->dst.rt_uncached))))
a68886a6 2751 dst_ret = rt6_dst_from_check(rt, from, cookie);
3da59bd9 2752 else
a68886a6 2753 dst_ret = rt6_check(rt, from, cookie);
a87b7dc9
DA		 2754
2755 rcu_read_unlock();
2756
2757 return dst_ret;
1da177e4 2758}
9c97921a 2759EXPORT_INDIRECT_CALLABLE(ip6_dst_check);
1da177e4
LT		 2760
2761static struct dst_entry *ip6_negative_advice(struct dst_entry *dst)
2762{
2763 struct rt6_info *rt = (struct rt6_info *) dst;
2764
2765 if (rt) {
54c1a859 2766 if (rt->rt6i_flags & RTF_CACHE) {
c3c14da0 2767 rcu_read_lock();
54c1a859 2768 if (rt6_check_expired(rt)) {
93531c67 2769 rt6_remove_exception_rt(rt);
54c1a859
YH		 2770 dst = NULL;
2771 }
c3c14da0 2772 rcu_read_unlock();
54c1a859 2773 } else {
1da177e4 2774 dst_release(dst);
54c1a859
YH		 2775 dst = NULL;
2776 }
1da177e4 2777 }
54c1a859 2778 return dst;
1da177e4
LT		 2779}
2780
2781static void ip6_link_failure(struct sk_buff *skb)
2782{
2783 struct rt6_info *rt;
2784
3ffe533c 2785 icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
1da177e4 2786
adf30907 2787 rt = (struct rt6_info *) skb_dst(skb);
1da177e4 2788 if (rt) {
8a14e46f 2789 rcu_read_lock();
1eb4f758 2790 if (rt->rt6i_flags & RTF_CACHE) {
761f6026 2791 rt6_remove_exception_rt(rt);
c5cff856 2792 } else {
a68886a6 2793 struct fib6_info *from;
c5cff856
WW		 2794 struct fib6_node *fn;
2795
a68886a6
DA		 2796 from = rcu_dereference(rt->from);
2797 if (from) {
2798 fn = rcu_dereference(from->fib6_node);
2799 if (fn && (rt->rt6i_flags & RTF_DEFAULT))
aafc2e32 2800 WRITE_ONCE(fn->fn_sernum, -1);
a68886a6 2801 }
1eb4f758 2802 }
8a14e46f 2803 rcu_read_unlock();
1da177e4
LT		 2804 }
2805}
2806
6a3e030f
DA		 2807static void rt6_update_expires(struct rt6_info *rt0, int timeout)
2808{
a68886a6
DA		 2809 if (!(rt0->rt6i_flags & RTF_EXPIRES)) {
2810 struct fib6_info *from;
2811
2812 rcu_read_lock();
2813 from = rcu_dereference(rt0->from);
2814 if (from)
2815 rt0->dst.expires = from->expires;
2816 rcu_read_unlock();
2817 }
6a3e030f
DA		 2818
2819 dst_set_expires(&rt0->dst, timeout);
2820 rt0->rt6i_flags |= RTF_EXPIRES;
2821}
2822
45e4fd26
MKL		 2823static void rt6_do_update_pmtu(struct rt6_info *rt, u32 mtu)
2824{
2825 struct net *net = dev_net(rt->dst.dev);
2826
d4ead6b3 2827 dst_metric_set(&rt->dst, RTAX_MTU, mtu);
45e4fd26 2828 rt->rt6i_flags |= RTF_MODIFIED;
45e4fd26
MKL		 2829 rt6_update_expires(rt, net->ipv6.sysctl.ip6_rt_mtu_expires);
2830}
2831
0d3f6d29
MKL		 2832static bool rt6_cache_allowed_for_pmtu(const struct rt6_info *rt)
2833{
2834 return !(rt->rt6i_flags & RTF_CACHE) &&
1490ed2a 2835 (rt->rt6i_flags & RTF_PCPU || rcu_access_pointer(rt->from));
0d3f6d29
MKL		 2836}
2837
45e4fd26 2838static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk,
bd085ef6
HL		 2839 const struct ipv6hdr *iph, u32 mtu,
2840 bool confirm_neigh)
1da177e4 2841{
0dec879f 2842 const struct in6_addr *daddr, *saddr;
67ba4152 2843 struct rt6_info *rt6 = (struct rt6_info *)dst;
1da177e4 2844
09454fd0
MŻ		 2845 /* Note: do *NOT* check dst_metric_locked(dst, RTAX_MTU)
2846 * IPv6 pmtu discovery isn't optional, so 'mtu lock' cannot disable it.
2847 * [see also comment in rt6_mtu_change_route()]
2848 */
19bda36c 2849
0dec879f
JA		 2850 if (iph) {
2851 daddr = &iph->daddr;
2852 saddr = &iph->saddr;
2853 } else if (sk) {
2854 daddr = &sk->sk_v6_daddr;
2855 saddr = &inet6_sk(sk)->saddr;
2856 } else {
2857 daddr = NULL;
2858 saddr = NULL;
2859 }
bd085ef6
HL		 2860
2861 if (confirm_neigh)
2862 dst_confirm_neigh(dst, daddr);
2863
4a65dff8
GK		 2864 if (mtu < IPV6_MIN_MTU)
2865 return;
45e4fd26
MKL		 2866 if (mtu >= dst_mtu(dst))
2867 return;
9d289715 2868
0d3f6d29 2869 if (!rt6_cache_allowed_for_pmtu(rt6)) {
45e4fd26 2870 rt6_do_update_pmtu(rt6, mtu);
2b760fcf
WW		 2871 /* update rt6_ex->stamp for cache */
2872 if (rt6->rt6i_flags & RTF_CACHE)
2873 rt6_update_exception_stamp_rt(rt6);
0dec879f 2874 } else if (daddr) {
85bd05de 2875 struct fib6_result res = {};
45e4fd26
MKL		 2876 struct rt6_info *nrt6;
2877
4d85cd0c 2878 rcu_read_lock();
85bd05de 2879 res.f6i = rcu_dereference(rt6->from);
43a4b60d
DA		 2880 if (!res.f6i)
2881 goto out_unlock;
2882
7d21fec9
DA		 2883 res.fib6_flags = res.f6i->fib6_flags;
2884 res.fib6_type = res.f6i->fib6_type;
2885
2d44234b
DA		 2886 if (res.f6i->nh) {
2887 struct fib6_nh_match_arg arg = {
2888 .dev = dst->dev,
2889 .gw = &rt6->rt6i_gateway,
2890 };
2891
2892 nexthop_for_each_fib6_nh(res.f6i->nh,
2893 fib6_nh_find_match, &arg);
2894
2895 /* fib6_info uses a nexthop that does not have fib6_nh
2896 * using the dst->dev + gw. Should be impossible.
2897 */
43a4b60d
DA		 2898 if (!arg.match)
2899 goto out_unlock;
2d44234b
DA		 2900
2901 res.nh = arg.match;
2902 } else {
2903 res.nh = res.f6i->fib6_nh;
2904 }
2905
85bd05de 2906 nrt6 = ip6_rt_cache_alloc(&res, daddr, saddr);
45e4fd26
MKL		 2907 if (nrt6) {
2908 rt6_do_update_pmtu(nrt6, mtu);
5012f0a5 2909 if (rt6_insert_exception(nrt6, &res))
2b760fcf 2910 dst_release_immediate(&nrt6->dst);
45e4fd26 2911 }
43a4b60d 2912out_unlock:
a68886a6 2913 rcu_read_unlock();
1da177e4
LT		 2914 }
2915}
2916
45e4fd26 2917static void ip6_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
bd085ef6
HL		 2918 struct sk_buff *skb, u32 mtu,
2919 bool confirm_neigh)
45e4fd26 2920{
bd085ef6
HL		 2921 __ip6_rt_update_pmtu(dst, sk, skb ? ipv6_hdr(skb) : NULL, mtu,
2922 confirm_neigh);
45e4fd26
MKL		 2923}
2924
42ae66c8 2925void ip6_update_pmtu(struct sk_buff *skb, struct net *net, __be32 mtu,
e2d118a1 2926 int oif, u32 mark, kuid_t uid)
81aded24
DM		 2927{
2928 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
2929 struct dst_entry *dst;
dc92095d
MŻ		 2930 struct flowi6 fl6 = {
2931 .flowi6_oif = oif,
2932 .flowi6_mark = mark ? mark : IP6_REPLY_MARK(net, skb->mark),
2933 .daddr = iph->daddr,
2934 .saddr = iph->saddr,
2935 .flowlabel = ip6_flowinfo(iph),
2936 .flowi6_uid = uid,
2937 };
81aded24
DM		 2938
2939 dst = ip6_route_output(net, NULL, &fl6);
2940 if (!dst->error)
bd085ef6 2941 __ip6_rt_update_pmtu(dst, NULL, iph, ntohl(mtu), true);
81aded24
DM		 2942 dst_release(dst);
2943}
2944EXPORT_SYMBOL_GPL(ip6_update_pmtu);
2945
2946void ip6_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, __be32 mtu)
2947{
7ddacfa5 2948 int oif = sk->sk_bound_dev_if;
33c162a9
MKL		 2949 struct dst_entry *dst;
2950
7ddacfa5
DA		 2951 if (!oif && skb->dev)
2952 oif = l3mdev_master_ifindex(skb->dev);
2953
2954 ip6_update_pmtu(skb, sock_net(sk), mtu, oif, sk->sk_mark, sk->sk_uid);
33c162a9
MKL		 2955
2956 dst = __sk_dst_get(sk);
2957 if (!dst || !dst->obsolete ||
2958 dst->ops->check(dst, inet6_sk(sk)->dst_cookie))
2959 return;
2960
2961 bh_lock_sock(sk);
2962 if (!sock_owned_by_user(sk) && !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
2963 ip6_datagram_dst_update(sk, false);
2964 bh_unlock_sock(sk);
81aded24
DM		 2965}
2966EXPORT_SYMBOL_GPL(ip6_sk_update_pmtu);
2967
7d6850f7
AK		 2968void ip6_sk_dst_store_flow(struct sock *sk, struct dst_entry *dst,
2969 const struct flowi6 *fl6)
2970{
2971#ifdef CONFIG_IPV6_SUBTREES
2972 struct ipv6_pinfo *np = inet6_sk(sk);
2973#endif
2974
2975 ip6_dst_store(sk, dst,
2976 ipv6_addr_equal(&fl6->daddr, &sk->sk_v6_daddr) ?
2977 &sk->sk_v6_daddr : NULL,
2978#ifdef CONFIG_IPV6_SUBTREES
2979 ipv6_addr_equal(&fl6->saddr, &np->saddr) ?
2980 &np->saddr :
2981#endif
2982 NULL);
2983}
2984
9b6b35ab 2985static bool ip6_redirect_nh_match(const struct fib6_result *res,
0b34eb00
DA		 2986 struct flowi6 *fl6,
2987 const struct in6_addr *gw,
2988 struct rt6_info **ret)
2989{
9b6b35ab
DA		 2990 const struct fib6_nh *nh = res->nh;
2991
0b34eb00
DA		 2992 if (nh->fib_nh_flags & RTNH_F_DEAD || !nh->fib_nh_gw_family ||
2993 fl6->flowi6_oif != nh->fib_nh_dev->ifindex)
2994 return false;
2995
2996 /* rt_cache's gateway might be different from its 'parent'
2997 * in the case of an ip redirect.
2998 * So we keep searching in the exception table if the gateway
2999 * is different.
3000 */
3001 if (!ipv6_addr_equal(gw, &nh->fib_nh_gw6)) {
3002 struct rt6_info *rt_cache;
3003
9b6b35ab 3004 rt_cache = rt6_find_cached_rt(res, &fl6->daddr, &fl6->saddr);
0b34eb00
DA		 3005 if (rt_cache &&
3006 ipv6_addr_equal(gw, &rt_cache->rt6i_gateway)) {
3007 *ret = rt_cache;
3008 return true;
3009 }
3010 return false;
3011 }
3012 return true;
3013}
3014
c55c8988
DA		 3015struct fib6_nh_rd_arg {
3016 struct fib6_result *res;
3017 struct flowi6 *fl6;
3018 const struct in6_addr *gw;
3019 struct rt6_info **ret;
3020};
3021
3022static int fib6_nh_redirect_match(struct fib6_nh *nh, void *_arg)
3023{
3024 struct fib6_nh_rd_arg *arg = _arg;
3025
3026 arg->res->nh = nh;
3027 return ip6_redirect_nh_match(arg->res, arg->fl6, arg->gw, arg->ret);
3028}
3029
b55b76b2
DJ		 3030/* Handle redirects */
3031struct ip6rd_flowi {
3032 struct flowi6 fl6;
3033 struct in6_addr gateway;
3034};
3035
55cced4f 3036INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net,
b55b76b2
DJ		 3037 struct fib6_table *table,
3038 struct flowi6 *fl6,
b75cc8f9 3039 const struct sk_buff *skb,
b55b76b2
DJ		 3040 int flags)
3041{
3042 struct ip6rd_flowi *rdfl = (struct ip6rd_flowi *)fl6;
0b34eb00 3043 struct rt6_info *ret = NULL;
9b6b35ab 3044 struct fib6_result res = {};
c55c8988
DA		 3045 struct fib6_nh_rd_arg arg = {
3046 .res = &res,
3047 .fl6 = fl6,
3048 .gw = &rdfl->gateway,
3049 .ret = &ret
3050 };
8d1c802b 3051 struct fib6_info *rt;
b55b76b2
DJ		 3052 struct fib6_node *fn;
3053
3054 /* Get the "current" route for this destination and
67c408cf 3055 * check if the redirect has come from appropriate router.
b55b76b2
DJ		 3056 *
3057 * RFC 4861 specifies that redirects should only be
3058 * accepted if they come from the nexthop to the target.
3059 * Due to the way the routes are chosen, this notion
3060 * is a bit fuzzy and one might need to check all possible
3061 * routes.
3062 */
3063
66f5d6ce 3064 rcu_read_lock();
6454743b 3065 fn = fib6_node_lookup(&table->tb6_root, &fl6->daddr, &fl6->saddr);
b55b76b2 3066restart:
66f5d6ce 3067 for_each_fib6_node_rt_rcu(fn) {
9b6b35ab 3068 res.f6i = rt;
14895687 3069 if (fib6_check_expired(rt))
b55b76b2 3070 continue;
93c2fb25 3071 if (rt->fib6_flags & RTF_REJECT)
b55b76b2 3072 break;
c55c8988
DA		 3073 if (unlikely(rt->nh)) {
3074 if (nexthop_is_blackhole(rt->nh))
3075 continue;
3076 /* on match, res->nh is filled in and potentially ret */
3077 if (nexthop_for_each_fib6_nh(rt->nh,
3078 fib6_nh_redirect_match,
3079 &arg))
3080 goto out;
3081 } else {
3082 res.nh = rt->fib6_nh;
3083 if (ip6_redirect_nh_match(&res, fl6, &rdfl->gateway,
3084 &ret))
3085 goto out;
3086 }
b55b76b2
DJ		 3087 }
3088
3089 if (!rt)
421842ed 3090 rt = net->ipv6.fib6_null_entry;
93c2fb25 3091 else if (rt->fib6_flags & RTF_REJECT) {
23fb93a4 3092 ret = net->ipv6.ip6_null_entry;
b0a1ba59
MKL		 3093 goto out;
3094 }
3095
421842ed 3096 if (rt == net->ipv6.fib6_null_entry) {
a3c00e46
MKL		 3097 fn = fib6_backtrack(fn, &fl6->saddr);
3098 if (fn)
3099 goto restart;
b55b76b2 3100 }
a3c00e46 3101
9b6b35ab 3102 res.f6i = rt;
1cf844c7 3103 res.nh = rt->fib6_nh;
b0a1ba59 3104out:
7d21fec9 3105 if (ret) {
10585b43 3106 ip6_hold_safe(net, &ret);
7d21fec9
DA		 3107 } else {
3108 res.fib6_flags = res.f6i->fib6_flags;
3109 res.fib6_type = res.f6i->fib6_type;
9b6b35ab 3110 ret = ip6_create_rt_rcu(&res);
7d21fec9 3111 }
b55b76b2 3112
66f5d6ce 3113 rcu_read_unlock();
b55b76b2 3114
8ff2e5b2 3115 trace_fib6_table_lookup(net, &res, table, fl6);
23fb93a4 3116 return ret;
b55b76b2
DJ		 3117};
3118
3119static struct dst_entry *ip6_route_redirect(struct net *net,
b75cc8f9
DA		 3120 const struct flowi6 *fl6,
3121 const struct sk_buff *skb,
3122 const struct in6_addr *gateway)
b55b76b2
DJ		 3123{
3124 int flags = RT6_LOOKUP_F_HAS_SADDR;
3125 struct ip6rd_flowi rdfl;
3126
3127 rdfl.fl6 = *fl6;
3128 rdfl.gateway = *gateway;
3129
b75cc8f9 3130 return fib6_rule_lookup(net, &rdfl.fl6, skb,
b55b76b2
DJ		 3131 flags, __ip6_route_redirect);
3132}
3133
e2d118a1
LC		 3134void ip6_redirect(struct sk_buff *skb, struct net *net, int oif, u32 mark,
3135 kuid_t uid)
3a5ad2ee
DM		 3136{
3137 const struct ipv6hdr *iph = (struct ipv6hdr *) skb->data;
3138 struct dst_entry *dst;
1f7f10ac
MŻ		 3139 struct flowi6 fl6 = {
3140 .flowi6_iif = LOOPBACK_IFINDEX,
3141 .flowi6_oif = oif,
3142 .flowi6_mark = mark,
3143 .daddr = iph->daddr,
3144 .saddr = iph->saddr,
3145 .flowlabel = ip6_flowinfo(iph),
3146 .flowi6_uid = uid,
3147 };
3a5ad2ee 3148
b75cc8f9 3149 dst = ip6_route_redirect(net, &fl6, skb, &ipv6_hdr(skb)->saddr);
b55b76b2 3150 rt6_do_redirect(dst, NULL, skb);
3a5ad2ee
DM		 3151 dst_release(dst);
3152}
3153EXPORT_SYMBOL_GPL(ip6_redirect);
3154
d456336d 3155void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif)
c92a59ec
DJ		 3156{
3157 const struct ipv6hdr *iph = ipv6_hdr(skb);
3158 const struct rd_msg *msg = (struct rd_msg *)icmp6_hdr(skb);
3159 struct dst_entry *dst;
0b26fb17
MŻ		 3160 struct flowi6 fl6 = {
3161 .flowi6_iif = LOOPBACK_IFINDEX,
3162 .flowi6_oif = oif,
0b26fb17
MŻ		 3163 .daddr = msg->dest,
3164 .saddr = iph->daddr,
3165 .flowi6_uid = sock_net_uid(net, NULL),
3166 };
c92a59ec 3167
b75cc8f9 3168 dst = ip6_route_redirect(net, &fl6, skb, &iph->saddr);
b55b76b2 3169 rt6_do_redirect(dst, NULL, skb);
c92a59ec
DJ		 3170 dst_release(dst);
3171}
3172
3a5ad2ee
DM		 3173void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk)
3174{
e2d118a1
LC		 3175 ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, sk->sk_mark,
3176 sk->sk_uid);
3a5ad2ee
DM		 3177}
3178EXPORT_SYMBOL_GPL(ip6_sk_redirect);
3179
0dbaee3b 3180static unsigned int ip6_default_advmss(const struct dst_entry *dst)
1da177e4 3181{
0dbaee3b
DM		 3182 struct net_device *dev = dst->dev;
3183 unsigned int mtu = dst_mtu(dst);
3184 struct net *net = dev_net(dev);
3185
1da177e4
LT		 3186 mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
3187
5578689a
DL		 3188 if (mtu < net->ipv6.sysctl.ip6_rt_min_advmss)
3189 mtu = net->ipv6.sysctl.ip6_rt_min_advmss;
1da177e4
LT		 3190
3191 /*
1ab1457c
YH		 3192 * Maximal non-jumbo IPv6 payload is IPV6_MAXPLEN and
3193 * corresponding MSS is IPV6_MAXPLEN - tcp_header_size.
3194 * IPV6_MAXPLEN is also valid and means: "any MSS,
1da177e4
LT		 3195 * rely only on pmtu discovery"
3196 */
3197 if (mtu > IPV6_MAXPLEN - sizeof(struct tcphdr))
3198 mtu = IPV6_MAXPLEN;
3199 return mtu;
3200}
3201
f67fbeae 3202INDIRECT_CALLABLE_SCOPE unsigned int ip6_mtu(const struct dst_entry *dst)
d33e4553 3203{
427faee1 3204 return ip6_dst_mtu_maybe_forward(dst, false);
d33e4553 3205}
9c97921a 3206EXPORT_INDIRECT_CALLABLE(ip6_mtu);
d33e4553 3207
901731b8
DA		 3208/* MTU selection:
3209 * 1. mtu on route is locked - use it
3210 * 2. mtu from nexthop exception
3211 * 3. mtu from egress device
3212 *
3213 * based on ip6_dst_mtu_forward and exception logic of
3214 * rt6_find_cached_rt; called with rcu_read_lock
3215 */
b748f260
DA		 3216u32 ip6_mtu_from_fib6(const struct fib6_result *res,
3217 const struct in6_addr *daddr,
3218 const struct in6_addr *saddr)
901731b8 3219{
b748f260
DA		 3220 const struct fib6_nh *nh = res->nh;
3221 struct fib6_info *f6i = res->f6i;
901731b8 3222 struct inet6_dev *idev;
510e2ced 3223 struct rt6_info *rt;
901731b8
DA		 3224 u32 mtu = 0;
3225
3226 if (unlikely(fib6_metric_locked(f6i, RTAX_MTU))) {
3227 mtu = f6i->fib6_pmtu;
3228 if (mtu)
3229 goto out;
3230 }
3231
510e2ced
WW		 3232 rt = rt6_find_cached_rt(res, daddr, saddr);
3233 if (unlikely(rt)) {
3234 mtu = dst_metric_raw(&rt->dst, RTAX_MTU);
3235 } else {
b748f260 3236 struct net_device *dev = nh->fib_nh_dev;
901731b8
DA		 3237
3238 mtu = IPV6_MIN_MTU;
3239 idev = __in6_dev_get(dev);
3240 if (idev && idev->cnf.mtu6 > mtu)
3241 mtu = idev->cnf.mtu6;
3242 }
3243
3244 mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
3245out:
b748f260 3246 return mtu - lwtunnel_headroom(nh->fib_nh_lws, mtu);
901731b8
DA		 3247}
3248
3b00944c 3249struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
87a11578 3250 struct flowi6 *fl6)
1da177e4 3251{
87a11578 3252 struct dst_entry *dst;
1da177e4
LT		 3253 struct rt6_info *rt;
3254 struct inet6_dev *idev = in6_dev_get(dev);
c346dca1 3255 struct net *net = dev_net(dev);
1da177e4 3256
38308473 3257 if (unlikely(!idev))
122bdf67 3258 return ERR_PTR(-ENODEV);
1da177e4 3259
ad706862 3260 rt = ip6_dst_alloc(net, dev, 0);
38308473 3261 if (unlikely(!rt)) {
1da177e4 3262 in6_dev_put(idev);
87a11578 3263 dst = ERR_PTR(-ENOMEM);
1da177e4
LT		 3264 goto out;
3265 }
3266
588753f1 3267 rt->dst.input = ip6_input;
8e2ec639 3268 rt->dst.output = ip6_output;
550bab42 3269 rt->rt6i_gateway = fl6->daddr;
87a11578 3270 rt->rt6i_dst.addr = fl6->daddr;
8e2ec639
YZ		 3271 rt->rt6i_dst.plen = 128;
3272 rt->rt6i_idev = idev;
14edd87d 3273 dst_metric_set(&rt->dst, RTAX_HOPLIMIT, 0);
1da177e4 3274
4c981e28 3275 /* Add this dst into uncached_list so that rt6_disable_ip() can
587fea74
WW		 3276 * do proper release of the net_device
3277 */
3278 rt6_uncached_list_add(rt);
1da177e4 3279
87a11578
DM		 3280 dst = xfrm_lookup(net, &rt->dst, flowi6_to_flowi(fl6), NULL, 0);
3281
1da177e4 3282out:
87a11578 3283 return dst;
1da177e4
LT		 3284}
3285
af6d1034 3286static void ip6_dst_gc(struct dst_ops *ops)
1da177e4 3287{
86393e52 3288 struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
7019b78e 3289 int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
7019b78e
DL		 3290 int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
3291 int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
3292 unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
9cb7c013 3293 unsigned int val;
fc66f95c 3294 int entries;
7019b78e 3295
af6d1034 3296 if (time_after(rt_last_gc + rt_min_interval, jiffies))
1da177e4
LT		 3297 goto out;
3298
9cb7c013 3299 fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
fc66f95c
ED		 3300 entries = dst_entries_get_slow(ops);
3301 if (entries < ops->gc_thresh)
9cb7c013 3302 atomic_set(&net->ipv6.ip6_rt_gc_expire, rt_gc_timeout >> 1);
1da177e4 3303out:
9cb7c013
ED		 3304 val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
3305 atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
1da177e4
LT		 3306}
3307
b2c709cc
DA		 3308static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
3309 const struct in6_addr *gw_addr, u32 tbid,
3310 int flags, struct fib6_result *res)
8c14586f
DA		 3311{
3312 struct flowi6 fl6 = {
3313 .flowi6_oif = cfg->fc_ifindex,
3314 .daddr = *gw_addr,
3315 .saddr = cfg->fc_prefsrc,
3316 };
3317 struct fib6_table *table;
b2c709cc 3318 int err;
8c14586f 3319
f4797b33 3320 table = fib6_get_table(net, tbid);
8c14586f 3321 if (!table)
b2c709cc 3322 return -EINVAL;
8c14586f
DA		 3323
3324 if (!ipv6_addr_any(&cfg->fc_prefsrc))
3325 flags |= RT6_LOOKUP_F_HAS_SADDR;
3326
f4797b33 3327 flags |= RT6_LOOKUP_F_IGNORE_LINKSTATE;
8c14586f 3328
b2c709cc
DA		 3329 err = fib6_table_lookup(net, table, cfg->fc_ifindex, &fl6, res, flags);
3330 if (!err && res->f6i != net->ipv6.fib6_null_entry)
3331 fib6_select_path(net, res, &fl6, cfg->fc_ifindex,
3332 cfg->fc_ifindex != 0, NULL, flags);
8c14586f 3333
b2c709cc 3334 return err;
8c14586f
DA		 3335}
3336
fc1e64e1
DA		 3337static int ip6_route_check_nh_onlink(struct net *net,
3338 struct fib6_config *cfg,
9fbb704c 3339 const struct net_device *dev,
fc1e64e1
DA		 3340 struct netlink_ext_ack *extack)
3341{
b2c709cc 3342 u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
fc1e64e1 3343 const struct in6_addr *gw_addr = &cfg->fc_gateway;
b2c709cc 3344 struct fib6_result res = {};
fc1e64e1
DA		 3345 int err;
3346
b2c709cc
DA		 3347 err = ip6_nh_lookup_table(net, cfg, gw_addr, tbid, 0, &res);
3348 if (!err && !(res.fib6_flags & RTF_REJECT) &&
3349 /* ignore match if it is the default route */
3350 !ipv6_addr_any(&res.f6i->fib6_dst.addr) &&
3351 (res.fib6_type != RTN_UNICAST || dev != res.nh->fib_nh_dev)) {
3352 NL_SET_ERR_MSG(extack,
3353 "Nexthop has invalid gateway or device mismatch");
3354 err = -EINVAL;
fc1e64e1
DA		 3355 }
3356
3357 return err;
3358}
3359
1edce99f
DA		 3360static int ip6_route_check_nh(struct net *net,
3361 struct fib6_config *cfg,
3362 struct net_device **_dev,
3363 struct inet6_dev **idev)
3364{
3365 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3366 struct net_device *dev = _dev ? *_dev : NULL;
b2c709cc
DA		 3367 int flags = RT6_LOOKUP_F_IFACE;
3368 struct fib6_result res = {};
1edce99f
DA		 3369 int err = -EHOSTUNREACH;
3370
3371 if (cfg->fc_table) {
b2c709cc
DA		 3372 err = ip6_nh_lookup_table(net, cfg, gw_addr,
3373 cfg->fc_table, flags, &res);
3374 /* gw_addr can not require a gateway or resolve to a reject
3375 * route. If a device is given, it must match the result.
3376 */
3377 if (err || res.fib6_flags & RTF_REJECT ||
3378 res.nh->fib_nh_gw_family ||
3379 (dev && dev != res.nh->fib_nh_dev))
3380 err = -EHOSTUNREACH;
1edce99f
DA		 3381 }
3382
b2c709cc
DA		 3383 if (err < 0) {
3384 struct flowi6 fl6 = {
3385 .flowi6_oif = cfg->fc_ifindex,
3386 .daddr = *gw_addr,
3387 };
1edce99f 3388
b2c709cc
DA		 3389 err = fib6_lookup(net, cfg->fc_ifindex, &fl6, &res, flags);
3390 if (err || res.fib6_flags & RTF_REJECT ||
3391 res.nh->fib_nh_gw_family)
3392 err = -EHOSTUNREACH;
3393
3394 if (err)
3395 return err;
3396
3397 fib6_select_path(net, &res, &fl6, cfg->fc_ifindex,
3398 cfg->fc_ifindex != 0, NULL, flags);
3399 }
1edce99f 3400
b2c709cc 3401 err = 0;
1edce99f 3402 if (dev) {
b2c709cc
DA		 3403 if (dev != res.nh->fib_nh_dev)
3404 err = -EHOSTUNREACH;
1edce99f 3405 } else {
b2c709cc 3406 *_dev = dev = res.nh->fib_nh_dev;
1edce99f 3407 dev_hold(dev);
b2c709cc 3408 *idev = in6_dev_get(dev);
1edce99f
DA		 3409 }
3410
1edce99f
DA		 3411 return err;
3412}
3413
9fbb704c
DA		 3414static int ip6_validate_gw(struct net *net, struct fib6_config *cfg,
3415 struct net_device **_dev, struct inet6_dev **idev,
3416 struct netlink_ext_ack *extack)
3417{
3418 const struct in6_addr *gw_addr = &cfg->fc_gateway;
3419 int gwa_type = ipv6_addr_type(gw_addr);
232378e8 3420 bool skip_dev = gwa_type & IPV6_ADDR_LINKLOCAL ? false : true;
9fbb704c 3421 const struct net_device *dev = *_dev;
232378e8 3422 bool need_addr_check = !dev;
9fbb704c
DA		 3423 int err = -EINVAL;
3424
3425 /* if gw_addr is local we will fail to detect this in case
3426 * address is still TENTATIVE (DAD in progress). rt6_lookup()
3427 * will return already-added prefix route via interface that
3428 * prefix route was assigned to, which might be non-loopback.
3429 */
232378e8
DA		 3430 if (dev &&
3431 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
3432 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
9fbb704c
DA		 3433 goto out;
3434 }
3435
3436 if (gwa_type != (IPV6_ADDR_LINKLOCAL | IPV6_ADDR_UNICAST)) {
3437 /* IPv6 strictly inhibits using not link-local
3438 * addresses as nexthop address.
3439 * Otherwise, router will not able to send redirects.
3440 * It is very good, but in some (rare!) circumstances
3441 * (SIT, PtP, NBMA NOARP links) it is handy to allow
3442 * some exceptions. --ANK
3443 * We allow IPv4-mapped nexthops to support RFC4798-type
3444 * addressing
3445 */
3446 if (!(gwa_type & (IPV6_ADDR_UNICAST | IPV6_ADDR_MAPPED))) {
3447 NL_SET_ERR_MSG(extack, "Invalid gateway address");
3448 goto out;
3449 }
3450
b2c709cc
DA		 3451 rcu_read_lock();
3452
9fbb704c
DA		 3453 if (cfg->fc_flags & RTNH_F_ONLINK)
3454 err = ip6_route_check_nh_onlink(net, cfg, dev, extack);
3455 else
3456 err = ip6_route_check_nh(net, cfg, _dev, idev);
3457
b2c709cc
DA		 3458 rcu_read_unlock();
3459
9fbb704c
DA		 3460 if (err)
3461 goto out;
3462 }
3463
3464 /* reload in case device was changed */
3465 dev = *_dev;
3466
3467 err = -EINVAL;
3468 if (!dev) {
3469 NL_SET_ERR_MSG(extack, "Egress device not specified");
3470 goto out;
3471 } else if (dev->flags & IFF_LOOPBACK) {
3472 NL_SET_ERR_MSG(extack,
3473 "Egress device can not be loopback device for this route");
3474 goto out;
3475 }
232378e8
DA		 3476
3477 /* if we did not check gw_addr above, do so now that the
3478 * egress device has been resolved.
3479 */
3480 if (need_addr_check &&
3481 ipv6_chk_addr_and_flags(net, gw_addr, dev, skip_dev, 0, 0)) {
3482 NL_SET_ERR_MSG(extack, "Gateway can not be a local address");
3483 goto out;
3484 }
3485
9fbb704c
DA		 3486 err = 0;
3487out:
3488 return err;
3489}
3490
83c44251
DA		 3491static bool fib6_is_reject(u32 flags, struct net_device *dev, int addr_type)
3492{
3493 if ((flags & RTF_REJECT) ||
3494 (dev && (dev->flags & IFF_LOOPBACK) &&
3495 !(addr_type & IPV6_ADDR_LOOPBACK) &&
aea23c32 3496 !(flags & (RTF_ANYCAST | RTF_LOCAL))))
83c44251
DA		 3497 return true;
3498
3499 return false;
3500}
3501
3502int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
3503 struct fib6_config *cfg, gfp_t gfp_flags,
3504 struct netlink_ext_ack *extack)
3505{
3506 struct net_device *dev = NULL;
3507 struct inet6_dev *idev = NULL;
3508 int addr_type;
3509 int err;
3510
f1741730 3511 fib6_nh->fib_nh_family = AF_INET6;
1bef4c22
ED		 3512#ifdef CONFIG_IPV6_ROUTER_PREF
3513 fib6_nh->last_probe = jiffies;
3514#endif
38428d68
RP		 3515 if (cfg->fc_is_fdb) {
3516 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
3517 fib6_nh->fib_nh_gw_family = AF_INET6;
3518 return 0;
3519 }
f1741730 3520
83c44251
DA		 3521 err = -ENODEV;
3522 if (cfg->fc_ifindex) {
3523 dev = dev_get_by_index(net, cfg->fc_ifindex);
3524 if (!dev)
3525 goto out;
3526 idev = in6_dev_get(dev);
3527 if (!idev)
3528 goto out;
3529 }
3530
3531 if (cfg->fc_flags & RTNH_F_ONLINK) {
3532 if (!dev) {
3533 NL_SET_ERR_MSG(extack,
3534 "Nexthop device required for onlink");
3535 goto out;
3536 }
3537
3538 if (!(dev->flags & IFF_UP)) {
3539 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3540 err = -ENETDOWN;
3541 goto out;
3542 }
3543
ad1601ae 3544 fib6_nh->fib_nh_flags |= RTNH_F_ONLINK;
83c44251
DA		 3545 }
3546
ad1601ae 3547 fib6_nh->fib_nh_weight = 1;
83c44251
DA		 3548
3549 /* We cannot add true routes via loopback here,
3550 * they would result in kernel looping; promote them to reject routes
3551 */
3552 addr_type = ipv6_addr_type(&cfg->fc_dst);
3553 if (fib6_is_reject(cfg->fc_flags, dev, addr_type)) {
3554 /* hold loopback dev/idev if we haven't done so. */
3555 if (dev != net->loopback_dev) {
3556 if (dev) {
3557 dev_put(dev);
3558 in6_dev_put(idev);
3559 }
3560 dev = net->loopback_dev;
3561 dev_hold(dev);
3562 idev = in6_dev_get(dev);
3563 if (!idev) {
3564 err = -ENODEV;
3565 goto out;
3566 }
3567 }
7dd73168 3568 goto pcpu_alloc;
83c44251
DA		 3569 }
3570
3571 if (cfg->fc_flags & RTF_GATEWAY) {
3572 err = ip6_validate_gw(net, cfg, &dev, &idev, extack);
3573 if (err)
3574 goto out;
3575
ad1601ae 3576 fib6_nh->fib_nh_gw6 = cfg->fc_gateway;
bdf00467 3577 fib6_nh->fib_nh_gw_family = AF_INET6;
83c44251
DA		 3578 }
3579
3580 err = -ENODEV;
3581 if (!dev)
3582 goto out;
3583
3584 if (idev->cnf.disable_ipv6) {
3585 NL_SET_ERR_MSG(extack, "IPv6 is disabled on nexthop device");
3586 err = -EACCES;
3587 goto out;
3588 }
3589
3590 if (!(dev->flags & IFF_UP) && !cfg->fc_ignore_dev_down) {
3591 NL_SET_ERR_MSG(extack, "Nexthop device is not up");
3592 err = -ENETDOWN;
3593 goto out;
3594 }
3595
3596 if (!(cfg->fc_flags & (RTF_LOCAL | RTF_ANYCAST)) &&
3597 !netif_carrier_ok(dev))
ad1601ae 3598 fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
83c44251 3599
faee6769 3600 err = fib_nh_common_init(net, &fib6_nh->nh_common, cfg->fc_encap,
7dd73168
DA		 3601 cfg->fc_encap_type, cfg, gfp_flags, extack);
3602 if (err)
3603 goto out;
3604
3605pcpu_alloc:
f40b6ae2
DA		 3606 fib6_nh->rt6i_pcpu = alloc_percpu_gfp(struct rt6_info *, gfp_flags);
3607 if (!fib6_nh->rt6i_pcpu) {
3608 err = -ENOMEM;
3609 goto out;
3610 }
3611
ad1601ae 3612 fib6_nh->fib_nh_dev = dev;
e44b14eb
ED		 3613 netdev_tracker_alloc(dev, &fib6_nh->fib_nh_dev_tracker, gfp_flags);
3614
f1741730 3615 fib6_nh->fib_nh_oif = dev->ifindex;
83c44251
DA		 3616 err = 0;
3617out:
3618 if (idev)
3619 in6_dev_put(idev);
3620
3621 if (err) {
ad1601ae
DA		 3622 lwtstate_put(fib6_nh->fib_nh_lws);
3623 fib6_nh->fib_nh_lws = NULL;
1160dfa1 3624 dev_put(dev);
83c44251
DA		 3625 }
3626
3627 return err;
3628}
3629
dac7d0f2
DA		 3630void fib6_nh_release(struct fib6_nh *fib6_nh)
3631{
cc5c073a
DA		 3632 struct rt6_exception_bucket *bucket;
3633
3634 rcu_read_lock();
3635
3636 fib6_nh_flush_exceptions(fib6_nh, NULL);
3637 bucket = fib6_nh_get_excptn_bucket(fib6_nh, NULL);
3638 if (bucket) {
3639 rcu_assign_pointer(fib6_nh->rt6i_exception_bucket, NULL);
3640 kfree(bucket);
3641 }
3642
3643 rcu_read_unlock();
3644
61308050
NA		 3645 fib6_nh_release_dsts(fib6_nh);
3646 free_percpu(fib6_nh->rt6i_pcpu);
f40b6ae2 3647
979e276e 3648 fib_nh_common_release(&fib6_nh->nh_common);
dac7d0f2
DA		 3649}
3650
8837cbbf
NA		 3651void fib6_nh_release_dsts(struct fib6_nh *fib6_nh)
3652{
3653 int cpu;
3654
3655 if (!fib6_nh->rt6i_pcpu)
3656 return;
3657
3658 for_each_possible_cpu(cpu) {
3659 struct rt6_info *pcpu_rt, **ppcpu_rt;
3660
3661 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
3662 pcpu_rt = xchg(ppcpu_rt, NULL);
3663 if (pcpu_rt) {
3664 dst_dev_put(&pcpu_rt->dst);
3665 dst_release(&pcpu_rt->dst);
3666 }
3667 }
3668}
3669
8d1c802b 3670static struct fib6_info *ip6_route_info_create(struct fib6_config *cfg,
acb54e3c 3671 gfp_t gfp_flags,
333c4301 3672 struct netlink_ext_ack *extack)
1da177e4 3673{
5578689a 3674 struct net *net = cfg->fc_nlinfo.nl_net;
8d1c802b 3675 struct fib6_info *rt = NULL;
f88d8ea6 3676 struct nexthop *nh = NULL;
c71099ac 3677 struct fib6_table *table;
f88d8ea6 3678 struct fib6_nh *fib6_nh;
8c5b83f0 3679 int err = -EINVAL;
83c44251 3680 int addr_type;
1da177e4 3681
557c44be 3682 /* RTF_PCPU is an internal flag; can not be set by userspace */
d5d531cb
DA		 3683 if (cfg->fc_flags & RTF_PCPU) {
3684 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_PCPU");
557c44be 3685 goto out;
d5d531cb 3686 }
557c44be 3687
2ea2352e
WW		 3688 /* RTF_CACHE is an internal flag; can not be set by userspace */
3689 if (cfg->fc_flags & RTF_CACHE) {
3690 NL_SET_ERR_MSG(extack, "Userspace can not set RTF_CACHE");
3691 goto out;
3692 }
3693
e8478e80
DA		 3694 if (cfg->fc_type > RTN_MAX) {
3695 NL_SET_ERR_MSG(extack, "Invalid route type");
3696 goto out;
3697 }
3698
d5d531cb
DA		 3699 if (cfg->fc_dst_len > 128) {
3700 NL_SET_ERR_MSG(extack, "Invalid prefix length");
3701 goto out;
3702 }
3703 if (cfg->fc_src_len > 128) {
3704 NL_SET_ERR_MSG(extack, "Invalid source address length");
8c5b83f0 3705 goto out;
d5d531cb 3706 }
1da177e4 3707#ifndef CONFIG_IPV6_SUBTREES
d5d531cb
DA		 3708 if (cfg->fc_src_len) {
3709 NL_SET_ERR_MSG(extack,
3710 "Specifying source address requires IPV6_SUBTREES to be enabled");
8c5b83f0 3711 goto out;
d5d531cb 3712 }
1da177e4 3713#endif
5b98324e
DA		 3714 if (cfg->fc_nh_id) {
3715 nh = nexthop_find_by_id(net, cfg->fc_nh_id);
3716 if (!nh) {
3717 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
3718 goto out;
3719 }
3720 err = fib6_check_nexthop(nh, cfg, extack);
3721 if (err)
3722 goto out;
3723 }
fc1e64e1 3724
d71314b4 3725 err = -ENOBUFS;
38308473
DM		 3726 if (cfg->fc_nlinfo.nlh &&
3727 !(cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_CREATE)) {
d71314b4 3728 table = fib6_get_table(net, cfg->fc_table);
38308473 3729 if (!table) {
f3213831 3730 pr_warn("NLM_F_CREATE should be specified when creating new route\n");
d71314b4
MV		 3731 table = fib6_new_table(net, cfg->fc_table);
3732 }
3733 } else {
3734 table = fib6_new_table(net, cfg->fc_table);
3735 }
38308473
DM		 3736
3737 if (!table)
c71099ac 3738 goto out;
c71099ac 3739
93531c67 3740 err = -ENOMEM;
f88d8ea6 3741 rt = fib6_info_alloc(gfp_flags, !nh);
93531c67 3742 if (!rt)
1da177e4 3743 goto out;
93531c67 3744
d7e774f3
DA		 3745 rt->fib6_metrics = ip_fib_metrics_init(net, cfg->fc_mx, cfg->fc_mx_len,
3746 extack);
767a2217
DA		 3747 if (IS_ERR(rt->fib6_metrics)) {
3748 err = PTR_ERR(rt->fib6_metrics);
fda21d46
ED		 3749 /* Do not leave garbage there. */
3750 rt->fib6_metrics = (struct dst_metrics *)&dst_default_metrics;
8fb4792f 3751 goto out_free;
767a2217
DA		 3752 }
3753
93531c67
DA		 3754 if (cfg->fc_flags & RTF_ADDRCONF)
3755 rt->dst_nocount = true;
1da177e4 3756
1716a961 3757 if (cfg->fc_flags & RTF_EXPIRES)
14895687 3758 fib6_set_expires(rt, jiffies +
1716a961
G		 3759 clock_t_to_jiffies(cfg->fc_expires));
3760 else
14895687 3761 fib6_clean_expires(rt);
1da177e4 3762
86872cb5
TG		 3763 if (cfg->fc_protocol == RTPROT_UNSPEC)
3764 cfg->fc_protocol = RTPROT_BOOT;
93c2fb25 3765 rt->fib6_protocol = cfg->fc_protocol;
86872cb5 3766
83c44251
DA		 3767 rt->fib6_table = table;
3768 rt->fib6_metric = cfg->fc_metric;
c7036d97 3769 rt->fib6_type = cfg->fc_type ? : RTN_UNICAST;
2b2450ca 3770 rt->fib6_flags = cfg->fc_flags & ~RTF_GATEWAY;
19e42e45 3771
93c2fb25
DA		 3772 ipv6_addr_prefix(&rt->fib6_dst.addr, &cfg->fc_dst, cfg->fc_dst_len);
3773 rt->fib6_dst.plen = cfg->fc_dst_len;
e5fd387a 3774
1da177e4 3775#ifdef CONFIG_IPV6_SUBTREES
93c2fb25
DA		 3776 ipv6_addr_prefix(&rt->fib6_src.addr, &cfg->fc_src, cfg->fc_src_len);
3777 rt->fib6_src.plen = cfg->fc_src_len;
1da177e4 3778#endif
f88d8ea6 3779 if (nh) {
f88d8ea6 3780 if (rt->fib6_src.plen) {
4daa95af 3781 NL_SET_ERR_MSG(extack, "Nexthops can not be used with source routing");
821bbf79 3782 goto out_free;
f88d8ea6 3783 }
706ec919
XY		 3784 if (!nexthop_get(nh)) {
3785 NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
821bbf79 3786 goto out_free;
706ec919 3787 }
f88d8ea6
DA		 3788 rt->nh = nh;
3789 fib6_nh = nexthop_fib6_nh(rt->nh);
3790 } else {
3791 err = fib6_nh_init(net, rt->fib6_nh, cfg, gfp_flags, extack);
3792 if (err)
3793 goto out;
1da177e4 3794
f88d8ea6
DA		 3795 fib6_nh = rt->fib6_nh;
3796
3797 /* We cannot add true routes via loopback here, they would
3798 * result in kernel looping; promote them to reject routes
3799 */
3800 addr_type = ipv6_addr_type(&cfg->fc_dst);
3801 if (fib6_is_reject(cfg->fc_flags, rt->fib6_nh->fib_nh_dev,
3802 addr_type))
3803 rt->fib6_flags = RTF_REJECT | RTF_NONEXTHOP;
3804 }
955ec4cb 3805
c3968a85 3806 if (!ipv6_addr_any(&cfg->fc_prefsrc)) {
f88d8ea6 3807 struct net_device *dev = fib6_nh->fib_nh_dev;
83c44251 3808
c3968a85 3809 if (!ipv6_chk_addr(net, &cfg->fc_prefsrc, dev, 0)) {
d5d531cb 3810 NL_SET_ERR_MSG(extack, "Invalid source address");
c3968a85
DW		 3811 err = -EINVAL;
3812 goto out;
3813 }
93c2fb25
DA		 3814 rt->fib6_prefsrc.addr = cfg->fc_prefsrc;
3815 rt->fib6_prefsrc.plen = 128;
c3968a85 3816 } else
93c2fb25 3817 rt->fib6_prefsrc.plen = 0;
c3968a85 3818
8c5b83f0 3819 return rt;
6b9ea5a6 3820out:
93531c67 3821 fib6_info_release(rt);
8c5b83f0 3822 return ERR_PTR(err);
821bbf79
CL		 3823out_free:
3824 ip_fib_metrics_put(rt->fib6_metrics);
3825 kfree(rt);
3826 return ERR_PTR(err);
6b9ea5a6
RP		 3827}
3828
acb54e3c 3829int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags,
333c4301 3830 struct netlink_ext_ack *extack)
6b9ea5a6 3831{
8d1c802b 3832 struct fib6_info *rt;
6b9ea5a6
RP		 3833 int err;
3834
acb54e3c 3835 rt = ip6_route_info_create(cfg, gfp_flags, extack);
d4ead6b3
DA		 3836 if (IS_ERR(rt))
3837 return PTR_ERR(rt);
6b9ea5a6 3838
d4ead6b3 3839 err = __ip6_ins_rt(rt, &cfg->fc_nlinfo, extack);
93531c67 3840 fib6_info_release(rt);
6b9ea5a6 3841
1da177e4
LT		 3842 return err;
3843}
3844
8d1c802b 3845static int __ip6_del_rt(struct fib6_info *rt, struct nl_info *info)
1da177e4 3846{
afb1d4b5 3847 struct net *net = info->nl_net;
c71099ac 3848 struct fib6_table *table;
afb1d4b5 3849 int err;
1da177e4 3850
421842ed 3851 if (rt == net->ipv6.fib6_null_entry) {
6825a26c
G		 3852 err = -ENOENT;
3853 goto out;
3854 }
6c813a72 3855
93c2fb25 3856 table = rt->fib6_table;
66f5d6ce 3857 spin_lock_bh(&table->tb6_lock);
86872cb5 3858 err = fib6_del(rt, info);
66f5d6ce 3859 spin_unlock_bh(&table->tb6_lock);
1da177e4 3860
6825a26c 3861out:
93531c67 3862 fib6_info_release(rt);
1da177e4
LT		 3863 return err;
3864}
3865
11dd74b3 3866int ip6_del_rt(struct net *net, struct fib6_info *rt, bool skip_notify)
e0a1ad73 3867{
11dd74b3
RP		 3868 struct nl_info info = {
3869 .nl_net = net,
3870 .skip_notify = skip_notify
3871 };
afb1d4b5 3872
528c4ceb 3873 return __ip6_del_rt(rt, &info);
e0a1ad73
TG		 3874}
3875
8d1c802b 3876static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg)
0ae81335
DA		 3877{
3878 struct nl_info *info = &cfg->fc_nlinfo;
e3330039 3879 struct net *net = info->nl_net;
16a16cd3 3880 struct sk_buff *skb = NULL;
0ae81335 3881 struct fib6_table *table;
e3330039 3882 int err = -ENOENT;
0ae81335 3883
421842ed 3884 if (rt == net->ipv6.fib6_null_entry)
e3330039 3885 goto out_put;
93c2fb25 3886 table = rt->fib6_table;
66f5d6ce 3887 spin_lock_bh(&table->tb6_lock);
0ae81335 3888
93c2fb25 3889 if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) {
8d1c802b 3890 struct fib6_info *sibling, *next_sibling;
0284696b 3891 struct fib6_node *fn;
0ae81335 3892
16a16cd3
DA		 3893 /* prefer to send a single notification with all hops */
3894 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
3895 if (skb) {
3896 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
3897
d4ead6b3 3898 if (rt6_fill_node(net, skb, rt, NULL,
16a16cd3
DA		 3899 NULL, NULL, 0, RTM_DELROUTE,
3900 info->portid, seq, 0) < 0) {
3901 kfree_skb(skb);
3902 skb = NULL;
3903 } else
3904 info->skip_notify = 1;
3905 }
3906
0284696b
IS		 3907 /* 'rt' points to the first sibling route. If it is not the
3908 * leaf, then we do not need to send a notification. Otherwise,
3909 * we need to check if the last sibling has a next route or not
3910 * and emit a replace or delete notification, respectively.
3911 */
2881fd61 3912 info->skip_notify_kernel = 1;
0284696b
IS		 3913 fn = rcu_dereference_protected(rt->fib6_node,
3914 lockdep_is_held(&table->tb6_lock));
3915 if (rcu_access_pointer(fn->leaf) == rt) {
3916 struct fib6_info *last_sibling, *replace_rt;
3917
3918 last_sibling = list_last_entry(&rt->fib6_siblings,
3919 struct fib6_info,
3920 fib6_siblings);
3921 replace_rt = rcu_dereference_protected(
3922 last_sibling->fib6_next,
3923 lockdep_is_held(&table->tb6_lock));
3924 if (replace_rt)
3925 call_fib6_entry_notifiers_replace(net,
3926 replace_rt);
3927 else
3928 call_fib6_multipath_entry_notifiers(net,
caafb250 3929 FIB_EVENT_ENTRY_DEL,
0284696b
IS		 3930 rt, rt->fib6_nsiblings,
3931 NULL);
3932 }
0ae81335 3933 list_for_each_entry_safe(sibling, next_sibling,
93c2fb25
DA		 3934 &rt->fib6_siblings,
3935 fib6_siblings) {
0ae81335
DA		 3936 err = fib6_del(sibling, info);
3937 if (err)
e3330039 3938 goto out_unlock;
0ae81335
DA		 3939 }
3940 }
3941
3942 err = fib6_del(rt, info);
e3330039 3943out_unlock:
66f5d6ce 3944 spin_unlock_bh(&table->tb6_lock);
e3330039 3945out_put:
93531c67 3946 fib6_info_release(rt);
16a16cd3
DA		 3947
3948 if (skb) {
e3330039 3949 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
16a16cd3
DA		 3950 info->nlh, gfp_any());
3951 }
0ae81335
DA		 3952 return err;
3953}
3954
0fa6efc5 3955static int __ip6_del_cached_rt(struct rt6_info *rt, struct fib6_config *cfg)
23fb93a4
DA		 3956{
3957 int rc = -ESRCH;
3958
3959 if (cfg->fc_ifindex && rt->dst.dev->ifindex != cfg->fc_ifindex)
3960 goto out;
3961
3962 if (cfg->fc_flags & RTF_GATEWAY &&
3963 !ipv6_addr_equal(&cfg->fc_gateway, &rt->rt6i_gateway))
3964 goto out;
761f6026
XL		 3965
3966 rc = rt6_remove_exception_rt(rt);
23fb93a4
DA		 3967out:
3968 return rc;
3969}
3970
0fa6efc5
DA		 3971static int ip6_del_cached_rt(struct fib6_config *cfg, struct fib6_info *rt,
3972 struct fib6_nh *nh)
3973{
3974 struct fib6_result res = {
3975 .f6i = rt,
3976 .nh = nh,
3977 };
3978 struct rt6_info *rt_cache;
3979
3980 rt_cache = rt6_find_cached_rt(&res, &cfg->fc_dst, &cfg->fc_src);
3981 if (rt_cache)
3982 return __ip6_del_cached_rt(rt_cache, cfg);
3983
3984 return 0;
3985}
3986
5b98324e
DA		 3987struct fib6_nh_del_cached_rt_arg {
3988 struct fib6_config *cfg;
3989 struct fib6_info *f6i;
3990};
3991
3992static int fib6_nh_del_cached_rt(struct fib6_nh *nh, void *_arg)
3993{
3994 struct fib6_nh_del_cached_rt_arg *arg = _arg;
3995 int rc;
3996
3997 rc = ip6_del_cached_rt(arg->cfg, arg->f6i, nh);
3998 return rc != -ESRCH ? rc : 0;
3999}
4000
4001static int ip6_del_cached_rt_nh(struct fib6_config *cfg, struct fib6_info *f6i)
4002{
4003 struct fib6_nh_del_cached_rt_arg arg = {
4004 .cfg = cfg,
4005 .f6i = f6i
4006 };
4007
4008 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_del_cached_rt, &arg);
4009}
4010
333c4301
DA		 4011static int ip6_route_del(struct fib6_config *cfg,
4012 struct netlink_ext_ack *extack)
1da177e4 4013{
c71099ac 4014 struct fib6_table *table;
8d1c802b 4015 struct fib6_info *rt;
1da177e4 4016 struct fib6_node *fn;
1da177e4
LT		 4017 int err = -ESRCH;
4018
5578689a 4019 table = fib6_get_table(cfg->fc_nlinfo.nl_net, cfg->fc_table);
d5d531cb
DA		 4020 if (!table) {
4021 NL_SET_ERR_MSG(extack, "FIB table does not exist");
c71099ac 4022 return err;
d5d531cb 4023 }
c71099ac 4024
66f5d6ce 4025 rcu_read_lock();
1da177e4 4026
c71099ac 4027 fn = fib6_locate(&table->tb6_root,
86872cb5 4028 &cfg->fc_dst, cfg->fc_dst_len,
38fbeeee 4029 &cfg->fc_src, cfg->fc_src_len,
2b760fcf 4030 !(cfg->fc_flags & RTF_CACHE));
1ab1457c 4031
1da177e4 4032 if (fn) {
66f5d6ce 4033 for_each_fib6_node_rt_rcu(fn) {
ad1601ae
DA		 4034 struct fib6_nh *nh;
4035
3401bfb1
SB		 4036 if (rt->nh && cfg->fc_nh_id &&
4037 rt->nh->id != cfg->fc_nh_id)
5b98324e 4038 continue;
23fb93a4 4039
5b98324e
DA		 4040 if (cfg->fc_flags & RTF_CACHE) {
4041 int rc = 0;
4042
4043 if (rt->nh) {
4044 rc = ip6_del_cached_rt_nh(cfg, rt);
4045 } else if (cfg->fc_nh_id) {
4046 continue;
4047 } else {
4048 nh = rt->fib6_nh;
4049 rc = ip6_del_cached_rt(cfg, rt, nh);
4050 }
0fa6efc5
DA		 4051 if (rc != -ESRCH) {
4052 rcu_read_unlock();
4053 return rc;
23fb93a4
DA		 4054 }
4055 continue;
2b760fcf 4056 }
ad1601ae 4057
5b98324e
DA		 4058 if (cfg->fc_metric && cfg->fc_metric != rt->fib6_metric)
4059 continue;
4060 if (cfg->fc_protocol &&
4061 cfg->fc_protocol != rt->fib6_protocol)
4062 continue;
4063
4064 if (rt->nh) {
4065 if (!fib6_info_hold_safe(rt))
4066 continue;
4067 rcu_read_unlock();
4068
4069 return __ip6_del_rt(rt, &cfg->fc_nlinfo);
4070 }
4071 if (cfg->fc_nh_id)
4072 continue;
4073
4074 nh = rt->fib6_nh;
86872cb5 4075 if (cfg->fc_ifindex &&
ad1601ae
DA		 4076 (!nh->fib_nh_dev ||
4077 nh->fib_nh_dev->ifindex != cfg->fc_ifindex))
1da177e4 4078 continue;
86872cb5 4079 if (cfg->fc_flags & RTF_GATEWAY &&
ad1601ae 4080 !ipv6_addr_equal(&cfg->fc_gateway, &nh->fib_nh_gw6))
1da177e4 4081 continue;
e873e4b9
WW		 4082 if (!fib6_info_hold_safe(rt))
4083 continue;
66f5d6ce 4084 rcu_read_unlock();
1da177e4 4085
0ae81335
DA		 4086 /* if gateway was specified only delete the one hop */
4087 if (cfg->fc_flags & RTF_GATEWAY)
4088 return __ip6_del_rt(rt, &cfg->fc_nlinfo);
4089
4090 return __ip6_del_rt_siblings(rt, cfg);
1da177e4
LT		 4091 }
4092 }
66f5d6ce 4093 rcu_read_unlock();
1da177e4
LT		 4094
4095 return err;
4096}
4097
6700c270 4098static void rt6_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
a6279458 4099{
a6279458 4100 struct netevent_redirect netevent;
e8599ff4 4101 struct rt6_info *rt, *nrt = NULL;
85bd05de 4102 struct fib6_result res = {};
e8599ff4
DM		 4103 struct ndisc_options ndopts;
4104 struct inet6_dev *in6_dev;
4105 struct neighbour *neigh;
71bcdba0 4106 struct rd_msg *msg;
6e157b6a
DM		 4107 int optlen, on_link;
4108 u8 *lladdr;
e8599ff4 4109
29a3cad5 4110 optlen = skb_tail_pointer(skb) - skb_transport_header(skb);
71bcdba0 4111 optlen -= sizeof(*msg);
e8599ff4
DM		 4112
4113 if (optlen < 0) {
6e157b6a 4114 net_dbg_ratelimited("rt6_do_redirect: packet too short\n");
e8599ff4
DM		 4115 return;
4116 }
4117
71bcdba0 4118 msg = (struct rd_msg *)icmp6_hdr(skb);
e8599ff4 4119
71bcdba0 4120 if (ipv6_addr_is_multicast(&msg->dest)) {
6e157b6a 4121 net_dbg_ratelimited("rt6_do_redirect: destination address is multicast\n");
e8599ff4
DM		 4122 return;
4123 }
4124
6e157b6a 4125 on_link = 0;
71bcdba0 4126 if (ipv6_addr_equal(&msg->dest, &msg->target)) {
e8599ff4 4127 on_link = 1;
71bcdba0 4128 } else if (ipv6_addr_type(&msg->target) !=
e8599ff4 4129 (IPV6_ADDR_UNICAST|IPV6_ADDR_LINKLOCAL)) {
6e157b6a 4130 net_dbg_ratelimited("rt6_do_redirect: target address is not link-local unicast\n");
e8599ff4
DM		 4131 return;
4132 }
4133
4134 in6_dev = __in6_dev_get(skb->dev);
4135 if (!in6_dev)
4136 return;
4137 if (in6_dev->cnf.forwarding || !in6_dev->cnf.accept_redirects)
4138 return;
4139
4140 /* RFC2461 8.1:
4141 * The IP source address of the Redirect MUST be the same as the current
4142 * first-hop router for the specified ICMP Destination Address.
4143 */
4144
f997c55c 4145 if (!ndisc_parse_options(skb->dev, msg->opt, optlen, &ndopts)) {
e8599ff4
DM		 4146 net_dbg_ratelimited("rt6_redirect: invalid ND options\n");
4147 return;
4148 }
6e157b6a
DM		 4149
4150 lladdr = NULL;
e8599ff4
DM		 4151 if (ndopts.nd_opts_tgt_lladdr) {
4152 lladdr = ndisc_opt_addr_data(ndopts.nd_opts_tgt_lladdr,
4153 skb->dev);
4154 if (!lladdr) {
4155 net_dbg_ratelimited("rt6_redirect: invalid link-layer address length\n");
4156 return;
4157 }
4158 }
4159
6e157b6a 4160 rt = (struct rt6_info *) dst;
ec13ad1d 4161 if (rt->rt6i_flags & RTF_REJECT) {
6e157b6a 4162 net_dbg_ratelimited("rt6_redirect: source isn't a valid nexthop for redirect target\n");
e8599ff4 4163 return;
6e157b6a 4164 }
e8599ff4 4165
6e157b6a
DM		 4166 /* Redirect received -> path was valid.
4167 * Look, redirects are sent only in response to data packets,
4168 * so that this nexthop apparently is reachable. --ANK
4169 */
0dec879f 4170 dst_confirm_neigh(&rt->dst, &ipv6_hdr(skb)->saddr);
a6279458 4171
71bcdba0 4172 neigh = __neigh_lookup(&nd_tbl, &msg->target, skb->dev, 1);
6e157b6a
DM		 4173 if (!neigh)
4174 return;
a6279458 4175
1da177e4
LT		 4176 /*
4177 * We have finally decided to accept it.
4178 */
4179
f997c55c 4180 ndisc_update(skb->dev, neigh, lladdr, NUD_STALE,
1da177e4
LT		 4181 NEIGH_UPDATE_F_WEAK_OVERRIDE|
4182 NEIGH_UPDATE_F_OVERRIDE|
4183 (on_link ? 0 : (NEIGH_UPDATE_F_OVERRIDE_ISROUTER|
f997c55c
AA		 4184 NEIGH_UPDATE_F_ISROUTER)),
4185 NDISC_REDIRECT, &ndopts);
1da177e4 4186
4d85cd0c 4187 rcu_read_lock();
85bd05de 4188 res.f6i = rcu_dereference(rt->from);
ff24e498 4189 if (!res.f6i)
886b7a50 4190 goto out;
8a14e46f 4191
49d5b8ef
DA		 4192 if (res.f6i->nh) {
4193 struct fib6_nh_match_arg arg = {
4194 .dev = dst->dev,
4195 .gw = &rt->rt6i_gateway,
4196 };
4197
4198 nexthop_for_each_fib6_nh(res.f6i->nh,
4199 fib6_nh_find_match, &arg);
4200
4201 /* fib6_info uses a nexthop that does not have fib6_nh
4202 * using the dst->dev. Should be impossible
4203 */
4204 if (!arg.match)
4205 goto out;
4206 res.nh = arg.match;
4207 } else {
4208 res.nh = res.f6i->fib6_nh;
4209 }
4210
7d21fec9
DA		 4211 res.fib6_flags = res.f6i->fib6_flags;
4212 res.fib6_type = res.f6i->fib6_type;
85bd05de 4213 nrt = ip6_rt_cache_alloc(&res, &msg->dest, NULL);
38308473 4214 if (!nrt)
1da177e4
LT		 4215 goto out;
4216
4217 nrt->rt6i_flags = RTF_GATEWAY|RTF_UP|RTF_DYNAMIC|RTF_CACHE;
4218 if (on_link)
4219 nrt->rt6i_flags &= ~RTF_GATEWAY;
4220
4e3fd7a0 4221 nrt->rt6i_gateway = *(struct in6_addr *)neigh->primary_key;
1da177e4 4222
886b7a50 4223 /* rt6_insert_exception() will take care of duplicated exceptions */
5012f0a5 4224 if (rt6_insert_exception(nrt, &res)) {
2b760fcf
WW		 4225 dst_release_immediate(&nrt->dst);
4226 goto out;
4227 }
1da177e4 4228
d8d1f30b
CG		 4229 netevent.old = &rt->dst;
4230 netevent.new = &nrt->dst;
71bcdba0 4231 netevent.daddr = &msg->dest;
60592833 4232 netevent.neigh = neigh;
8d71740c
TT		 4233 call_netevent_notifiers(NETEVENT_REDIRECT, &netevent);
4234
1da177e4 4235out:
886b7a50 4236 rcu_read_unlock();
e8599ff4 4237 neigh_release(neigh);
6e157b6a
DM		 4238}
4239
70ceb4f5 4240#ifdef CONFIG_IPV6_ROUTE_INFO
8d1c802b 4241static struct fib6_info *rt6_get_route_info(struct net *net,
b71d1d42 4242 const struct in6_addr *prefix, int prefixlen,
830218c1
DA		 4243 const struct in6_addr *gwaddr,
4244 struct net_device *dev)
70ceb4f5 4245{
830218c1
DA		 4246 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4247 int ifindex = dev->ifindex;
70ceb4f5 4248 struct fib6_node *fn;
8d1c802b 4249 struct fib6_info *rt = NULL;
c71099ac
TG		 4250 struct fib6_table *table;
4251
830218c1 4252 table = fib6_get_table(net, tb_id);
38308473 4253 if (!table)
c71099ac 4254 return NULL;
70ceb4f5 4255
66f5d6ce 4256 rcu_read_lock();
38fbeeee 4257 fn = fib6_locate(&table->tb6_root, prefix, prefixlen, NULL, 0, true);
70ceb4f5
YH		 4258 if (!fn)
4259 goto out;
4260
66f5d6ce 4261 for_each_fib6_node_rt_rcu(fn) {
f88d8ea6
DA		 4262 /* these routes do not use nexthops */
4263 if (rt->nh)
4264 continue;
1cf844c7 4265 if (rt->fib6_nh->fib_nh_dev->ifindex != ifindex)
70ceb4f5 4266 continue;
2b2450ca 4267 if (!(rt->fib6_flags & RTF_ROUTEINFO) ||
1cf844c7 4268 !rt->fib6_nh->fib_nh_gw_family)
70ceb4f5 4269 continue;
1cf844c7 4270 if (!ipv6_addr_equal(&rt->fib6_nh->fib_nh_gw6, gwaddr))
70ceb4f5 4271 continue;
e873e4b9
WW		 4272 if (!fib6_info_hold_safe(rt))
4273 continue;
70ceb4f5
YH		 4274 break;
4275 }
4276out:
66f5d6ce 4277 rcu_read_unlock();
70ceb4f5
YH		 4278 return rt;
4279}
4280
8d1c802b 4281static struct fib6_info *rt6_add_route_info(struct net *net,
b71d1d42 4282 const struct in6_addr *prefix, int prefixlen,
830218c1
DA		 4283 const struct in6_addr *gwaddr,
4284 struct net_device *dev,
95c96174 4285 unsigned int pref)
70ceb4f5 4286{
86872cb5 4287 struct fib6_config cfg = {
238fc7ea 4288 .fc_metric = IP6_RT_PRIO_USER,
830218c1 4289 .fc_ifindex = dev->ifindex,
86872cb5
TG		 4290 .fc_dst_len = prefixlen,
4291 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO |
4292 RTF_UP | RTF_PREF(pref),
b91d5329 4293 .fc_protocol = RTPROT_RA,
e8478e80 4294 .fc_type = RTN_UNICAST,
15e47304 4295 .fc_nlinfo.portid = 0,
efa2cea0
DL		 4296 .fc_nlinfo.nlh = NULL,
4297 .fc_nlinfo.nl_net = net,
86872cb5
TG		 4298 };
4299
91b2c9a0 4300 cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO;
4e3fd7a0
AD		 4301 cfg.fc_dst = *prefix;
4302 cfg.fc_gateway = *gwaddr;
70ceb4f5 4303
e317da96
YH		 4304 /* We should treat it as a default route if prefix length is 0. */
4305 if (!prefixlen)
86872cb5 4306 cfg.fc_flags |= RTF_DEFAULT;
70ceb4f5 4307
acb54e3c 4308 ip6_route_add(&cfg, GFP_ATOMIC, NULL);
70ceb4f5 4309
830218c1 4310 return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev);
70ceb4f5
YH		 4311}
4312#endif
4313
8d1c802b 4314struct fib6_info *rt6_get_dflt_router(struct net *net,
afb1d4b5
DA		 4315 const struct in6_addr *addr,
4316 struct net_device *dev)
1ab1457c 4317{
830218c1 4318 u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT;
8d1c802b 4319 struct fib6_info *rt;
c71099ac 4320 struct fib6_table *table;
1da177e4 4321
afb1d4b5 4322 table = fib6_get_table(net, tb_id);
38308473 4323 if (!table)
c71099ac 4324 return NULL;
1da177e4 4325
66f5d6ce
WW		 4326 rcu_read_lock();
4327 for_each_fib6_node_rt_rcu(&table->tb6_root) {
f88d8ea6 4328 struct fib6_nh *nh;
ad1601ae 4329
f88d8ea6
DA		 4330 /* RA routes do not use nexthops */
4331 if (rt->nh)
4332 continue;
4333
4334 nh = rt->fib6_nh;
ad1601ae 4335 if (dev == nh->fib_nh_dev &&
93c2fb25 4336 ((rt->fib6_flags & (RTF_ADDRCONF | RTF_DEFAULT)) == (RTF_ADDRCONF | RTF_DEFAULT)) &&
ad1601ae 4337 ipv6_addr_equal(&nh->fib_nh_gw6, addr))
1da177e4
LT		 4338 break;
4339 }
e873e4b9
WW		 4340 if (rt && !fib6_info_hold_safe(rt))
4341 rt = NULL;
66f5d6ce 4342 rcu_read_unlock();
1da177e4
LT		 4343 return rt;
4344}
4345
8d1c802b 4346struct fib6_info *rt6_add_dflt_router(struct net *net,
afb1d4b5 4347 const struct in6_addr *gwaddr,
ebacaaa0 4348 struct net_device *dev,
6b2e04bc
PC		 4349 unsigned int pref,
4350 u32 defrtr_usr_metric)
1da177e4 4351{
86872cb5 4352 struct fib6_config cfg = {
ca254490 4353 .fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT,
6b2e04bc 4354 .fc_metric = defrtr_usr_metric,
86872cb5
TG		 4355 .fc_ifindex = dev->ifindex,
4356 .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_DEFAULT |
4357 RTF_UP | RTF_EXPIRES | RTF_PREF(pref),
b91d5329 4358 .fc_protocol = RTPROT_RA,
e8478e80 4359 .fc_type = RTN_UNICAST,
15e47304 4360 .fc_nlinfo.portid = 0,
5578689a 4361 .fc_nlinfo.nlh = NULL,
afb1d4b5 4362 .fc_nlinfo.nl_net = net,
86872cb5 4363 };
1da177e4 4364
4e3fd7a0 4365 cfg.fc_gateway = *gwaddr;
1da177e4 4366
acb54e3c 4367 if (!ip6_route_add(&cfg, GFP_ATOMIC, NULL)) {
830218c1
DA		 4368 struct fib6_table *table;
4369
4370 table = fib6_get_table(dev_net(dev), cfg.fc_table);
4371 if (table)
4372 table->flags |= RT6_TABLE_HAS_DFLT_ROUTER;
4373 }
1da177e4 4374
afb1d4b5 4375 return rt6_get_dflt_router(net, gwaddr, dev);
1da177e4
LT		 4376}
4377
afb1d4b5
DA		 4378static void __rt6_purge_dflt_routers(struct net *net,
4379 struct fib6_table *table)
1da177e4 4380{
8d1c802b 4381 struct fib6_info *rt;
1da177e4
LT		 4382
4383restart:
66f5d6ce
WW		 4384 rcu_read_lock();
4385 for_each_fib6_node_rt_rcu(&table->tb6_root) {
dcd1f572
DA		 4386 struct net_device *dev = fib6_info_nh_dev(rt);
4387 struct inet6_dev *idev = dev ? __in6_dev_get(dev) : NULL;
4388
93c2fb25 4389 if (rt->fib6_flags & (RTF_DEFAULT | RTF_ADDRCONF) &&
e873e4b9
WW		 4390 (!idev || idev->cnf.accept_ra != 2) &&
4391 fib6_info_hold_safe(rt)) {
93531c67 4392 rcu_read_unlock();
11dd74b3 4393 ip6_del_rt(net, rt, false);
1da177e4
LT		 4394 goto restart;
4395 }
4396 }
66f5d6ce 4397 rcu_read_unlock();
830218c1
DA		 4398
4399 table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER;
4400}
4401
4402void rt6_purge_dflt_routers(struct net *net)
4403{
4404 struct fib6_table *table;
4405 struct hlist_head *head;
4406 unsigned int h;
4407
4408 rcu_read_lock();
4409
4410 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
4411 head = &net->ipv6.fib_table_hash[h];
4412 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
4413 if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER)
afb1d4b5 4414 __rt6_purge_dflt_routers(net, table);
830218c1
DA		 4415 }
4416 }
4417
4418 rcu_read_unlock();
1da177e4
LT		 4419}
4420
5578689a
DL		 4421static void rtmsg_to_fib6_config(struct net *net,
4422 struct in6_rtmsg *rtmsg,
86872cb5
TG		 4423 struct fib6_config *cfg)
4424{
8823a3ac
MŻ		 4425 *cfg = (struct fib6_config){
4426 .fc_table = l3mdev_fib_table_by_index(net, rtmsg->rtmsg_ifindex) ?
4427 : RT6_TABLE_MAIN,
4428 .fc_ifindex = rtmsg->rtmsg_ifindex,
67f69513 4429 .fc_metric = rtmsg->rtmsg_metric ? : IP6_RT_PRIO_USER,
8823a3ac
MŻ		 4430 .fc_expires = rtmsg->rtmsg_info,
4431 .fc_dst_len = rtmsg->rtmsg_dst_len,
4432 .fc_src_len = rtmsg->rtmsg_src_len,
4433 .fc_flags = rtmsg->rtmsg_flags,
4434 .fc_type = rtmsg->rtmsg_type,
4435
4436 .fc_nlinfo.nl_net = net,
4437
4438 .fc_dst = rtmsg->rtmsg_dst,
4439 .fc_src = rtmsg->rtmsg_src,
4440 .fc_gateway = rtmsg->rtmsg_gateway,
4441 };
86872cb5
TG		 4442}
4443
7c1552da 4444int ipv6_route_ioctl(struct net *net, unsigned int cmd, struct in6_rtmsg *rtmsg)
1da177e4 4445{
86872cb5 4446 struct fib6_config cfg;
1da177e4
LT		 4447 int err;
4448
7c1552da
CH		 4449 if (cmd != SIOCADDRT && cmd != SIOCDELRT)
4450 return -EINVAL;
4451 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
4452 return -EPERM;
86872cb5 4453
7c1552da 4454 rtmsg_to_fib6_config(net, rtmsg, &cfg);
1da177e4 4455
7c1552da
CH		 4456 rtnl_lock();
4457 switch (cmd) {
4458 case SIOCADDRT:
4459 err = ip6_route_add(&cfg, GFP_KERNEL, NULL);
4460 break;
4461 case SIOCDELRT:
4462 err = ip6_route_del(&cfg, NULL);
4463 break;
3ff50b79 4464 }
7c1552da
CH		 4465 rtnl_unlock();
4466 return err;
1da177e4
LT		 4467}
4468
4469/*
4470 * Drop the packet on the floor
4471 */
4472
d5fdd6ba 4473static int ip6_pkt_drop(struct sk_buff *skb, u8 code, int ipstats_mib_noroutes)
1da177e4 4474{
adf30907 4475 struct dst_entry *dst = skb_dst(skb);
1d3fd8a1
SS		 4476 struct net *net = dev_net(dst->dev);
4477 struct inet6_dev *idev;
3ae42cc8 4478 SKB_DR(reason);
1d3fd8a1
SS		 4479 int type;
4480
1158f79f 4481 if (netif_is_l3_master(skb->dev) ||
1d3fd8a1
SS		 4482 dst->dev == net->loopback_dev)
4483 idev = __in6_dev_get_safely(dev_get_by_index_rcu(net, IP6CB(skb)->iif));
4484 else
4485 idev = ip6_dst_idev(dst);
4486
612f09e8
YH		 4487 switch (ipstats_mib_noroutes) {
4488 case IPSTATS_MIB_INNOROUTES:
0660e03f 4489 type = ipv6_addr_type(&ipv6_hdr(skb)->daddr);
45bb0060 4490 if (type == IPV6_ADDR_ANY) {
3ae42cc8 4491 SKB_DR_SET(reason, IP_INADDRERRORS);
1d3fd8a1 4492 IP6_INC_STATS(net, idev, IPSTATS_MIB_INADDRERRORS);
612f09e8
YH		 4493 break;
4494 }
3ae42cc8 4495 SKB_DR_SET(reason, IP_INNOROUTES);
a8eceea8 4496 fallthrough;
612f09e8 4497 case IPSTATS_MIB_OUTNOROUTES:
3ae42cc8 4498 SKB_DR_OR(reason, IP_OUTNOROUTES);
1d3fd8a1 4499 IP6_INC_STATS(net, idev, ipstats_mib_noroutes);
612f09e8
YH		 4500 break;
4501 }
1d3fd8a1
SS		 4502
4503 /* Start over by dropping the dst for l3mdev case */
4504 if (netif_is_l3_master(skb->dev))
4505 skb_dst_drop(skb);
4506
3ffe533c 4507 icmpv6_send(skb, ICMPV6_DEST_UNREACH, code, 0);
3ae42cc8 4508 kfree_skb_reason(skb, reason);
1da177e4
LT		 4509 return 0;
4510}
4511
9ce8ade0
TG		 4512static int ip6_pkt_discard(struct sk_buff *skb)
4513{
612f09e8 4514 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_INNOROUTES);
9ce8ade0
TG		 4515}
4516
ede2059d 4517static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb)
1da177e4 4518{
adf30907 4519 skb->dev = skb_dst(skb)->dev;
612f09e8 4520 return ip6_pkt_drop(skb, ICMPV6_NOROUTE, IPSTATS_MIB_OUTNOROUTES);
1da177e4
LT		 4521}
4522
9ce8ade0
TG		 4523static int ip6_pkt_prohibit(struct sk_buff *skb)
4524{
612f09e8 4525 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_INNOROUTES);
9ce8ade0
TG		 4526}
4527
ede2059d 4528static int ip6_pkt_prohibit_out(struct net *net, struct sock *sk, struct sk_buff *skb)
9ce8ade0 4529{
adf30907 4530 skb->dev = skb_dst(skb)->dev;
612f09e8 4531 return ip6_pkt_drop(skb, ICMPV6_ADM_PROHIBITED, IPSTATS_MIB_OUTNOROUTES);
9ce8ade0
TG		 4532}
4533
1da177e4
LT		 4534/*
4535 * Allocate a dst for local (unicast / anycast) address.
4536 */
4537
360a9887
DA		 4538struct fib6_info *addrconf_f6i_alloc(struct net *net,
4539 struct inet6_dev *idev,
4540 const struct in6_addr *addr,
4541 bool anycast, gfp_t gfp_flags)
1da177e4 4542{
c7a1ce39
DA		 4543 struct fib6_config cfg = {
4544 .fc_table = l3mdev_fib_table(idev->dev) ? : RT6_TABLE_LOCAL,
4545 .fc_ifindex = idev->dev->ifindex,
d55a2e37 4546 .fc_flags = RTF_UP | RTF_NONEXTHOP,
c7a1ce39
DA		 4547 .fc_dst = *addr,
4548 .fc_dst_len = 128,
4549 .fc_protocol = RTPROT_KERNEL,
4550 .fc_nlinfo.nl_net = net,
4551 .fc_ignore_dev_down = true,
4552 };
d55a2e37 4553 struct fib6_info *f6i;
1da177e4 4554
e8478e80 4555 if (anycast) {
c7a1ce39
DA		 4556 cfg.fc_type = RTN_ANYCAST;
4557 cfg.fc_flags |= RTF_ANYCAST;
e8478e80 4558 } else {
c7a1ce39
DA		 4559 cfg.fc_type = RTN_LOCAL;
4560 cfg.fc_flags |= RTF_LOCAL;
e8478e80 4561 }
1da177e4 4562
d55a2e37 4563 f6i = ip6_route_info_create(&cfg, gfp_flags, NULL);
3b0dc529 4564 if (!IS_ERR(f6i)) {
d55a2e37 4565 f6i->dst_nocount = true;
3b0dc529
ND		 4566
4567 if (!anycast &&
4568 (net->ipv6.devconf_all->disable_policy ||
4569 idev->cnf.disable_policy))
4570 f6i->dst_nopolicy = true;
4571 }
4572
d55a2e37 4573 return f6i;
1da177e4
LT		 4574}
4575
c3968a85
DW		 4576/* remove deleted ip from prefsrc entries */
4577struct arg_dev_net_ip {
4578 struct net_device *dev;
4579 struct net *net;
4580 struct in6_addr *addr;
4581};
4582
8d1c802b 4583static int fib6_remove_prefsrc(struct fib6_info *rt, void *arg)
c3968a85
DW		 4584{
4585 struct net_device *dev = ((struct arg_dev_net_ip *)arg)->dev;
4586 struct net *net = ((struct arg_dev_net_ip *)arg)->net;
4587 struct in6_addr *addr = ((struct arg_dev_net_ip *)arg)->addr;
4588
f88d8ea6
DA		 4589 if (!rt->nh &&
4590 ((void *)rt->fib6_nh->fib_nh_dev == dev || !dev) &&
421842ed 4591 rt != net->ipv6.fib6_null_entry &&
93c2fb25 4592 ipv6_addr_equal(addr, &rt->fib6_prefsrc.addr)) {
60006a48 4593 spin_lock_bh(&rt6_exception_lock);
c3968a85 4594 /* remove prefsrc entry */
93c2fb25 4595 rt->fib6_prefsrc.plen = 0;
60006a48 4596 spin_unlock_bh(&rt6_exception_lock);
c3968a85
DW		 4597 }
4598 return 0;
4599}
4600
4601void rt6_remove_prefsrc(struct inet6_ifaddr *ifp)
4602{
4603 struct net *net = dev_net(ifp->idev->dev);
4604 struct arg_dev_net_ip adni = {
4605 .dev = ifp->idev->dev,
4606 .net = net,
4607 .addr = &ifp->addr,
4608 };
0c3584d5 4609 fib6_clean_all(net, fib6_remove_prefsrc, &adni);
c3968a85
DW		 4610}
4611
2b2450ca 4612#define RTF_RA_ROUTER (RTF_ADDRCONF | RTF_DEFAULT)
be7a010d
DJ		 4613
4614/* Remove routers and update dst entries when gateway turn into host. */
8d1c802b 4615static int fib6_clean_tohost(struct fib6_info *rt, void *arg)
be7a010d
DJ		 4616{
4617 struct in6_addr *gateway = (struct in6_addr *)arg;
f88d8ea6
DA		 4618 struct fib6_nh *nh;
4619
4620 /* RA routes do not use nexthops */
4621 if (rt->nh)
4622 return 0;
be7a010d 4623
f88d8ea6 4624 nh = rt->fib6_nh;
93c2fb25 4625 if (((rt->fib6_flags & RTF_RA_ROUTER) == RTF_RA_ROUTER) &&
cc5c073a 4626 nh->fib_nh_gw_family && ipv6_addr_equal(gateway, &nh->fib_nh_gw6))
be7a010d 4627 return -1;
b16cb459
WW		 4628
4629 /* Further clean up cached routes in exception table.
4630 * This is needed because cached route may have a different
4631 * gateway than its 'parent' in the case of an ip redirect.
4632 */
cc5c073a 4633 fib6_nh_exceptions_clean_tohost(nh, gateway);
b16cb459 4634
be7a010d
DJ		 4635 return 0;
4636}
4637
4638void rt6_clean_tohost(struct net *net, struct in6_addr *gateway)
4639{
4640 fib6_clean_all(net, fib6_clean_tohost, gateway);
4641}
4642
2127d95a
IS		 4643struct arg_netdev_event {
4644 const struct net_device *dev;
4c981e28 4645 union {
ecc5663c 4646 unsigned char nh_flags;
4c981e28
IS		 4647 unsigned long event;
4648 };
2127d95a
IS		 4649};
4650
8d1c802b 4651static struct fib6_info *rt6_multipath_first_sibling(const struct fib6_info *rt)
d7dedee1 4652{
8d1c802b 4653 struct fib6_info *iter;
d7dedee1
IS		 4654 struct fib6_node *fn;
4655
93c2fb25
DA		 4656 fn = rcu_dereference_protected(rt->fib6_node,
4657 lockdep_is_held(&rt->fib6_table->tb6_lock));
d7dedee1 4658 iter = rcu_dereference_protected(fn->leaf,
93c2fb25 4659 lockdep_is_held(&rt->fib6_table->tb6_lock));
d7dedee1 4660 while (iter) {
93c2fb25 4661 if (iter->fib6_metric == rt->fib6_metric &&
33bd5ac5 4662 rt6_qualify_for_ecmp(iter))
d7dedee1 4663 return iter;
8fb11a9a 4664 iter = rcu_dereference_protected(iter->fib6_next,
93c2fb25 4665 lockdep_is_held(&rt->fib6_table->tb6_lock));
d7dedee1
IS		 4666 }
4667
4668 return NULL;
4669}
4670
f88d8ea6 4671/* only called for fib entries with builtin fib6_nh */
8d1c802b 4672static bool rt6_is_dead(const struct fib6_info *rt)
d7dedee1 4673{
1cf844c7
DA		 4674 if (rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD ||
4675 (rt->fib6_nh->fib_nh_flags & RTNH_F_LINKDOWN &&
4676 ip6_ignore_linkdown(rt->fib6_nh->fib_nh_dev)))
d7dedee1
IS		 4677 return true;
4678
4679 return false;
4680}
4681
8d1c802b 4682static int rt6_multipath_total_weight(const struct fib6_info *rt)
d7dedee1 4683{
8d1c802b 4684 struct fib6_info *iter;
d7dedee1
IS		 4685 int total = 0;
4686
4687 if (!rt6_is_dead(rt))
1cf844c7 4688 total += rt->fib6_nh->fib_nh_weight;
d7dedee1 4689
93c2fb25 4690 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
d7dedee1 4691 if (!rt6_is_dead(iter))
1cf844c7 4692 total += iter->fib6_nh->fib_nh_weight;
d7dedee1
IS		 4693 }
4694
4695 return total;
4696}
4697
8d1c802b 4698static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total)
d7dedee1
IS		 4699{
4700 int upper_bound = -1;
4701
4702 if (!rt6_is_dead(rt)) {
1cf844c7 4703 *weight += rt->fib6_nh->fib_nh_weight;
d7dedee1
IS		 4704 upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31,
4705 total) - 1;
4706 }
1cf844c7 4707 atomic_set(&rt->fib6_nh->fib_nh_upper_bound, upper_bound);
d7dedee1
IS		 4708}
4709
8d1c802b 4710static void rt6_multipath_upper_bound_set(struct fib6_info *rt, int total)
d7dedee1 4711{
8d1c802b 4712 struct fib6_info *iter;
d7dedee1
IS		 4713 int weight = 0;
4714
4715 rt6_upper_bound_set(rt, &weight, total);
4716
93c2fb25 4717 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
d7dedee1
IS		 4718 rt6_upper_bound_set(iter, &weight, total);
4719}
4720
8d1c802b 4721void rt6_multipath_rebalance(struct fib6_info *rt)
d7dedee1 4722{
8d1c802b 4723 struct fib6_info *first;
d7dedee1
IS		 4724 int total;
4725
4726 /* In case the entire multipath route was marked for flushing,
4727 * then there is no need to rebalance upon the removal of every
4728 * sibling route.
4729 */
93c2fb25 4730 if (!rt->fib6_nsiblings || rt->should_flush)
d7dedee1
IS		 4731 return;
4732
4733 /* During lookup routes are evaluated in order, so we need to
4734 * make sure upper bounds are assigned from the first sibling
4735 * onwards.
4736 */
4737 first = rt6_multipath_first_sibling(rt);
4738 if (WARN_ON_ONCE(!first))
4739 return;
4740
4741 total = rt6_multipath_total_weight(first);
4742 rt6_multipath_upper_bound_set(first, total);
4743}
4744
8d1c802b 4745static int fib6_ifup(struct fib6_info *rt, void *p_arg)
2127d95a
IS		 4746{
4747 const struct arg_netdev_event *arg = p_arg;
7aef6859 4748 struct net *net = dev_net(arg->dev);
2127d95a 4749
f88d8ea6 4750 if (rt != net->ipv6.fib6_null_entry && !rt->nh &&
1cf844c7
DA		 4751 rt->fib6_nh->fib_nh_dev == arg->dev) {
4752 rt->fib6_nh->fib_nh_flags &= ~arg->nh_flags;
7aef6859 4753 fib6_update_sernum_upto_root(net, rt);
d7dedee1 4754 rt6_multipath_rebalance(rt);
1de178ed 4755 }
2127d95a
IS		 4756
4757 return 0;
4758}
4759
ecc5663c 4760void rt6_sync_up(struct net_device *dev, unsigned char nh_flags)
2127d95a
IS		 4761{
4762 struct arg_netdev_event arg = {
4763 .dev = dev,
6802f3ad
IS		 4764 {
4765 .nh_flags = nh_flags,
4766 },
2127d95a
IS		 4767 };
4768
4769 if (nh_flags & RTNH_F_DEAD && netif_carrier_ok(dev))
4770 arg.nh_flags |= RTNH_F_LINKDOWN;
4771
4772 fib6_clean_all(dev_net(dev), fib6_ifup, &arg);
4773}
4774
f88d8ea6 4775/* only called for fib entries with inline fib6_nh */
8d1c802b 4776static bool rt6_multipath_uses_dev(const struct fib6_info *rt,
1de178ed
IS		 4777 const struct net_device *dev)
4778{
8d1c802b 4779 struct fib6_info *iter;
1de178ed 4780
1cf844c7 4781 if (rt->fib6_nh->fib_nh_dev == dev)
1de178ed 4782 return true;
93c2fb25 4783 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
1cf844c7 4784 if (iter->fib6_nh->fib_nh_dev == dev)
1de178ed
IS		 4785 return true;
4786
4787 return false;
4788}
4789
8d1c802b 4790static void rt6_multipath_flush(struct fib6_info *rt)
1de178ed 4791{
8d1c802b 4792 struct fib6_info *iter;
1de178ed
IS		 4793
4794 rt->should_flush = 1;
93c2fb25 4795 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
1de178ed
IS		 4796 iter->should_flush = 1;
4797}
4798
8d1c802b 4799static unsigned int rt6_multipath_dead_count(const struct fib6_info *rt,
1de178ed
IS		 4800 const struct net_device *down_dev)
4801{
8d1c802b 4802 struct fib6_info *iter;
1de178ed
IS		 4803 unsigned int dead = 0;
4804
1cf844c7
DA		 4805 if (rt->fib6_nh->fib_nh_dev == down_dev ||
4806 rt->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
1de178ed 4807 dead++;
93c2fb25 4808 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
1cf844c7
DA		 4809 if (iter->fib6_nh->fib_nh_dev == down_dev ||
4810 iter->fib6_nh->fib_nh_flags & RTNH_F_DEAD)
1de178ed
IS		 4811 dead++;
4812
4813 return dead;
4814}
4815
8d1c802b 4816static void rt6_multipath_nh_flags_set(struct fib6_info *rt,
1de178ed 4817 const struct net_device *dev,
ecc5663c 4818 unsigned char nh_flags)
1de178ed 4819{
8d1c802b 4820 struct fib6_info *iter;
1de178ed 4821
1cf844c7
DA		 4822 if (rt->fib6_nh->fib_nh_dev == dev)
4823 rt->fib6_nh->fib_nh_flags |= nh_flags;
93c2fb25 4824 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings)
1cf844c7
DA		 4825 if (iter->fib6_nh->fib_nh_dev == dev)
4826 iter->fib6_nh->fib_nh_flags |= nh_flags;
1de178ed
IS		 4827}
4828
a1a22c12 4829/* called with write lock held for table with rt */
8d1c802b 4830static int fib6_ifdown(struct fib6_info *rt, void *p_arg)
1da177e4 4831{
4c981e28
IS		 4832 const struct arg_netdev_event *arg = p_arg;
4833 const struct net_device *dev = arg->dev;
7aef6859 4834 struct net *net = dev_net(dev);
8ed67789 4835
f88d8ea6 4836 if (rt == net->ipv6.fib6_null_entry || rt->nh)
27c6fa73
IS		 4837 return 0;
4838
4839 switch (arg->event) {
4840 case NETDEV_UNREGISTER:
1cf844c7 4841 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
27c6fa73 4842 case NETDEV_DOWN:
1de178ed 4843 if (rt->should_flush)
27c6fa73 4844 return -1;
93c2fb25 4845 if (!rt->fib6_nsiblings)
1cf844c7 4846 return rt->fib6_nh->fib_nh_dev == dev ? -1 : 0;
1de178ed
IS		 4847 if (rt6_multipath_uses_dev(rt, dev)) {
4848 unsigned int count;
4849
4850 count = rt6_multipath_dead_count(rt, dev);
93c2fb25 4851 if (rt->fib6_nsiblings + 1 == count) {
1de178ed
IS		 4852 rt6_multipath_flush(rt);
4853 return -1;
4854 }
4855 rt6_multipath_nh_flags_set(rt, dev, RTNH_F_DEAD |
4856 RTNH_F_LINKDOWN);
7aef6859 4857 fib6_update_sernum(net, rt);
d7dedee1 4858 rt6_multipath_rebalance(rt);
1de178ed
IS		 4859 }
4860 return -2;
27c6fa73 4861 case NETDEV_CHANGE:
1cf844c7 4862 if (rt->fib6_nh->fib_nh_dev != dev ||
93c2fb25 4863 rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
27c6fa73 4864 break;
1cf844c7 4865 rt->fib6_nh->fib_nh_flags |= RTNH_F_LINKDOWN;
d7dedee1 4866 rt6_multipath_rebalance(rt);
27c6fa73 4867 break;
2b241361 4868 }
c159d30c 4869
1da177e4
LT		 4870 return 0;
4871}
4872
27c6fa73 4873void rt6_sync_down_dev(struct net_device *dev, unsigned long event)
1da177e4 4874{
4c981e28 4875 struct arg_netdev_event arg = {
8ed67789 4876 .dev = dev,
6802f3ad
IS		 4877 {
4878 .event = event,
4879 },
8ed67789 4880 };
7c6bb7d2 4881 struct net *net = dev_net(dev);
8ed67789 4882
7c6bb7d2
DA		 4883 if (net->ipv6.sysctl.skip_notify_on_dev_down)
4884 fib6_clean_all_skip_notify(net, fib6_ifdown, &arg);
4885 else
4886 fib6_clean_all(net, fib6_ifdown, &arg);
4c981e28
IS		 4887}
4888
4889void rt6_disable_ip(struct net_device *dev, unsigned long event)
4890{
4891 rt6_sync_down_dev(dev, event);
e5f80fcf 4892 rt6_uncached_list_flush_dev(dev);
4c981e28 4893 neigh_ifdown(&nd_tbl, dev);
1da177e4
LT		 4894}
4895
95c96174 4896struct rt6_mtu_change_arg {
1da177e4 4897 struct net_device *dev;
95c96174 4898 unsigned int mtu;
c0b220cf 4899 struct fib6_info *f6i;
1da177e4
LT		 4900};
4901
cc5c073a 4902static int fib6_nh_mtu_change(struct fib6_nh *nh, void *_arg)
c0b220cf
DA		 4903{
4904 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *)_arg;
cc5c073a 4905 struct fib6_info *f6i = arg->f6i;
c0b220cf
DA		 4906
4907 /* For administrative MTU increase, there is no way to discover
4908 * IPv6 PMTU increase, so PMTU increase should be updated here.
4909 * Since RFC 1981 doesn't include administrative MTU increase
4910 * update PMTU increase is a MUST. (i.e. jumbo frame)
4911 */
4912 if (nh->fib_nh_dev == arg->dev) {
4913 struct inet6_dev *idev = __in6_dev_get(arg->dev);
4914 u32 mtu = f6i->fib6_pmtu;
4915
4916 if (mtu >= arg->mtu ||
4917 (mtu < arg->mtu && mtu == idev->cnf.mtu6))
4918 fib6_metric_set(f6i, RTAX_MTU, arg->mtu);
4919
4920 spin_lock_bh(&rt6_exception_lock);
cc5c073a 4921 rt6_exceptions_update_pmtu(idev, nh, arg->mtu);
c0b220cf
DA		 4922 spin_unlock_bh(&rt6_exception_lock);
4923 }
4924
4925 return 0;
4926}
4927
4928static int rt6_mtu_change_route(struct fib6_info *f6i, void *p_arg)
1da177e4
LT		 4929{
4930 struct rt6_mtu_change_arg *arg = (struct rt6_mtu_change_arg *) p_arg;
4931 struct inet6_dev *idev;
4932
4933 /* In IPv6 pmtu discovery is not optional,
4934 so that RTAX_MTU lock cannot disable it.
4935 We still use this lock to block changes
4936 caused by addrconf/ndisc.
4937 */
4938
4939 idev = __in6_dev_get(arg->dev);
38308473 4940 if (!idev)
1da177e4
LT		 4941 return 0;
4942
c0b220cf
DA		 4943 if (fib6_metric_locked(f6i, RTAX_MTU))
4944 return 0;
d4ead6b3 4945
c0b220cf 4946 arg->f6i = f6i;
2d44234b
DA		 4947 if (f6i->nh) {
4948 /* fib6_nh_mtu_change only returns 0, so this is safe */
4949 return nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_mtu_change,
4950 arg);
4951 }
4952
1cf844c7 4953 return fib6_nh_mtu_change(f6i->fib6_nh, arg);
1da177e4
LT		 4954}
4955
95c96174 4956void rt6_mtu_change(struct net_device *dev, unsigned int mtu)
1da177e4 4957{
c71099ac
TG		 4958 struct rt6_mtu_change_arg arg = {
4959 .dev = dev,
4960 .mtu = mtu,
4961 };
1da177e4 4962
0c3584d5 4963 fib6_clean_all(dev_net(dev), rt6_mtu_change_route, &arg);
1da177e4
LT		 4964}
4965
ef7c79ed 4966static const struct nla_policy rtm_ipv6_policy[RTA_MAX+1] = {
75425657 4967 [RTA_UNSPEC] = { .strict_start_type = RTA_DPORT + 1 },
5176f91e 4968 [RTA_GATEWAY] = { .len = sizeof(struct in6_addr) },
aa8f8778 4969 [RTA_PREFSRC] = { .len = sizeof(struct in6_addr) },
86872cb5 4970 [RTA_OIF] = { .type = NLA_U32 },
ab364a6f 4971 [RTA_IIF] = { .type = NLA_U32 },
86872cb5
TG		 4972 [RTA_PRIORITY] = { .type = NLA_U32 },
4973 [RTA_METRICS] = { .type = NLA_NESTED },
51ebd318 4974 [RTA_MULTIPATH] = { .len = sizeof(struct rtnexthop) },
c78ba6d6 4975 [RTA_PREF] = { .type = NLA_U8 },
19e42e45
RP		 4976 [RTA_ENCAP_TYPE] = { .type = NLA_U16 },
4977 [RTA_ENCAP] = { .type = NLA_NESTED },
32bc201e 4978 [RTA_EXPIRES] = { .type = NLA_U32 },
622ec2c9 4979 [RTA_UID] = { .type = NLA_U32 },
3b45a410 4980 [RTA_MARK] = { .type = NLA_U32 },
aa8f8778 4981 [RTA_TABLE] = { .type = NLA_U32 },
eacb9384
RP		 4982 [RTA_IP_PROTO] = { .type = NLA_U8 },
4983 [RTA_SPORT] = { .type = NLA_U16 },
4984 [RTA_DPORT] = { .type = NLA_U16 },
5b98324e 4985 [RTA_NH_ID] = { .type = NLA_U32 },
86872cb5
TG		 4986};
4987
4988static int rtm_to_fib6_config(struct sk_buff *skb, struct nlmsghdr *nlh,
333c4301
DA		 4989 struct fib6_config *cfg,
4990 struct netlink_ext_ack *extack)
1da177e4 4991{
86872cb5
TG		 4992 struct rtmsg *rtm;
4993 struct nlattr *tb[RTA_MAX+1];
c78ba6d6 4994 unsigned int pref;
86872cb5 4995 int err;
1da177e4 4996
8cb08174
JB		 4997 err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
4998 rtm_ipv6_policy, extack);
86872cb5
TG		 4999 if (err < 0)
5000 goto errout;
1da177e4 5001
86872cb5
TG		 5002 err = -EINVAL;
5003 rtm = nlmsg_data(nlh);
86872cb5 5004
b9605161
GN		 5005 if (rtm->rtm_tos) {
5006 NL_SET_ERR_MSG(extack,
5007 "Invalid dsfield (tos): option not available for IPv6");
5008 goto errout;
5009 }
5010
84db8407
MŻ		 5011 *cfg = (struct fib6_config){
5012 .fc_table = rtm->rtm_table,
5013 .fc_dst_len = rtm->rtm_dst_len,
5014 .fc_src_len = rtm->rtm_src_len,
5015 .fc_flags = RTF_UP,
5016 .fc_protocol = rtm->rtm_protocol,
5017 .fc_type = rtm->rtm_type,
5018
5019 .fc_nlinfo.portid = NETLINK_CB(skb).portid,
5020 .fc_nlinfo.nlh = nlh,
5021 .fc_nlinfo.nl_net = sock_net(skb->sk),
5022 };
86872cb5 5023
ef2c7d7b
ND		 5024 if (rtm->rtm_type == RTN_UNREACHABLE ||
5025 rtm->rtm_type == RTN_BLACKHOLE ||
b4949ab2
ND		 5026 rtm->rtm_type == RTN_PROHIBIT ||
5027 rtm->rtm_type == RTN_THROW)
86872cb5
TG		 5028 cfg->fc_flags |= RTF_REJECT;
5029
ab79ad14
MŻ		 5030 if (rtm->rtm_type == RTN_LOCAL)
5031 cfg->fc_flags |= RTF_LOCAL;
5032
1f56a01f
MKL		 5033 if (rtm->rtm_flags & RTM_F_CLONED)
5034 cfg->fc_flags |= RTF_CACHE;
5035
fc1e64e1
DA		 5036 cfg->fc_flags |= (rtm->rtm_flags & RTNH_F_ONLINK);
5037
5b98324e
DA		 5038 if (tb[RTA_NH_ID]) {
5039 if (tb[RTA_GATEWAY] || tb[RTA_OIF] ||
5040 tb[RTA_MULTIPATH] || tb[RTA_ENCAP]) {
5041 NL_SET_ERR_MSG(extack,
5042 "Nexthop specification and nexthop id are mutually exclusive");
5043 goto errout;
5044 }
5045 cfg->fc_nh_id = nla_get_u32(tb[RTA_NH_ID]);
5046 }
5047
86872cb5 5048 if (tb[RTA_GATEWAY]) {
67b61f6c 5049 cfg->fc_gateway = nla_get_in6_addr(tb[RTA_GATEWAY]);
86872cb5 5050 cfg->fc_flags |= RTF_GATEWAY;
1da177e4 5051 }
e3818541
DA		 5052 if (tb[RTA_VIA]) {
5053 NL_SET_ERR_MSG(extack, "IPv6 does not support RTA_VIA attribute");
5054 goto errout;
5055 }
86872cb5
TG		 5056
5057 if (tb[RTA_DST]) {
5058 int plen = (rtm->rtm_dst_len + 7) >> 3;
5059
5060 if (nla_len(tb[RTA_DST]) < plen)
5061 goto errout;
5062
5063 nla_memcpy(&cfg->fc_dst, tb[RTA_DST], plen);
1da177e4 5064 }
86872cb5
TG		 5065
5066 if (tb[RTA_SRC]) {
5067 int plen = (rtm->rtm_src_len + 7) >> 3;
5068
5069 if (nla_len(tb[RTA_SRC]) < plen)
5070 goto errout;
5071
5072 nla_memcpy(&cfg->fc_src, tb[RTA_SRC], plen);
1da177e4 5073 }
86872cb5 5074
c3968a85 5075 if (tb[RTA_PREFSRC])
67b61f6c 5076 cfg->fc_prefsrc = nla_get_in6_addr(tb[RTA_PREFSRC]);
c3968a85 5077
86872cb5
TG		 5078 if (tb[RTA_OIF])
5079 cfg->fc_ifindex = nla_get_u32(tb[RTA_OIF]);
5080
5081 if (tb[RTA_PRIORITY])
5082 cfg->fc_metric = nla_get_u32(tb[RTA_PRIORITY]);
5083
5084 if (tb[RTA_METRICS]) {
5085 cfg->fc_mx = nla_data(tb[RTA_METRICS]);
5086 cfg->fc_mx_len = nla_len(tb[RTA_METRICS]);
1da177e4 5087 }
86872cb5
TG		 5088
5089 if (tb[RTA_TABLE])
5090 cfg->fc_table = nla_get_u32(tb[RTA_TABLE]);
5091
51ebd318
ND		 5092 if (tb[RTA_MULTIPATH]) {
5093 cfg->fc_mp = nla_data(tb[RTA_MULTIPATH]);
5094 cfg->fc_mp_len = nla_len(tb[RTA_MULTIPATH]);
9ed59592
DA		 5095
5096 err = lwtunnel_valid_encap_type_attr(cfg->fc_mp,
c255bd68 5097 cfg->fc_mp_len, extack);
9ed59592
DA		 5098 if (err < 0)
5099 goto errout;
51ebd318
ND		 5100 }
5101
c78ba6d6
LR		 5102 if (tb[RTA_PREF]) {
5103 pref = nla_get_u8(tb[RTA_PREF]);
5104 if (pref != ICMPV6_ROUTER_PREF_LOW &&
5105 pref != ICMPV6_ROUTER_PREF_HIGH)
5106 pref = ICMPV6_ROUTER_PREF_MEDIUM;
5107 cfg->fc_flags |= RTF_PREF(pref);
5108 }
5109
19e42e45
RP		 5110 if (tb[RTA_ENCAP])
5111 cfg->fc_encap = tb[RTA_ENCAP];
5112
9ed59592 5113 if (tb[RTA_ENCAP_TYPE]) {
19e42e45
RP		 5114 cfg->fc_encap_type = nla_get_u16(tb[RTA_ENCAP_TYPE]);
5115
c255bd68 5116 err = lwtunnel_valid_encap_type(cfg->fc_encap_type, extack);
9ed59592
DA		 5117 if (err < 0)
5118 goto errout;
5119 }
5120
32bc201e
XL		 5121 if (tb[RTA_EXPIRES]) {
5122 unsigned long timeout = addrconf_timeout_fixup(nla_get_u32(tb[RTA_EXPIRES]), HZ);
5123
5124 if (addrconf_finite_timeout(timeout)) {
5125 cfg->fc_expires = jiffies_to_clock_t(timeout * HZ);
5126 cfg->fc_flags |= RTF_EXPIRES;
5127 }
5128 }
5129
86872cb5
TG		 5130 err = 0;
5131errout:
5132 return err;
1da177e4
LT		 5133}
5134
6b9ea5a6 5135struct rt6_nh {
8d1c802b 5136 struct fib6_info *fib6_info;
6b9ea5a6 5137 struct fib6_config r_cfg;
6b9ea5a6
RP		 5138 struct list_head next;
5139};
5140
d4ead6b3
DA		 5141static int ip6_route_info_append(struct net *net,
5142 struct list_head *rt6_nh_list,
8d1c802b
DA		 5143 struct fib6_info *rt,
5144 struct fib6_config *r_cfg)
6b9ea5a6
RP		 5145{
5146 struct rt6_nh *nh;
6b9ea5a6
RP		 5147 int err = -EEXIST;
5148
5149 list_for_each_entry(nh, rt6_nh_list, next) {
8d1c802b
DA		 5150 /* check if fib6_info already exists */
5151 if (rt6_duplicate_nexthop(nh->fib6_info, rt))
6b9ea5a6
RP		 5152 return err;
5153 }
5154
5155 nh = kzalloc(sizeof(*nh), GFP_KERNEL);
5156 if (!nh)
5157 return -ENOMEM;
8d1c802b 5158 nh->fib6_info = rt;
6b9ea5a6
RP		 5159 memcpy(&nh->r_cfg, r_cfg, sizeof(*r_cfg));
5160 list_add_tail(&nh->next, rt6_nh_list);
5161
5162 return 0;
5163}
5164
8d1c802b
DA		 5165static void ip6_route_mpath_notify(struct fib6_info *rt,
5166 struct fib6_info *rt_last,
3b1137fe
DA		 5167 struct nl_info *info,
5168 __u16 nlflags)
5169{
5170 /* if this is an APPEND route, then rt points to the first route
5171 * inserted and rt_last points to last route inserted. Userspace
5172 * wants a consistent dump of the route which starts at the first
5173 * nexthop. Since sibling routes are always added at the end of
5174 * the list, find the first sibling of the last route appended
5175 */
93c2fb25
DA		 5176 if ((nlflags & NLM_F_APPEND) && rt_last && rt_last->fib6_nsiblings) {
5177 rt = list_first_entry(&rt_last->fib6_siblings,
8d1c802b 5178 struct fib6_info,
93c2fb25 5179 fib6_siblings);
3b1137fe
DA		 5180 }
5181
5182 if (rt)
5183 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
5184}
5185
0ee0f47c
IS		 5186static bool ip6_route_mpath_should_notify(const struct fib6_info *rt)
5187{
5188 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
5189 bool should_notify = false;
5190 struct fib6_info *leaf;
5191 struct fib6_node *fn;
5192
5193 rcu_read_lock();
5194 fn = rcu_dereference(rt->fib6_node);
5195 if (!fn)
5196 goto out;
5197
5198 leaf = rcu_dereference(fn->leaf);
5199 if (!leaf)
5200 goto out;
5201
5202 if (rt == leaf ||
5203 (rt_can_ecmp && rt->fib6_metric == leaf->fib6_metric &&
5204 rt6_qualify_for_ecmp(leaf)))
5205 should_notify = true;
5206out:
5207 rcu_read_unlock();
5208
5209 return should_notify;
5210}
5211
4619bcf9
DA		 5212static int fib6_gw_from_attr(struct in6_addr *gw, struct nlattr *nla,
5213 struct netlink_ext_ack *extack)
5214{
5215 if (nla_len(nla) < sizeof(*gw)) {
5216 NL_SET_ERR_MSG(extack, "Invalid IPv6 address in RTA_GATEWAY");
5217 return -EINVAL;
5218 }
5219
5220 *gw = nla_get_in6_addr(nla);
5221
5222 return 0;
5223}
5224
333c4301
DA		 5225static int ip6_route_multipath_add(struct fib6_config *cfg,
5226 struct netlink_ext_ack *extack)
51ebd318 5227{
8d1c802b 5228 struct fib6_info *rt_notif = NULL, *rt_last = NULL;
3b1137fe 5229 struct nl_info *info = &cfg->fc_nlinfo;
51ebd318
ND		 5230 struct fib6_config r_cfg;
5231 struct rtnexthop *rtnh;
8d1c802b 5232 struct fib6_info *rt;
6b9ea5a6
RP		 5233 struct rt6_nh *err_nh;
5234 struct rt6_nh *nh, *nh_safe;
3b1137fe 5235 __u16 nlflags;
51ebd318
ND		 5236 int remaining;
5237 int attrlen;
6b9ea5a6
RP		 5238 int err = 1;
5239 int nhn = 0;
5240 int replace = (cfg->fc_nlinfo.nlh &&
5241 (cfg->fc_nlinfo.nlh->nlmsg_flags & NLM_F_REPLACE));
5242 LIST_HEAD(rt6_nh_list);
51ebd318 5243
3b1137fe
DA		 5244 nlflags = replace ? NLM_F_REPLACE : NLM_F_CREATE;
5245 if (info->nlh && info->nlh->nlmsg_flags & NLM_F_APPEND)
5246 nlflags |= NLM_F_APPEND;
5247
35f1b4e9 5248 remaining = cfg->fc_mp_len;
51ebd318 5249 rtnh = (struct rtnexthop *)cfg->fc_mp;
51ebd318 5250
6b9ea5a6 5251 /* Parse a Multipath Entry and build a list (rt6_nh_list) of
8d1c802b 5252 * fib6_info structs per nexthop
6b9ea5a6 5253 */
51ebd318
ND		 5254 while (rtnh_ok(rtnh, remaining)) {
5255 memcpy(&r_cfg, cfg, sizeof(*cfg));
5256 if (rtnh->rtnh_ifindex)
5257 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5258
5259 attrlen = rtnh_attrlen(rtnh);
5260 if (attrlen > 0) {
5261 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5262
5263 nla = nla_find(attrs, attrlen, RTA_GATEWAY);
5264 if (nla) {
95bdba23 5265 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
4619bcf9 5266 extack);
95bdba23
DA		 5267 if (err)
5268 goto cleanup;
4619bcf9 5269
51ebd318
ND		 5270 r_cfg.fc_flags |= RTF_GATEWAY;
5271 }
19e42e45 5272 r_cfg.fc_encap = nla_find(attrs, attrlen, RTA_ENCAP);
8bda81a4
DA		 5273
5274 /* RTA_ENCAP_TYPE length checked in
5275 * lwtunnel_valid_encap_type_attr
5276 */
19e42e45
RP		 5277 nla = nla_find(attrs, attrlen, RTA_ENCAP_TYPE);
5278 if (nla)
5279 r_cfg.fc_encap_type = nla_get_u16(nla);
51ebd318 5280 }
6b9ea5a6 5281
68e2ffde 5282 r_cfg.fc_flags |= (rtnh->rtnh_flags & RTNH_F_ONLINK);
acb54e3c 5283 rt = ip6_route_info_create(&r_cfg, GFP_KERNEL, extack);
8c5b83f0
RP		 5284 if (IS_ERR(rt)) {
5285 err = PTR_ERR(rt);
5286 rt = NULL;
6b9ea5a6 5287 goto cleanup;
8c5b83f0 5288 }
b5d2d75e
DA		 5289 if (!rt6_qualify_for_ecmp(rt)) {
5290 err = -EINVAL;
5291 NL_SET_ERR_MSG(extack,
5292 "Device only routes can not be added for IPv6 using the multipath API.");
5293 fib6_info_release(rt);
5294 goto cleanup;
5295 }
6b9ea5a6 5296
1cf844c7 5297 rt->fib6_nh->fib_nh_weight = rtnh->rtnh_hops + 1;
398958ae 5298
d4ead6b3
DA		 5299 err = ip6_route_info_append(info->nl_net, &rt6_nh_list,
5300 rt, &r_cfg);
51ebd318 5301 if (err) {
93531c67 5302 fib6_info_release(rt);
6b9ea5a6
RP		 5303 goto cleanup;
5304 }
5305
5306 rtnh = rtnh_next(rtnh, &remaining);
5307 }
5308
9eee3b49
IS		 5309 if (list_empty(&rt6_nh_list)) {
5310 NL_SET_ERR_MSG(extack,
5311 "Invalid nexthop configuration - no valid nexthops");
5312 return -EINVAL;
5313 }
5314
3b1137fe
DA		 5315 /* for add and replace send one notification with all nexthops.
5316 * Skip the notification in fib6_add_rt2node and send one with
5317 * the full route when done
5318 */
5319 info->skip_notify = 1;
5320
ebee3cad
IS		 5321 /* For add and replace, send one notification with all nexthops. For
5322 * append, send one notification with all appended nexthops.
5323 */
5324 info->skip_notify_kernel = 1;
5325
6b9ea5a6
RP		 5326 err_nh = NULL;
5327 list_for_each_entry(nh, &rt6_nh_list, next) {
8d1c802b
DA		 5328 err = __ip6_ins_rt(nh->fib6_info, info, extack);
5329 fib6_info_release(nh->fib6_info);
93531c67 5330
f7225172
DA		 5331 if (!err) {
5332 /* save reference to last route successfully inserted */
5333 rt_last = nh->fib6_info;
5334
5335 /* save reference to first route for notification */
5336 if (!rt_notif)
5337 rt_notif = nh->fib6_info;
5338 }
3b1137fe 5339
8d1c802b
DA		 5340 /* nh->fib6_info is used or freed at this point, reset to NULL*/
5341 nh->fib6_info = NULL;
6b9ea5a6
RP		 5342 if (err) {
5343 if (replace && nhn)
a5a82d84
JK		 5344 NL_SET_ERR_MSG_MOD(extack,
5345 "multipath route replace failed (check consistency of installed routes)");
6b9ea5a6
RP		 5346 err_nh = nh;
5347 goto add_errout;
51ebd318 5348 }
6b9ea5a6 5349
1a72418b 5350 /* Because each route is added like a single route we remove
27596472
MK		 5351 * these flags after the first nexthop: if there is a collision,
5352 * we have already failed to add the first nexthop:
5353 * fib6_add_rt2node() has rejected it; when replacing, old
5354 * nexthops have been replaced by first new, the rest should
5355 * be added to it.
1a72418b 5356 */
864db232
MUA		 5357 if (cfg->fc_nlinfo.nlh) {
5358 cfg->fc_nlinfo.nlh->nlmsg_flags &= ~(NLM_F_EXCL |
5359 NLM_F_REPLACE);
5360 cfg->fc_nlinfo.nlh->nlmsg_flags |= NLM_F_CREATE;
5361 }
6b9ea5a6
RP		 5362 nhn++;
5363 }
5364
0ee0f47c
IS		 5365 /* An in-kernel notification should only be sent in case the new
5366 * multipath route is added as the first route in the node, or if
5367 * it was appended to it. We pass 'rt_notif' since it is the first
5368 * sibling and might allow us to skip some checks in the replace case.
5369 */
5370 if (ip6_route_mpath_should_notify(rt_notif)) {
5371 enum fib_event_type fib_event;
5372
5373 if (rt_notif->fib6_nsiblings != nhn - 1)
5374 fib_event = FIB_EVENT_ENTRY_APPEND;
5375 else
caafb250 5376 fib_event = FIB_EVENT_ENTRY_REPLACE;
0ee0f47c
IS		 5377
5378 err = call_fib6_multipath_entry_notifiers(info->nl_net,
5379 fib_event, rt_notif,
5380 nhn - 1, extack);
5381 if (err) {
5382 /* Delete all the siblings that were just added */
5383 err_nh = NULL;
5384 goto add_errout;
5385 }
5386 }
ebee3cad 5387
3b1137fe
DA		 5388 /* success ... tell user about new route */
5389 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
6b9ea5a6
RP		 5390 goto cleanup;
5391
5392add_errout:
3b1137fe
DA		 5393 /* send notification for routes that were added so that
5394 * the delete notifications sent by ip6_route_del are
5395 * coherent
5396 */
5397 if (rt_notif)
5398 ip6_route_mpath_notify(rt_notif, rt_last, info, nlflags);
5399
6b9ea5a6
RP		 5400 /* Delete routes that were already added */
5401 list_for_each_entry(nh, &rt6_nh_list, next) {
5402 if (err_nh == nh)
5403 break;
333c4301 5404 ip6_route_del(&nh->r_cfg, extack);
6b9ea5a6
RP		 5405 }
5406
5407cleanup:
5408 list_for_each_entry_safe(nh, nh_safe, &rt6_nh_list, next) {
8d1c802b
DA		 5409 if (nh->fib6_info)
5410 fib6_info_release(nh->fib6_info);
6b9ea5a6
RP		 5411 list_del(&nh->next);
5412 kfree(nh);
5413 }
5414
5415 return err;
5416}
5417
333c4301
DA		 5418static int ip6_route_multipath_del(struct fib6_config *cfg,
5419 struct netlink_ext_ack *extack)
6b9ea5a6
RP		 5420{
5421 struct fib6_config r_cfg;
5422 struct rtnexthop *rtnh;
2291267e 5423 int last_err = 0;
6b9ea5a6
RP		 5424 int remaining;
5425 int attrlen;
2291267e 5426 int err;
6b9ea5a6
RP		 5427
5428 remaining = cfg->fc_mp_len;
5429 rtnh = (struct rtnexthop *)cfg->fc_mp;
5430
5431 /* Parse a Multipath Entry */
5432 while (rtnh_ok(rtnh, remaining)) {
5433 memcpy(&r_cfg, cfg, sizeof(*cfg));
5434 if (rtnh->rtnh_ifindex)
5435 r_cfg.fc_ifindex = rtnh->rtnh_ifindex;
5436
5437 attrlen = rtnh_attrlen(rtnh);
5438 if (attrlen > 0) {
5439 struct nlattr *nla, *attrs = rtnh_attrs(rtnh);
5440
5441 nla = nla_find(attrs, attrlen, RTA_GATEWAY);
5442 if (nla) {
1ff15a71
DA		 5443 err = fib6_gw_from_attr(&r_cfg.fc_gateway, nla,
5444 extack);
e30a845b
DA		 5445 if (err) {
5446 last_err = err;
5447 goto next_rtnh;
5448 }
1ff15a71 5449
6b9ea5a6
RP		 5450 r_cfg.fc_flags |= RTF_GATEWAY;
5451 }
5452 }
333c4301 5453 err = ip6_route_del(&r_cfg, extack);
6b9ea5a6
RP		 5454 if (err)
5455 last_err = err;
5456
e30a845b 5457next_rtnh:
51ebd318
ND		 5458 rtnh = rtnh_next(rtnh, &remaining);
5459 }
5460
5461 return last_err;
5462}
5463
c21ef3e3
DA		 5464static int inet6_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5465 struct netlink_ext_ack *extack)
1da177e4 5466{
86872cb5
TG		 5467 struct fib6_config cfg;
5468 int err;
1da177e4 5469
333c4301 5470 err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
86872cb5
TG		 5471 if (err < 0)
5472 return err;
5473
5b98324e
DA		 5474 if (cfg.fc_nh_id &&
5475 !nexthop_find_by_id(sock_net(skb->sk), cfg.fc_nh_id)) {
5476 NL_SET_ERR_MSG(extack, "Nexthop id does not exist");
5477 return -EINVAL;
5478 }
5479
51ebd318 5480 if (cfg.fc_mp)
333c4301 5481 return ip6_route_multipath_del(&cfg, extack);
0ae81335
DA		 5482 else {
5483 cfg.fc_delete_all_nh = 1;
333c4301 5484 return ip6_route_del(&cfg, extack);
0ae81335 5485 }
1da177e4
LT		 5486}
5487
c21ef3e3
DA		 5488static int inet6_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
5489 struct netlink_ext_ack *extack)
1da177e4 5490{
86872cb5
TG		 5491 struct fib6_config cfg;
5492 int err;
1da177e4 5493
333c4301 5494 err = rtm_to_fib6_config(skb, nlh, &cfg, extack);
86872cb5
TG		 5495 if (err < 0)
5496 return err;
5497
67f69513
DA		 5498 if (cfg.fc_metric == 0)
5499 cfg.fc_metric = IP6_RT_PRIO_USER;
5500
51ebd318 5501 if (cfg.fc_mp)
333c4301 5502 return ip6_route_multipath_add(&cfg, extack);
51ebd318 5503 else
acb54e3c 5504 return ip6_route_add(&cfg, GFP_KERNEL, extack);
1da177e4
LT		 5505}
5506
a1b7a1f0
DA		 5507/* add the overhead of this fib6_nh to nexthop_len */
5508static int rt6_nh_nlmsg_size(struct fib6_nh *nh, void *arg)
339bf98f 5509{
a1b7a1f0 5510 int *nexthop_len = arg;
beb1afac 5511
a1b7a1f0
DA		 5512 *nexthop_len += nla_total_size(0) /* RTA_MULTIPATH */
5513 + NLA_ALIGN(sizeof(struct rtnexthop))
5514 + nla_total_size(16); /* RTA_GATEWAY */
5515
5516 if (nh->fib_nh_lws) {
5517 /* RTA_ENCAP_TYPE */
5518 *nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
5519 /* RTA_ENCAP */
5520 *nexthop_len += nla_total_size(2);
5521 }
f88d8ea6 5522
a1b7a1f0
DA		 5523 return 0;
5524}
beb1afac 5525
a1b7a1f0
DA		 5526static size_t rt6_nlmsg_size(struct fib6_info *f6i)
5527{
5528 int nexthop_len;
5529
5530 if (f6i->nh) {
5531 nexthop_len = nla_total_size(4); /* RTA_NH_ID */
5532 nexthop_for_each_fib6_nh(f6i->nh, rt6_nh_nlmsg_size,
5533 &nexthop_len);
5534 } else {
4cc59f38 5535 struct fib6_info *sibling, *next_sibling;
a1b7a1f0
DA		 5536 struct fib6_nh *nh = f6i->fib6_nh;
5537
5538 nexthop_len = 0;
5539 if (f6i->fib6_nsiblings) {
4cc59f38 5540 rt6_nh_nlmsg_size(nh, &nexthop_len);
a1b7a1f0 5541
4cc59f38
LW		 5542 list_for_each_entry_safe(sibling, next_sibling,
5543 &f6i->fib6_siblings, fib6_siblings) {
5544 rt6_nh_nlmsg_size(sibling->fib6_nh, &nexthop_len);
5545 }
a1b7a1f0
DA		 5546 }
5547 nexthop_len += lwtunnel_get_encap_size(nh->fib_nh_lws);
beb1afac
DA		 5548 }
5549
339bf98f
TG		 5550 return NLMSG_ALIGN(sizeof(struct rtmsg))
5551 + nla_total_size(16) /* RTA_SRC */
5552 + nla_total_size(16) /* RTA_DST */
5553 + nla_total_size(16) /* RTA_GATEWAY */
5554 + nla_total_size(16) /* RTA_PREFSRC */
5555 + nla_total_size(4) /* RTA_TABLE */
5556 + nla_total_size(4) /* RTA_IIF */
5557 + nla_total_size(4) /* RTA_OIF */
5558 + nla_total_size(4) /* RTA_PRIORITY */
6a2b9ce0 5559 + RTAX_MAX * nla_total_size(4) /* RTA_METRICS */
ea697639 5560 + nla_total_size(sizeof(struct rta_cacheinfo))
c78ba6d6 5561 + nla_total_size(TCP_CA_NAME_MAX) /* RTAX_CC_ALGO */
19e42e45 5562 + nla_total_size(1) /* RTA_PREF */
beb1afac
DA		 5563 + nexthop_len;
5564}
5565
f88d8ea6
DA		 5566static int rt6_fill_node_nexthop(struct sk_buff *skb, struct nexthop *nh,
5567 unsigned char *flags)
5568{
5569 if (nexthop_is_multipath(nh)) {
5570 struct nlattr *mp;
5571
4255ff05 5572 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
f88d8ea6
DA		 5573 if (!mp)
5574 goto nla_put_failure;
5575
7bdf4de1 5576 if (nexthop_mpath_fill_node(skb, nh, AF_INET6))
f88d8ea6
DA		 5577 goto nla_put_failure;
5578
5579 nla_nest_end(skb, mp);
5580 } else {
5581 struct fib6_nh *fib6_nh;
5582
5583 fib6_nh = nexthop_fib6_nh(nh);
7bdf4de1 5584 if (fib_nexthop_info(skb, &fib6_nh->nh_common, AF_INET6,
f88d8ea6
DA		 5585 flags, false) < 0)
5586 goto nla_put_failure;
5587 }
5588
5589 return 0;
5590
5591nla_put_failure:
5592 return -EMSGSIZE;
5593}
5594
d4ead6b3 5595static int rt6_fill_node(struct net *net, struct sk_buff *skb,
8d1c802b 5596 struct fib6_info *rt, struct dst_entry *dst,
d4ead6b3 5597 struct in6_addr *dest, struct in6_addr *src,
15e47304 5598 int iif, int type, u32 portid, u32 seq,
f8cfe2ce 5599 unsigned int flags)
1da177e4 5600{
22d0bd82
XL		 5601 struct rt6_info *rt6 = (struct rt6_info *)dst;
5602 struct rt6key *rt6_dst, *rt6_src;
5603 u32 *pmetrics, table, rt6_flags;
f88d8ea6 5604 unsigned char nh_flags = 0;
2d7202bf 5605 struct nlmsghdr *nlh;
22d0bd82 5606 struct rtmsg *rtm;
d4ead6b3 5607 long expires = 0;
1da177e4 5608
15e47304 5609 nlh = nlmsg_put(skb, portid, seq, type, sizeof(*rtm), flags);
38308473 5610 if (!nlh)
26932566 5611 return -EMSGSIZE;
2d7202bf 5612
22d0bd82
XL		 5613 if (rt6) {
5614 rt6_dst = &rt6->rt6i_dst;
5615 rt6_src = &rt6->rt6i_src;
5616 rt6_flags = rt6->rt6i_flags;
5617 } else {
5618 rt6_dst = &rt->fib6_dst;
5619 rt6_src = &rt->fib6_src;
5620 rt6_flags = rt->fib6_flags;
5621 }
5622
2d7202bf 5623 rtm = nlmsg_data(nlh);
1da177e4 5624 rtm->rtm_family = AF_INET6;
22d0bd82
XL		 5625 rtm->rtm_dst_len = rt6_dst->plen;
5626 rtm->rtm_src_len = rt6_src->plen;
1da177e4 5627 rtm->rtm_tos = 0;
93c2fb25
DA		 5628 if (rt->fib6_table)
5629 table = rt->fib6_table->tb6_id;
c71099ac 5630 else
9e762a4a 5631 table = RT6_TABLE_UNSPEC;
97f0082a 5632 rtm->rtm_table = table < 256 ? table : RT_TABLE_COMPAT;
c78679e8
DM		 5633 if (nla_put_u32(skb, RTA_TABLE, table))
5634 goto nla_put_failure;
e8478e80
DA		 5635
5636 rtm->rtm_type = rt->fib6_type;
1da177e4
LT		 5637 rtm->rtm_flags = 0;
5638 rtm->rtm_scope = RT_SCOPE_UNIVERSE;
93c2fb25 5639 rtm->rtm_protocol = rt->fib6_protocol;
1da177e4 5640
22d0bd82 5641 if (rt6_flags & RTF_CACHE)
1da177e4
LT		 5642 rtm->rtm_flags |= RTM_F_CLONED;
5643
d4ead6b3
DA		 5644 if (dest) {
5645 if (nla_put_in6_addr(skb, RTA_DST, dest))
c78679e8 5646 goto nla_put_failure;
1ab1457c 5647 rtm->rtm_dst_len = 128;
1da177e4 5648 } else if (rtm->rtm_dst_len)
22d0bd82 5649 if (nla_put_in6_addr(skb, RTA_DST, &rt6_dst->addr))
c78679e8 5650 goto nla_put_failure;
1da177e4
LT		 5651#ifdef CONFIG_IPV6_SUBTREES
5652 if (src) {
930345ea 5653 if (nla_put_in6_addr(skb, RTA_SRC, src))
c78679e8 5654 goto nla_put_failure;
1ab1457c 5655 rtm->rtm_src_len = 128;
c78679e8 5656 } else if (rtm->rtm_src_len &&
22d0bd82 5657 nla_put_in6_addr(skb, RTA_SRC, &rt6_src->addr))
c78679e8 5658 goto nla_put_failure;
1da177e4 5659#endif
7bc570c8
YH		 5660 if (iif) {
5661#ifdef CONFIG_IPV6_MROUTE
22d0bd82 5662 if (ipv6_addr_is_multicast(&rt6_dst->addr)) {
fd61c6ba
DA		 5663 int err = ip6mr_get_route(net, skb, rtm, portid);
5664
5665 if (err == 0)
5666 return 0;
5667 if (err < 0)
5668 goto nla_put_failure;
7bc570c8
YH		 5669 } else
5670#endif
c78679e8
DM		 5671 if (nla_put_u32(skb, RTA_IIF, iif))
5672 goto nla_put_failure;
d4ead6b3 5673 } else if (dest) {
1da177e4 5674 struct in6_addr saddr_buf;
d4ead6b3 5675 if (ip6_route_get_saddr(net, rt, dest, 0, &saddr_buf) == 0 &&
930345ea 5676 nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
c78679e8 5677 goto nla_put_failure;
1da177e4 5678 }
2d7202bf 5679
93c2fb25 5680 if (rt->fib6_prefsrc.plen) {
c3968a85 5681 struct in6_addr saddr_buf;
93c2fb25 5682 saddr_buf = rt->fib6_prefsrc.addr;
930345ea 5683 if (nla_put_in6_addr(skb, RTA_PREFSRC, &saddr_buf))
c78679e8 5684 goto nla_put_failure;
c3968a85
DW		 5685 }
5686
d4ead6b3
DA		 5687 pmetrics = dst ? dst_metrics_ptr(dst) : rt->fib6_metrics->metrics;
5688 if (rtnetlink_put_metrics(skb, pmetrics) < 0)
2d7202bf
TG		 5689 goto nla_put_failure;
5690
93c2fb25 5691 if (nla_put_u32(skb, RTA_PRIORITY, rt->fib6_metric))
c78679e8 5692 goto nla_put_failure;
8253947e 5693
beb1afac
DA		 5694 /* For multipath routes, walk the siblings list and add
5695 * each as a nexthop within RTA_MULTIPATH.
5696 */
22d0bd82
XL		 5697 if (rt6) {
5698 if (rt6_flags & RTF_GATEWAY &&
5699 nla_put_in6_addr(skb, RTA_GATEWAY, &rt6->rt6i_gateway))
5700 goto nla_put_failure;
5701
5702 if (dst->dev && nla_put_u32(skb, RTA_OIF, dst->dev->ifindex))
5703 goto nla_put_failure;
6b13d8f7
OH		 5704
5705 if (dst->lwtstate &&
5706 lwtunnel_fill_encap(skb, dst->lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
5707 goto nla_put_failure;
22d0bd82 5708 } else if (rt->fib6_nsiblings) {
8d1c802b 5709 struct fib6_info *sibling, *next_sibling;
beb1afac
DA		 5710 struct nlattr *mp;
5711
ae0be8de 5712 mp = nla_nest_start_noflag(skb, RTA_MULTIPATH);
beb1afac
DA		 5713 if (!mp)
5714 goto nla_put_failure;
5715
1cf844c7 5716 if (fib_add_nexthop(skb, &rt->fib6_nh->nh_common,
597aa16c
XL		 5717 rt->fib6_nh->fib_nh_weight, AF_INET6,
5718 0) < 0)
beb1afac
DA		 5719 goto nla_put_failure;
5720
5721 list_for_each_entry_safe(sibling, next_sibling,
93c2fb25 5722 &rt->fib6_siblings, fib6_siblings) {
1cf844c7 5723 if (fib_add_nexthop(skb, &sibling->fib6_nh->nh_common,
7bdf4de1 5724 sibling->fib6_nh->fib_nh_weight,
597aa16c 5725 AF_INET6, 0) < 0)
beb1afac
DA		 5726 goto nla_put_failure;
5727 }
5728
5729 nla_nest_end(skb, mp);
f88d8ea6
DA		 5730 } else if (rt->nh) {
5731 if (nla_put_u32(skb, RTA_NH_ID, rt->nh->id))
5732 goto nla_put_failure;
ecc5663c 5733
f88d8ea6
DA		 5734 if (nexthop_is_blackhole(rt->nh))
5735 rtm->rtm_type = RTN_BLACKHOLE;
5736
bdf00bf2 5737 if (READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode) &&
4f80116d 5738 rt6_fill_node_nexthop(skb, rt->nh, &nh_flags) < 0)
f88d8ea6
DA		 5739 goto nla_put_failure;
5740
5741 rtm->rtm_flags |= nh_flags;
5742 } else {
7bdf4de1 5743 if (fib_nexthop_info(skb, &rt->fib6_nh->nh_common, AF_INET6,
ecc5663c 5744 &nh_flags, false) < 0)
beb1afac 5745 goto nla_put_failure;
ecc5663c
DA		 5746
5747 rtm->rtm_flags |= nh_flags;
beb1afac
DA		 5748 }
5749
22d0bd82 5750 if (rt6_flags & RTF_EXPIRES) {
14895687
DA		 5751 expires = dst ? dst->expires : rt->expires;
5752 expires -= jiffies;
5753 }
69cdf8f9 5754
bb3c4ab9 5755 if (!dst) {
d95d6320 5756 if (READ_ONCE(rt->offload))
bb3c4ab9 5757 rtm->rtm_flags |= RTM_F_OFFLOAD;
d95d6320 5758 if (READ_ONCE(rt->trap))
bb3c4ab9 5759 rtm->rtm_flags |= RTM_F_TRAP;
d95d6320 5760 if (READ_ONCE(rt->offload_failed))
0c5fcf9e 5761 rtm->rtm_flags |= RTM_F_OFFLOAD_FAILED;
bb3c4ab9
IS		 5762 }
5763
d4ead6b3 5764 if (rtnl_put_cacheinfo(skb, dst, 0, expires, dst ? dst->error : 0) < 0)
e3703b3d 5765 goto nla_put_failure;
2d7202bf 5766
22d0bd82 5767 if (nla_put_u8(skb, RTA_PREF, IPV6_EXTRACT_PREF(rt6_flags)))
c78ba6d6
LR		 5768 goto nla_put_failure;
5769
19e42e45 5770
053c095a
JB		 5771 nlmsg_end(skb, nlh);
5772 return 0;
2d7202bf
TG		 5773
5774nla_put_failure:
26932566
PM		 5775 nlmsg_cancel(skb, nlh);
5776 return -EMSGSIZE;
1da177e4
LT		 5777}
5778
2c170e07
DA		 5779static int fib6_info_nh_uses_dev(struct fib6_nh *nh, void *arg)
5780{
5781 const struct net_device *dev = arg;
5782
5783 if (nh->fib_nh_dev == dev)
5784 return 1;
5785
5786 return 0;
5787}
5788
13e38901
DA		 5789static bool fib6_info_uses_dev(const struct fib6_info *f6i,
5790 const struct net_device *dev)
5791{
2c170e07
DA		 5792 if (f6i->nh) {
5793 struct net_device *_dev = (struct net_device *)dev;
5794
5795 return !!nexthop_for_each_fib6_nh(f6i->nh,
5796 fib6_info_nh_uses_dev,
5797 _dev);
5798 }
5799
1cf844c7 5800 if (f6i->fib6_nh->fib_nh_dev == dev)
13e38901
DA		 5801 return true;
5802
5803 if (f6i->fib6_nsiblings) {
5804 struct fib6_info *sibling, *next_sibling;
5805
5806 list_for_each_entry_safe(sibling, next_sibling,
5807 &f6i->fib6_siblings, fib6_siblings) {
1cf844c7 5808 if (sibling->fib6_nh->fib_nh_dev == dev)
13e38901
DA		 5809 return true;
5810 }
5811 }
5812
5813 return false;
5814}
5815
1e47b483
SB		 5816struct fib6_nh_exception_dump_walker {
5817 struct rt6_rtnl_dump_arg *dump;
5818 struct fib6_info *rt;
5819 unsigned int flags;
5820 unsigned int skip;
5821 unsigned int count;
5822};
5823
5824static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg)
5825{
5826 struct fib6_nh_exception_dump_walker *w = arg;
5827 struct rt6_rtnl_dump_arg *dump = w->dump;
5828 struct rt6_exception_bucket *bucket;
5829 struct rt6_exception *rt6_ex;
5830 int i, err;
5831
5832 bucket = fib6_nh_get_excptn_bucket(nh, NULL);
5833 if (!bucket)
5834 return 0;
5835
5836 for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) {
5837 hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) {
5838 if (w->skip) {
5839 w->skip--;
5840 continue;
5841 }
5842
5843 /* Expiration of entries doesn't bump sernum, insertion
5844 * does. Removal is triggered by insertion, so we can
5845 * rely on the fact that if entries change between two
5846 * partial dumps, this node is scanned again completely,
5847 * see rt6_insert_exception() and fib6_dump_table().
5848 *
5849 * Count expired entries we go through as handled
5850 * entries that we'll skip next time, in case of partial
5851 * node dump. Otherwise, if entries expire meanwhile,
5852 * we'll skip the wrong amount.
5853 */
5854 if (rt6_check_expired(rt6_ex->rt6i)) {
5855 w->count++;
5856 continue;
5857 }
5858
5859 err = rt6_fill_node(dump->net, dump->skb, w->rt,
5860 &rt6_ex->rt6i->dst, NULL, NULL, 0,
5861 RTM_NEWROUTE,
5862 NETLINK_CB(dump->cb->skb).portid,
5863 dump->cb->nlh->nlmsg_seq, w->flags);
5864 if (err)
5865 return err;
5866
5867 w->count++;
5868 }
5869 bucket++;
5870 }
5871
5872 return 0;
5873}
5874
bf9a8a06 5875/* Return -1 if done with node, number of handled routes on partial dump */
1e47b483 5876int rt6_dump_route(struct fib6_info *rt, void *p_arg, unsigned int skip)
1da177e4
LT		 5877{
5878 struct rt6_rtnl_dump_arg *arg = (struct rt6_rtnl_dump_arg *) p_arg;
13e38901
DA		 5879 struct fib_dump_filter *filter = &arg->filter;
5880 unsigned int flags = NLM_F_MULTI;
1f17e2f2 5881 struct net *net = arg->net;
1e47b483 5882 int count = 0;
1f17e2f2 5883
421842ed 5884 if (rt == net->ipv6.fib6_null_entry)
bf9a8a06 5885 return -1;
1da177e4 5886
13e38901
DA		 5887 if ((filter->flags & RTM_F_PREFIX) &&
5888 !(rt->fib6_flags & RTF_PREFIX_RT)) {
5889 /* success since this is not a prefix route */
bf9a8a06 5890 return -1;
13e38901 5891 }
1e47b483
SB		 5892 if (filter->filter_set &&
5893 ((filter->rt_type && rt->fib6_type != filter->rt_type) ||
5894 (filter->dev && !fib6_info_uses_dev(rt, filter->dev)) ||
5895 (filter->protocol && rt->fib6_protocol != filter->protocol))) {
5896 return -1;
5897 }
5898
5899 if (filter->filter_set ||
5900 !filter->dump_routes || !filter->dump_exceptions) {
13e38901 5901 flags |= NLM_F_DUMP_FILTERED;
f8cfe2ce 5902 }
1da177e4 5903
1e47b483
SB		 5904 if (filter->dump_routes) {
5905 if (skip) {
5906 skip--;
5907 } else {
5908 if (rt6_fill_node(net, arg->skb, rt, NULL, NULL, NULL,
5909 0, RTM_NEWROUTE,
5910 NETLINK_CB(arg->cb->skb).portid,
5911 arg->cb->nlh->nlmsg_seq, flags)) {
5912 return 0;
5913 }
5914 count++;
5915 }
5916 }
5917
5918 if (filter->dump_exceptions) {
5919 struct fib6_nh_exception_dump_walker w = { .dump = arg,
5920 .rt = rt,
5921 .flags = flags,
5922 .skip = skip,
5923 .count = 0 };
5924 int err;
5925
3b525691 5926 rcu_read_lock();
1e47b483
SB		 5927 if (rt->nh) {
5928 err = nexthop_for_each_fib6_nh(rt->nh,
5929 rt6_nh_dump_exceptions,
5930 &w);
5931 } else {
5932 err = rt6_nh_dump_exceptions(rt->fib6_nh, &w);
5933 }
3b525691 5934 rcu_read_unlock();
1e47b483
SB		 5935
5936 if (err)
74fd304f 5937 return count + w.count;
1e47b483 5938 }
bf9a8a06
SB		 5939
5940 return -1;
1da177e4
LT		 5941}
5942
0eff0a27
JK		 5943static int inet6_rtm_valid_getroute_req(struct sk_buff *skb,
5944 const struct nlmsghdr *nlh,
5945 struct nlattr **tb,
5946 struct netlink_ext_ack *extack)
5947{
5948 struct rtmsg *rtm;
5949 int i, err;
5950
5951 if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
5952 NL_SET_ERR_MSG_MOD(extack,
5953 "Invalid header for get route request");
5954 return -EINVAL;
5955 }
5956
5957 if (!netlink_strict_get_check(skb))
8cb08174
JB		 5958 return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
5959 rtm_ipv6_policy, extack);
0eff0a27
JK		 5960
5961 rtm = nlmsg_data(nlh);
5962 if ((rtm->rtm_src_len && rtm->rtm_src_len != 128) ||
5963 (rtm->rtm_dst_len && rtm->rtm_dst_len != 128) ||
5964 rtm->rtm_table || rtm->rtm_protocol || rtm->rtm_scope ||
5965 rtm->rtm_type) {
5966 NL_SET_ERR_MSG_MOD(extack, "Invalid values in header for get route request");
5967 return -EINVAL;
5968 }
5969 if (rtm->rtm_flags & ~RTM_F_FIB_MATCH) {
5970 NL_SET_ERR_MSG_MOD(extack,
5971 "Invalid flags for get route request");
5972 return -EINVAL;
5973 }
5974
8cb08174
JB		 5975 err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
5976 rtm_ipv6_policy, extack);
0eff0a27
JK		 5977 if (err)
5978 return err;
5979
5980 if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
5981 (tb[RTA_DST] && !rtm->rtm_dst_len)) {
5982 NL_SET_ERR_MSG_MOD(extack, "rtm_src_len and rtm_dst_len must be 128 for IPv6");
5983 return -EINVAL;
5984 }
5985
5986 for (i = 0; i <= RTA_MAX; i++) {
5987 if (!tb[i])
5988 continue;
5989
5990 switch (i) {
5991 case RTA_SRC:
5992 case RTA_DST:
5993 case RTA_IIF:
5994 case RTA_OIF:
5995 case RTA_MARK:
5996 case RTA_UID:
5997 case RTA_SPORT:
5998 case RTA_DPORT:
5999 case RTA_IP_PROTO:
6000 break;
6001 default:
6002 NL_SET_ERR_MSG_MOD(extack, "Unsupported attribute in get route request");
6003 return -EINVAL;
6004 }
6005 }
6006
6007 return 0;
6008}
6009
c21ef3e3
DA		 6010static int inet6_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
6011 struct netlink_ext_ack *extack)
1da177e4 6012{
3b1e0a65 6013 struct net *net = sock_net(in_skb->sk);
ab364a6f 6014 struct nlattr *tb[RTA_MAX+1];
18c3a61c 6015 int err, iif = 0, oif = 0;
a68886a6 6016 struct fib6_info *from;
18c3a61c 6017 struct dst_entry *dst;
ab364a6f 6018 struct rt6_info *rt;
1da177e4 6019 struct sk_buff *skb;
ab364a6f 6020 struct rtmsg *rtm;
744486d4 6021 struct flowi6 fl6 = {};
18c3a61c 6022 bool fibmatch;
1da177e4 6023
0eff0a27 6024 err = inet6_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
ab364a6f
TG		 6025 if (err < 0)
6026 goto errout;
1da177e4 6027
ab364a6f 6028 err = -EINVAL;
38b7097b
HFS		 6029 rtm = nlmsg_data(nlh);
6030 fl6.flowlabel = ip6_make_flowinfo(rtm->rtm_tos, 0);
18c3a61c 6031 fibmatch = !!(rtm->rtm_flags & RTM_F_FIB_MATCH);
1da177e4 6032
ab364a6f
TG		 6033 if (tb[RTA_SRC]) {
6034 if (nla_len(tb[RTA_SRC]) < sizeof(struct in6_addr))
6035 goto errout;
6036
4e3fd7a0 6037 fl6.saddr = *(struct in6_addr *)nla_data(tb[RTA_SRC]);
ab364a6f
TG		 6038 }
6039
6040 if (tb[RTA_DST]) {
6041 if (nla_len(tb[RTA_DST]) < sizeof(struct in6_addr))
6042 goto errout;
6043
4e3fd7a0 6044 fl6.daddr = *(struct in6_addr *)nla_data(tb[RTA_DST]);
ab364a6f
TG		 6045 }
6046
6047 if (tb[RTA_IIF])
6048 iif = nla_get_u32(tb[RTA_IIF]);
6049
6050 if (tb[RTA_OIF])
72331bc0 6051 oif = nla_get_u32(tb[RTA_OIF]);
1da177e4 6052
2e47b291
LC		 6053 if (tb[RTA_MARK])
6054 fl6.flowi6_mark = nla_get_u32(tb[RTA_MARK]);
6055
622ec2c9
LC		 6056 if (tb[RTA_UID])
6057 fl6.flowi6_uid = make_kuid(current_user_ns(),
6058 nla_get_u32(tb[RTA_UID]));
6059 else
6060 fl6.flowi6_uid = iif ? INVALID_UID : current_uid();
6061
eacb9384
RP		 6062 if (tb[RTA_SPORT])
6063 fl6.fl6_sport = nla_get_be16(tb[RTA_SPORT]);
6064
6065 if (tb[RTA_DPORT])
6066 fl6.fl6_dport = nla_get_be16(tb[RTA_DPORT]);
6067
6068 if (tb[RTA_IP_PROTO]) {
6069 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
5e1a99ea
HL		 6070 &fl6.flowi6_proto, AF_INET6,
6071 extack);
eacb9384
RP		 6072 if (err)
6073 goto errout;
6074 }
6075
1da177e4
LT		 6076 if (iif) {
6077 struct net_device *dev;
72331bc0
SL		 6078 int flags = 0;
6079
121622db
FW		 6080 rcu_read_lock();
6081
6082 dev = dev_get_by_index_rcu(net, iif);
1da177e4 6083 if (!dev) {
121622db 6084 rcu_read_unlock();
1da177e4 6085 err = -ENODEV;
ab364a6f 6086 goto errout;
1da177e4 6087 }
72331bc0
SL		 6088
6089 fl6.flowi6_iif = iif;
6090
6091 if (!ipv6_addr_any(&fl6.saddr))
6092 flags |= RT6_LOOKUP_F_HAS_SADDR;
6093
b75cc8f9 6094 dst = ip6_route_input_lookup(net, dev, &fl6, NULL, flags);
121622db
FW		 6095
6096 rcu_read_unlock();
72331bc0
SL		 6097 } else {
6098 fl6.flowi6_oif = oif;
6099
58acfd71 6100 dst = ip6_route_output(net, NULL, &fl6);
18c3a61c
RP		 6101 }
6102
18c3a61c
RP		 6103
6104 rt = container_of(dst, struct rt6_info, dst);
6105 if (rt->dst.error) {
6106 err = rt->dst.error;
6107 ip6_rt_put(rt);
6108 goto errout;
1da177e4
LT		 6109 }
6110
9d6acb3b
WC		 6111 if (rt == net->ipv6.ip6_null_entry) {
6112 err = rt->dst.error;
6113 ip6_rt_put(rt);
6114 goto errout;
6115 }
6116
ab364a6f 6117 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
38308473 6118 if (!skb) {
94e187c0 6119 ip6_rt_put(rt);
ab364a6f
TG		 6120 err = -ENOBUFS;
6121 goto errout;
6122 }
1da177e4 6123
d8d1f30b 6124 skb_dst_set(skb, &rt->dst);
a68886a6
DA		 6125
6126 rcu_read_lock();
6127 from = rcu_dereference(rt->from);
886b7a50
MKL		 6128 if (from) {
6129 if (fibmatch)
6130 err = rt6_fill_node(net, skb, from, NULL, NULL, NULL,
6131 iif, RTM_NEWROUTE,
6132 NETLINK_CB(in_skb).portid,
6133 nlh->nlmsg_seq, 0);
6134 else
6135 err = rt6_fill_node(net, skb, from, dst, &fl6.daddr,
6136 &fl6.saddr, iif, RTM_NEWROUTE,
6137 NETLINK_CB(in_skb).portid,
6138 nlh->nlmsg_seq, 0);
6139 } else {
6140 err = -ENETUNREACH;
6141 }
a68886a6
DA		 6142 rcu_read_unlock();
6143
1da177e4 6144 if (err < 0) {
ab364a6f
TG		 6145 kfree_skb(skb);
6146 goto errout;
1da177e4
LT		 6147 }
6148
15e47304 6149 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
ab364a6f 6150errout:
1da177e4 6151 return err;
1da177e4
LT		 6152}
6153
8d1c802b 6154void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
37a1d361 6155 unsigned int nlm_flags)
1da177e4
LT		 6156{
6157 struct sk_buff *skb;
5578689a 6158 struct net *net = info->nl_net;
528c4ceb
DL		 6159 u32 seq;
6160 int err;
6161
6162 err = -ENOBUFS;
38308473 6163 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
86872cb5 6164
19e42e45 6165 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
38308473 6166 if (!skb)
21713ebc
TG		 6167 goto errout;
6168
d4ead6b3
DA		 6169 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
6170 event, info->portid, seq, nlm_flags);
26932566
PM		 6171 if (err < 0) {
6172 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6173 WARN_ON(err == -EMSGSIZE);
6174 kfree_skb(skb);
6175 goto errout;
6176 }
15e47304 6177 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
1ce85fe4
PNA		 6178 info->nlh, gfp_any());
6179 return;
21713ebc
TG		 6180errout:
6181 if (err < 0)
5578689a 6182 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
1da177e4
LT		 6183}
6184
19a3b7ee
DA		 6185void fib6_rt_update(struct net *net, struct fib6_info *rt,
6186 struct nl_info *info)
6187{
6188 u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
6189 struct sk_buff *skb;
6190 int err = -ENOBUFS;
6191
19a3b7ee
DA		 6192 skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any());
6193 if (!skb)
6194 goto errout;
6195
6196 err = rt6_fill_node(net, skb, rt, NULL, NULL, NULL, 0,
6197 RTM_NEWROUTE, info->portid, seq, NLM_F_REPLACE);
6198 if (err < 0) {
6199 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6200 WARN_ON(err == -EMSGSIZE);
6201 kfree_skb(skb);
6202 goto errout;
6203 }
6204 rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE,
6205 info->nlh, gfp_any());
6206 return;
6207errout:
6208 if (err < 0)
6209 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
6210}
6211
907eea48 6212void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
0c5fcf9e 6213 bool offload, bool trap, bool offload_failed)
907eea48
AC		 6214{
6215 struct sk_buff *skb;
6216 int err;
6217
d95d6320
ED		 6218 if (READ_ONCE(f6i->offload) == offload &&
6219 READ_ONCE(f6i->trap) == trap &&
6220 READ_ONCE(f6i->offload_failed) == offload_failed)
907eea48
AC		 6221 return;
6222
d95d6320
ED		 6223 WRITE_ONCE(f6i->offload, offload);
6224 WRITE_ONCE(f6i->trap, trap);
6fad361a
AC		 6225
6226 /* 2 means send notifications only if offload_failed was changed. */
6227 if (net->ipv6.sysctl.fib_notify_on_flag_change == 2 &&
d95d6320 6228 READ_ONCE(f6i->offload_failed) == offload_failed)
6fad361a
AC		 6229 return;
6230
d95d6320 6231 WRITE_ONCE(f6i->offload_failed, offload_failed);
907eea48
AC		 6232
6233 if (!rcu_access_pointer(f6i->fib6_node))
6234 /* The route was removed from the tree, do not send
89e8347f 6235 * notification.
907eea48
AC		 6236 */
6237 return;
6238
6239 if (!net->ipv6.sysctl.fib_notify_on_flag_change)
6240 return;
6241
6242 skb = nlmsg_new(rt6_nlmsg_size(f6i), GFP_KERNEL);
6243 if (!skb) {
6244 err = -ENOBUFS;
6245 goto errout;
6246 }
6247
6248 err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0,
6249 0, 0);
6250 if (err < 0) {
6251 /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */
6252 WARN_ON(err == -EMSGSIZE);
6253 kfree_skb(skb);
6254 goto errout;
6255 }
6256
6257 rtnl_notify(skb, net, 0, RTNLGRP_IPV6_ROUTE, NULL, GFP_KERNEL);
6258 return;
6259
6260errout:
6261 rtnl_set_sk_err(net, RTNLGRP_IPV6_ROUTE, err);
6262}
6263EXPORT_SYMBOL(fib6_info_hw_flags_set);
6264
8ed67789 6265static int ip6_route_dev_notify(struct notifier_block *this,
351638e7 6266 unsigned long event, void *ptr)
8ed67789 6267{
351638e7 6268 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
c346dca1 6269 struct net *net = dev_net(dev);
8ed67789 6270
242d3a49
WC		 6271 if (!(dev->flags & IFF_LOOPBACK))
6272 return NOTIFY_OK;
6273
6274 if (event == NETDEV_REGISTER) {
1cf844c7 6275 net->ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = dev;
d8d1f30b 6276 net->ipv6.ip6_null_entry->dst.dev = dev;
8ed67789
DL		 6277 net->ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(dev);
6278#ifdef CONFIG_IPV6_MULTIPLE_TABLES
d8d1f30b 6279 net->ipv6.ip6_prohibit_entry->dst.dev = dev;
8ed67789 6280 net->ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(dev);
d8d1f30b 6281 net->ipv6.ip6_blk_hole_entry->dst.dev = dev;
8ed67789 6282 net->ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(dev);
242d3a49 6283#endif
76da0704
WC		 6284 } else if (event == NETDEV_UNREGISTER &&
6285 dev->reg_state != NETREG_UNREGISTERED) {
6286 /* NETDEV_UNREGISTER could be fired for multiple times by
6287 * netdev_wait_allrefs(). Make sure we only call this once.
6288 */
12d94a80 6289 in6_dev_put_clear(&net->ipv6.ip6_null_entry->rt6i_idev);
242d3a49 6290#ifdef CONFIG_IPV6_MULTIPLE_TABLES
12d94a80
ED		 6291 in6_dev_put_clear(&net->ipv6.ip6_prohibit_entry->rt6i_idev);
6292 in6_dev_put_clear(&net->ipv6.ip6_blk_hole_entry->rt6i_idev);
8ed67789
DL		 6293#endif
6294 }
6295
6296 return NOTIFY_OK;
6297}
6298
1da177e4
LT		 6299/*
6300 * /proc
6301 */
6302
6303#ifdef CONFIG_PROC_FS
1da177e4
LT		 6304static int rt6_stats_seq_show(struct seq_file *seq, void *v)
6305{
69ddb805 6306 struct net *net = (struct net *)seq->private;
1da177e4 6307 seq_printf(seq, "%04x %04x %04x %04x %04x %04x %04x\n",
69ddb805
DL		 6308 net->ipv6.rt6_stats->fib_nodes,
6309 net->ipv6.rt6_stats->fib_route_nodes,
81eb8447 6310 atomic_read(&net->ipv6.rt6_stats->fib_rt_alloc),
69ddb805
DL		 6311 net->ipv6.rt6_stats->fib_rt_entries,
6312 net->ipv6.rt6_stats->fib_rt_cache,
fc66f95c 6313 dst_entries_get_slow(&net->ipv6.ip6_dst_ops),
69ddb805 6314 net->ipv6.rt6_stats->fib_discarded_routes);
1da177e4
LT		 6315
6316 return 0;
6317}
1da177e4
LT		 6318#endif /* CONFIG_PROC_FS */
6319
6320#ifdef CONFIG_SYSCTL
6321
32927393
CH		 6322static int ipv6_sysctl_rtcache_flush(struct ctl_table *ctl, int write,
6323 void *buffer, size_t *lenp, loff_t *ppos)
1da177e4 6324{
c486da34
LAG		 6325 struct net *net;
6326 int delay;
f0fb9b28 6327 int ret;
c486da34 6328 if (!write)
1da177e4 6329 return -EINVAL;
c486da34
LAG		 6330
6331 net = (struct net *)ctl->extra1;
6332 delay = net->ipv6.sysctl.flush_delay;
f0fb9b28
AP		 6333 ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
6334 if (ret)
6335 return ret;
6336
2ac3ac8f 6337 fib6_run_gc(delay <= 0 ? 0 : (unsigned long)delay, net, delay > 0);
c486da34 6338 return 0;
1da177e4
LT		 6339}
6340
ed792e28 6341static struct ctl_table ipv6_route_table_template[] = {
1ab1457c 6342 {
06e6c88f
AK		 6343 .procname = "max_size",
6344 .data = &init_net.ipv6.sysctl.ip6_rt_max_size,
1da177e4 6345 .maxlen = sizeof(int),
06e6c88f
AK		 6346 .mode = 0644,
6347 .proc_handler = proc_dointvec,
1da177e4
LT		 6348 },
6349 {
1da177e4 6350 .procname = "gc_thresh",
9a7ec3a9 6351 .data = &ip6_dst_ops_template.gc_thresh,
1da177e4
LT		 6352 .maxlen = sizeof(int),
6353 .mode = 0644,
6d9f239a 6354 .proc_handler = proc_dointvec,
1da177e4
LT		 6355 },
6356 {
06e6c88f
AK		 6357 .procname = "flush",
6358 .data = &init_net.ipv6.sysctl.flush_delay,
1da177e4 6359 .maxlen = sizeof(int),
06e6c88f
AK		 6360 .mode = 0200,
6361 .proc_handler = ipv6_sysctl_rtcache_flush
1da177e4
LT		 6362 },
6363 {
1da177e4 6364 .procname = "gc_min_interval",
4990509f 6365 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
1da177e4
LT		 6366 .maxlen = sizeof(int),
6367 .mode = 0644,
6d9f239a 6368 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 6369 },
6370 {
1da177e4 6371 .procname = "gc_timeout",
4990509f 6372 .data = &init_net.ipv6.sysctl.ip6_rt_gc_timeout,
1da177e4
LT		 6373 .maxlen = sizeof(int),
6374 .mode = 0644,
6d9f239a 6375 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 6376 },
6377 {
1da177e4 6378 .procname = "gc_interval",
4990509f 6379 .data = &init_net.ipv6.sysctl.ip6_rt_gc_interval,
1da177e4
LT		 6380 .maxlen = sizeof(int),
6381 .mode = 0644,
6d9f239a 6382 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 6383 },
6384 {
1da177e4 6385 .procname = "gc_elasticity",
4990509f 6386 .data = &init_net.ipv6.sysctl.ip6_rt_gc_elasticity,
1da177e4
LT		 6387 .maxlen = sizeof(int),
6388 .mode = 0644,
f3d3f616 6389 .proc_handler = proc_dointvec,
1da177e4
LT		 6390 },
6391 {
1da177e4 6392 .procname = "mtu_expires",
4990509f 6393 .data = &init_net.ipv6.sysctl.ip6_rt_mtu_expires,
1da177e4
LT		 6394 .maxlen = sizeof(int),
6395 .mode = 0644,
6d9f239a 6396 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 6397 },
6398 {
1da177e4 6399 .procname = "min_adv_mss",
4990509f 6400 .data = &init_net.ipv6.sysctl.ip6_rt_min_advmss,
1da177e4
LT		 6401 .maxlen = sizeof(int),
6402 .mode = 0644,
f3d3f616 6403 .proc_handler = proc_dointvec,
1da177e4
LT		 6404 },
6405 {
1da177e4 6406 .procname = "gc_min_interval_ms",
4990509f 6407 .data = &init_net.ipv6.sysctl.ip6_rt_gc_min_interval,
1da177e4
LT		 6408 .maxlen = sizeof(int),
6409 .mode = 0644,
6d9f239a 6410 .proc_handler = proc_dointvec_ms_jiffies,
1da177e4 6411 },
7c6bb7d2
DA		 6412 {
6413 .procname = "skip_notify_on_dev_down",
6414 .data = &init_net.ipv6.sysctl.skip_notify_on_dev_down,
6415 .maxlen = sizeof(int),
6416 .mode = 0644,
b8e8a863 6417 .proc_handler = proc_dointvec_minmax,
eec4844f
MC		 6418 .extra1 = SYSCTL_ZERO,
6419 .extra2 = SYSCTL_ONE,
7c6bb7d2 6420 },
f8572d8f 6421 { }
1da177e4
LT		 6422};
6423
2c8c1e72 6424struct ctl_table * __net_init ipv6_route_sysctl_init(struct net *net)
760f2d01
DL		 6425{
6426 struct ctl_table *table;
6427
6428 table = kmemdup(ipv6_route_table_template,
6429 sizeof(ipv6_route_table_template),
6430 GFP_KERNEL);
5ee09105
YH		 6431
6432 if (table) {
06e6c88f 6433 table[0].data = &net->ipv6.sysctl.ip6_rt_max_size;
86393e52 6434 table[1].data = &net->ipv6.ip6_dst_ops.gc_thresh;
06e6c88f
AK		 6435 table[2].data = &net->ipv6.sysctl.flush_delay;
6436 table[2].extra1 = net;
5ee09105
YH		 6437 table[3].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
6438 table[4].data = &net->ipv6.sysctl.ip6_rt_gc_timeout;
6439 table[5].data = &net->ipv6.sysctl.ip6_rt_gc_interval;
6440 table[6].data = &net->ipv6.sysctl.ip6_rt_gc_elasticity;
6441 table[7].data = &net->ipv6.sysctl.ip6_rt_mtu_expires;
6442 table[8].data = &net->ipv6.sysctl.ip6_rt_min_advmss;
9c69fabe 6443 table[9].data = &net->ipv6.sysctl.ip6_rt_gc_min_interval;
7c6bb7d2 6444 table[10].data = &net->ipv6.sysctl.skip_notify_on_dev_down;
464dc801
EB		 6445
6446 /* Don't export sysctls to unprivileged users */
6447 if (net->user_ns != &init_user_ns)
06e6c88f 6448 table[1].procname = NULL;
5ee09105
YH		 6449 }
6450
760f2d01
DL		 6451 return table;
6452}
1da177e4
LT		 6453#endif
6454
2c8c1e72 6455static int __net_init ip6_route_net_init(struct net *net)
cdb18761 6456{
633d424b 6457 int ret = -ENOMEM;
8ed67789 6458
86393e52
AD		 6459 memcpy(&net->ipv6.ip6_dst_ops, &ip6_dst_ops_template,
6460 sizeof(net->ipv6.ip6_dst_ops));
f2fc6a54 6461
fc66f95c
ED		 6462 if (dst_entries_init(&net->ipv6.ip6_dst_ops) < 0)
6463 goto out_ip6_dst_ops;
6464
1cf844c7 6465 net->ipv6.fib6_null_entry = fib6_info_alloc(GFP_KERNEL, true);
421842ed
DA		 6466 if (!net->ipv6.fib6_null_entry)
6467 goto out_ip6_dst_entries;
1cf844c7
DA		 6468 memcpy(net->ipv6.fib6_null_entry, &fib6_null_entry_template,
6469 sizeof(*net->ipv6.fib6_null_entry));
421842ed 6470
8ed67789
DL		 6471 net->ipv6.ip6_null_entry = kmemdup(&ip6_null_entry_template,
6472 sizeof(*net->ipv6.ip6_null_entry),
6473 GFP_KERNEL);
6474 if (!net->ipv6.ip6_null_entry)
421842ed 6475 goto out_fib6_null_entry;
d8d1f30b 6476 net->ipv6.ip6_null_entry->dst.ops = &net->ipv6.ip6_dst_ops;
62fa8a84
DM		 6477 dst_init_metrics(&net->ipv6.ip6_null_entry->dst,
6478 ip6_template_metrics, true);
d288a162 6479 INIT_LIST_HEAD(&net->ipv6.ip6_null_entry->dst.rt_uncached);
8ed67789
DL		 6480
6481#ifdef CONFIG_IPV6_MULTIPLE_TABLES
feca7d8c 6482 net->ipv6.fib6_has_custom_rules = false;
8ed67789
DL		 6483 net->ipv6.ip6_prohibit_entry = kmemdup(&ip6_prohibit_entry_template,
6484 sizeof(*net->ipv6.ip6_prohibit_entry),
6485 GFP_KERNEL);
68fffc67
PZ		 6486 if (!net->ipv6.ip6_prohibit_entry)
6487 goto out_ip6_null_entry;
d8d1f30b 6488 net->ipv6.ip6_prohibit_entry->dst.ops = &net->ipv6.ip6_dst_ops;
62fa8a84
DM		 6489 dst_init_metrics(&net->ipv6.ip6_prohibit_entry->dst,
6490 ip6_template_metrics, true);
d288a162 6491 INIT_LIST_HEAD(&net->ipv6.ip6_prohibit_entry->dst.rt_uncached);
8ed67789
DL		 6492
6493 net->ipv6.ip6_blk_hole_entry = kmemdup(&ip6_blk_hole_entry_template,
6494 sizeof(*net->ipv6.ip6_blk_hole_entry),
6495 GFP_KERNEL);
68fffc67
PZ		 6496 if (!net->ipv6.ip6_blk_hole_entry)
6497 goto out_ip6_prohibit_entry;
d8d1f30b 6498 net->ipv6.ip6_blk_hole_entry->dst.ops = &net->ipv6.ip6_dst_ops;
62fa8a84
DM		 6499 dst_init_metrics(&net->ipv6.ip6_blk_hole_entry->dst,
6500 ip6_template_metrics, true);
d288a162 6501 INIT_LIST_HEAD(&net->ipv6.ip6_blk_hole_entry->dst.rt_uncached);
b9b33e7c
PA		 6502#ifdef CONFIG_IPV6_SUBTREES
6503 net->ipv6.fib6_routes_require_src = 0;
6504#endif
8ed67789
DL		 6505#endif
6506
b339a47c 6507 net->ipv6.sysctl.flush_delay = 0;
af6d1034 6508 net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
b339a47c
PZ		 6509 net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
6510 net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
6511 net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
6512 net->ipv6.sysctl.ip6_rt_gc_elasticity = 9;
6513 net->ipv6.sysctl.ip6_rt_mtu_expires = 10*60*HZ;
6514 net->ipv6.sysctl.ip6_rt_min_advmss = IPV6_MIN_MTU - 20 - 40;
7c6bb7d2 6515 net->ipv6.sysctl.skip_notify_on_dev_down = 0;
b339a47c 6516
9cb7c013 6517 atomic_set(&net->ipv6.ip6_rt_gc_expire, 30*HZ);
6891a346 6518
8ed67789
DL		 6519 ret = 0;
6520out:
6521 return ret;
f2fc6a54 6522
68fffc67
PZ		 6523#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6524out_ip6_prohibit_entry:
6525 kfree(net->ipv6.ip6_prohibit_entry);
6526out_ip6_null_entry:
6527 kfree(net->ipv6.ip6_null_entry);
6528#endif
421842ed
DA		 6529out_fib6_null_entry:
6530 kfree(net->ipv6.fib6_null_entry);
fc66f95c
ED		 6531out_ip6_dst_entries:
6532 dst_entries_destroy(&net->ipv6.ip6_dst_ops);
f2fc6a54 6533out_ip6_dst_ops:
f2fc6a54 6534 goto out;
cdb18761
DL		 6535}
6536
2c8c1e72 6537static void __net_exit ip6_route_net_exit(struct net *net)
cdb18761 6538{
421842ed 6539 kfree(net->ipv6.fib6_null_entry);
8ed67789
DL		 6540 kfree(net->ipv6.ip6_null_entry);
6541#ifdef CONFIG_IPV6_MULTIPLE_TABLES
6542 kfree(net->ipv6.ip6_prohibit_entry);
6543 kfree(net->ipv6.ip6_blk_hole_entry);
6544#endif
41bb78b4 6545 dst_entries_destroy(&net->ipv6.ip6_dst_ops);
cdb18761
DL		 6546}
6547
d189634e
TG		 6548static int __net_init ip6_route_net_init_late(struct net *net)
6549{
6550#ifdef CONFIG_PROC_FS
768b3c74
ZS		 6551 if (!proc_create_net("ipv6_route", 0, net->proc_net,
6552 &ipv6_route_seq_ops,
6553 sizeof(struct ipv6_route_iter)))
6554 return -ENOMEM;
6555
6556 if (!proc_create_net_single("rt6_stats", 0444, net->proc_net,
6557 rt6_stats_seq_show, NULL)) {
6558 remove_proc_entry("ipv6_route", net->proc_net);
6559 return -ENOMEM;
6560 }
d189634e
TG		 6561#endif
6562 return 0;
6563}
6564
6565static void __net_exit ip6_route_net_exit_late(struct net *net)
6566{
6567#ifdef CONFIG_PROC_FS
ece31ffd
G		 6568 remove_proc_entry("ipv6_route", net->proc_net);
6569 remove_proc_entry("rt6_stats", net->proc_net);
d189634e
TG		 6570#endif
6571}
6572
cdb18761
DL		 6573static struct pernet_operations ip6_route_net_ops = {
6574 .init = ip6_route_net_init,
6575 .exit = ip6_route_net_exit,
6576};
6577
c3426b47
DM		 6578static int __net_init ipv6_inetpeer_init(struct net *net)
6579{
6580 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
6581
6582 if (!bp)
6583 return -ENOMEM;
6584 inet_peer_base_init(bp);
6585 net->ipv6.peers = bp;
6586 return 0;
6587}
6588
6589static void __net_exit ipv6_inetpeer_exit(struct net *net)
6590{
6591 struct inet_peer_base *bp = net->ipv6.peers;
6592
6593 net->ipv6.peers = NULL;
56a6b248 6594 inetpeer_invalidate_tree(bp);
c3426b47
DM		 6595 kfree(bp);
6596}
6597
2b823f72 6598static struct pernet_operations ipv6_inetpeer_ops = {
c3426b47
DM		 6599 .init = ipv6_inetpeer_init,
6600 .exit = ipv6_inetpeer_exit,
6601};
6602
d189634e
TG		 6603static struct pernet_operations ip6_route_net_late_ops = {
6604 .init = ip6_route_net_init_late,
6605 .exit = ip6_route_net_exit_late,
6606};
6607
8ed67789
DL		 6608static struct notifier_block ip6_route_dev_notifier = {
6609 .notifier_call = ip6_route_dev_notify,
242d3a49 6610 .priority = ADDRCONF_NOTIFY_PRIORITY - 10,
8ed67789
DL		 6611};
6612
2f460933
WC		 6613void __init ip6_route_init_special_entries(void)
6614{
6615 /* Registering of the loopback is done before this portion of code,
6616 * the loopback reference in rt6_info will not be taken, do it
6617 * manually for init_net */
1cf844c7 6618 init_net.ipv6.fib6_null_entry->fib6_nh->fib_nh_dev = init_net.loopback_dev;
2f460933
WC		 6619 init_net.ipv6.ip6_null_entry->dst.dev = init_net.loopback_dev;
6620 init_net.ipv6.ip6_null_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6621 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
6622 init_net.ipv6.ip6_prohibit_entry->dst.dev = init_net.loopback_dev;
6623 init_net.ipv6.ip6_prohibit_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6624 init_net.ipv6.ip6_blk_hole_entry->dst.dev = init_net.loopback_dev;
6625 init_net.ipv6.ip6_blk_hole_entry->rt6i_idev = in6_dev_get(init_net.loopback_dev);
6626 #endif
6627}
6628
138d0be3
YS		 6629#if IS_BUILTIN(CONFIG_IPV6)
6630#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6631DEFINE_BPF_ITER_FUNC(ipv6_route, struct bpf_iter_meta *meta, struct fib6_info *rt)
6632
951cf368
YS		 6633BTF_ID_LIST(btf_fib6_info_id)
6634BTF_ID(struct, fib6_info)
6635
14fc6bd6 6636static const struct bpf_iter_seq_info ipv6_route_seq_info = {
15172a46
YS		 6637 .seq_ops = &ipv6_route_seq_ops,
6638 .init_seq_private = bpf_iter_init_seq_net,
6639 .fini_seq_private = bpf_iter_fini_seq_net,
6640 .seq_priv_size = sizeof(struct ipv6_route_iter),
14fc6bd6
YS		 6641};
6642
6643static struct bpf_iter_reg ipv6_route_reg_info = {
6644 .target = "ipv6_route",
3c32cc1b
YS		 6645 .ctx_arg_info_size = 1,
6646 .ctx_arg_info = {
6647 { offsetof(struct bpf_iter__ipv6_route, rt),
6648 PTR_TO_BTF_ID_OR_NULL },
6649 },
14fc6bd6 6650 .seq_info = &ipv6_route_seq_info,
15172a46
YS		 6651};
6652
138d0be3
YS		 6653static int __init bpf_iter_register(void)
6654{
951cf368 6655 ipv6_route_reg_info.ctx_arg_info[0].btf_id = *btf_fib6_info_id;
15172a46 6656 return bpf_iter_reg_target(&ipv6_route_reg_info);
138d0be3
YS		 6657}
6658
6659static void bpf_iter_unregister(void)
6660{
ab2ee4fc 6661 bpf_iter_unreg_target(&ipv6_route_reg_info);
138d0be3
YS		 6662}
6663#endif
6664#endif
6665
433d49c3 6666int __init ip6_route_init(void)
1da177e4 6667{
433d49c3 6668 int ret;
8d0b94af 6669 int cpu;
433d49c3 6670
9a7ec3a9
DL		 6671 ret = -ENOMEM;
6672 ip6_dst_ops_template.kmem_cachep =
e5d679f3 6673 kmem_cache_create("ip6_dst_cache", sizeof(struct rt6_info), 0,
6126891c 6674 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
9a7ec3a9 6675 if (!ip6_dst_ops_template.kmem_cachep)
c19a28e1 6676 goto out;
14e50e57 6677
fc66f95c 6678 ret = dst_entries_init(&ip6_dst_blackhole_ops);
8ed67789 6679 if (ret)
bdb3289f 6680 goto out_kmem_cache;
bdb3289f 6681
c3426b47
DM		 6682 ret = register_pernet_subsys(&ipv6_inetpeer_ops);
6683 if (ret)
e8803b6c 6684 goto out_dst_entries;
2a0c451a 6685
7e52b33b
DM		 6686 ret = register_pernet_subsys(&ip6_route_net_ops);
6687 if (ret)
6688 goto out_register_inetpeer;
c3426b47 6689
5dc121e9
AE		 6690 ip6_dst_blackhole_ops.kmem_cachep = ip6_dst_ops_template.kmem_cachep;
6691
e8803b6c 6692 ret = fib6_init();
433d49c3 6693 if (ret)
8ed67789 6694 goto out_register_subsys;
433d49c3 6695
433d49c3
DL		 6696 ret = xfrm6_init();
6697 if (ret)
e8803b6c 6698 goto out_fib6_init;
c35b7e72 6699
433d49c3
DL		 6700 ret = fib6_rules_init();
6701 if (ret)
6702 goto xfrm6_init;
7e5449c2 6703
d189634e
TG		 6704 ret = register_pernet_subsys(&ip6_route_net_late_ops);
6705 if (ret)
6706 goto fib6_rules_init;
6707
16feebcf
FW		 6708 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_NEWROUTE,
6709 inet6_rtm_newroute, NULL, 0);
6710 if (ret < 0)
6711 goto out_register_late_subsys;
6712
6713 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_DELROUTE,
6714 inet6_rtm_delroute, NULL, 0);
6715 if (ret < 0)
6716 goto out_register_late_subsys;
6717
6718 ret = rtnl_register_module(THIS_MODULE, PF_INET6, RTM_GETROUTE,
6719 inet6_rtm_getroute, NULL,
6720 RTNL_FLAG_DOIT_UNLOCKED);
6721 if (ret < 0)
d189634e 6722 goto out_register_late_subsys;
c127ea2c 6723
8ed67789 6724 ret = register_netdevice_notifier(&ip6_route_dev_notifier);
cdb18761 6725 if (ret)
d189634e 6726 goto out_register_late_subsys;
8ed67789 6727
138d0be3
YS		 6728#if IS_BUILTIN(CONFIG_IPV6)
6729#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6730 ret = bpf_iter_register();
6731 if (ret)
6732 goto out_register_late_subsys;
6733#endif
6734#endif
6735
8d0b94af
MKL		 6736 for_each_possible_cpu(cpu) {
6737 struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
6738
6739 INIT_LIST_HEAD(&ul->head);
ba55ef81 6740 INIT_LIST_HEAD(&ul->quarantine);
8d0b94af
MKL		 6741 spin_lock_init(&ul->lock);
6742 }
6743
433d49c3
DL		 6744out:
6745 return ret;
6746
d189634e 6747out_register_late_subsys:
16feebcf 6748 rtnl_unregister_all(PF_INET6);
d189634e 6749 unregister_pernet_subsys(&ip6_route_net_late_ops);
433d49c3 6750fib6_rules_init:
433d49c3
DL		 6751 fib6_rules_cleanup();
6752xfrm6_init:
433d49c3 6753 xfrm6_fini();
2a0c451a
TG		 6754out_fib6_init:
6755 fib6_gc_cleanup();
8ed67789
DL		 6756out_register_subsys:
6757 unregister_pernet_subsys(&ip6_route_net_ops);
7e52b33b
DM		 6758out_register_inetpeer:
6759 unregister_pernet_subsys(&ipv6_inetpeer_ops);
fc66f95c
ED		 6760out_dst_entries:
6761 dst_entries_destroy(&ip6_dst_blackhole_ops);
433d49c3 6762out_kmem_cache:
f2fc6a54 6763 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
433d49c3 6764 goto out;
1da177e4
LT		 6765}
6766
6767void ip6_route_cleanup(void)
6768{
138d0be3
YS		 6769#if IS_BUILTIN(CONFIG_IPV6)
6770#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
6771 bpf_iter_unregister();
6772#endif
6773#endif
8ed67789 6774 unregister_netdevice_notifier(&ip6_route_dev_notifier);
d189634e 6775 unregister_pernet_subsys(&ip6_route_net_late_ops);
101367c2 6776 fib6_rules_cleanup();
1da177e4 6777 xfrm6_fini();
1da177e4 6778 fib6_gc_cleanup();
c3426b47 6779 unregister_pernet_subsys(&ipv6_inetpeer_ops);
8ed67789 6780 unregister_pernet_subsys(&ip6_route_net_ops);
41bb78b4 6781 dst_entries_destroy(&ip6_dst_blackhole_ops);
f2fc6a54 6782 kmem_cache_destroy(ip6_dst_ops_template.kmem_cachep);
1da177e4 6783}
Linux 6.x block layer and io_uring tree(s)