Merge tag 'pm-6.16-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
[linux-2.6-block.git] / net / ipv6 / ip6_fib.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4 2/*
1ab1457c 3 * Linux INET6 implementation
1da177e4
LT
4 * Forwarding Information Database
5 *
6 * Authors:
1ab1457c 7 * Pedro Roque <roque@di.fc.ul.pt>
1da177e4 8 *
8db46f1d
WY
9 * Changes:
10 * Yuji SEKIYA @USAGI: Support default route on router node;
11 * remove ip6_null_entry from the top of
12 * routing table.
13 * Ville Nuorvala: Fixed routing subtrees.
1da177e4 14 */
f3213831
JP
15
16#define pr_fmt(fmt) "IPv6: " fmt
17
b6459415 18#include <linux/bpf.h>
1da177e4
LT
19#include <linux/errno.h>
20#include <linux/types.h>
21#include <linux/net.h>
22#include <linux/route.h>
23#include <linux/netdevice.h>
24#include <linux/in6.h>
25#include <linux/init.h>
c71099ac 26#include <linux/list.h>
5a0e3ad6 27#include <linux/slab.h>
1da177e4 28
cc5f0eb2 29#include <net/ip.h>
1da177e4
LT
30#include <net/ipv6.h>
31#include <net/ndisc.h>
32#include <net/addrconf.h>
19e42e45 33#include <net/lwtunnel.h>
df77fe4d 34#include <net/fib_notifier.h>
1da177e4 35
ed13923f 36#include <net/ip_fib.h>
1da177e4
LT
37#include <net/ip6_fib.h>
38#include <net/ip6_route.h>
39
437de07c 40static struct kmem_cache *fib6_node_kmem __read_mostly;
1da177e4 41
94b2cfe0
HFS
42struct fib6_cleaner {
43 struct fib6_walker w;
ec7d43c2 44 struct net *net;
8d1c802b 45 int (*func)(struct fib6_info *, void *arg);
327571cb 46 int sernum;
1da177e4 47 void *arg;
7c6bb7d2 48 bool skip_notify;
1da177e4
LT
49};
50
1da177e4
LT
51#ifdef CONFIG_IPV6_SUBTREES
52#define FWS_INIT FWS_S
1da177e4
LT
53#else
54#define FWS_INIT FWS_L
1da177e4
LT
55#endif
56
8d1c802b 57static struct fib6_info *fib6_find_prefix(struct net *net,
66f5d6ce
WW
58 struct fib6_table *table,
59 struct fib6_node *fn);
60static struct fib6_node *fib6_repair_tree(struct net *net,
61 struct fib6_table *table,
62 struct fib6_node *fn);
9a03cd8f 63static int fib6_walk(struct net *net, struct fib6_walker *w);
94b2cfe0 64static int fib6_walk_continue(struct fib6_walker *w);
1da177e4
LT
65
66/*
67 * A routing update causes an increase of the serial number on the
68 * affected subtree. This allows for cached routes to be asynchronously
69 * tested when modifications are made to the destination cache as a
70 * result of redirects, path MTU changes, etc.
71 */
72
86cb30ec 73static void fib6_gc_timer_cb(struct timer_list *t);
5b7c931d 74
9a03cd8f
MK
75#define FOR_WALKERS(net, w) \
76 list_for_each_entry(w, &(net)->ipv6.fib6_walkers, lh)
1da177e4 77
9a03cd8f 78static void fib6_walker_link(struct net *net, struct fib6_walker *w)
90d41122 79{
9a03cd8f
MK
80 write_lock_bh(&net->ipv6.fib6_walker_lock);
81 list_add(&w->lh, &net->ipv6.fib6_walkers);
82 write_unlock_bh(&net->ipv6.fib6_walker_lock);
90d41122
AB
83}
84
9a03cd8f 85static void fib6_walker_unlink(struct net *net, struct fib6_walker *w)
90d41122 86{
9a03cd8f 87 write_lock_bh(&net->ipv6.fib6_walker_lock);
bbef49da 88 list_del(&w->lh);
9a03cd8f 89 write_unlock_bh(&net->ipv6.fib6_walker_lock);
90d41122 90}
94b2cfe0 91
812918c4 92static int fib6_new_sernum(struct net *net)
1da177e4 93{
30189806 94 int new, old = atomic_read(&net->ipv6.fib6_sernum);
42b18706
HFS
95
96 do {
42b18706 97 new = old < INT_MAX ? old + 1 : 1;
30189806
ED
98 } while (!atomic_try_cmpxchg(&net->ipv6.fib6_sernum, &old, new));
99
42b18706 100 return new;
1da177e4
LT
101}
102
327571cb
HFS
103enum {
104 FIB6_NO_SERNUM_CHANGE = 0,
105};
106
93c2fb25 107void fib6_update_sernum(struct net *net, struct fib6_info *f6i)
180ca444 108{
180ca444
WW
109 struct fib6_node *fn;
110
93c2fb25
DA
111 fn = rcu_dereference_protected(f6i->fib6_node,
112 lockdep_is_held(&f6i->fib6_table->tb6_lock));
180ca444 113 if (fn)
aafc2e32 114 WRITE_ONCE(fn->fn_sernum, fib6_new_sernum(net));
180ca444
WW
115}
116
1da177e4
LT
117/*
118 * Auxiliary address test functions for the radix tree.
119 *
1ab1457c 120 * These assume a 32bit processor (although it will work on
1da177e4
LT
121 * 64bit processors)
122 */
123
124/*
125 * test bit
126 */
02cdce53
YH
127#if defined(__LITTLE_ENDIAN)
128# define BITOP_BE32_SWIZZLE (0x1F & ~7)
129#else
130# define BITOP_BE32_SWIZZLE 0
131#endif
1da177e4 132
94b2cfe0 133static __be32 addr_bit_set(const void *token, int fn_bit)
1da177e4 134{
b71d1d42 135 const __be32 *addr = token;
02cdce53
YH
136 /*
137 * Here,
8db46f1d 138 * 1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)
02cdce53
YH
139 * is optimized version of
140 * htonl(1 << ((~fn_bit)&0x1F))
141 * See include/asm-generic/bitops/le.h.
142 */
0eae88f3
ED
143 return (__force __be32)(1 << ((~fn_bit ^ BITOP_BE32_SWIZZLE) & 0x1f)) &
144 addr[fn_bit >> 5];
1da177e4
LT
145}
146
1cf844c7 147struct fib6_info *fib6_info_alloc(gfp_t gfp_flags, bool with_fib6_nh)
a64efe14 148{
8d1c802b 149 struct fib6_info *f6i;
1cf844c7 150 size_t sz = sizeof(*f6i);
a64efe14 151
1cf844c7
DA
152 if (with_fib6_nh)
153 sz += sizeof(struct fib6_nh);
154
155 f6i = kzalloc(sz, gfp_flags);
a64efe14
DA
156 if (!f6i)
157 return NULL;
158
f88d8ea6 159 /* fib6_siblings is a union with nh_list, so this initializes both */
93c2fb25 160 INIT_LIST_HEAD(&f6i->fib6_siblings);
f05713e0 161 refcount_set(&f6i->fib6_ref, 1);
a64efe14 162
5eb902b8
KFL
163 INIT_HLIST_NODE(&f6i->gc_link);
164
a64efe14
DA
165 return f6i;
166}
167
9b0a8da8 168void fib6_info_destroy_rcu(struct rcu_head *head)
a64efe14 169{
9b0a8da8 170 struct fib6_info *f6i = container_of(head, struct fib6_info, rcu);
a64efe14 171
93c2fb25 172 WARN_ON(f6i->fib6_node);
a64efe14 173
f88d8ea6
DA
174 if (f6i->nh)
175 nexthop_put(f6i->nh);
176 else
177 fib6_nh_release(f6i->fib6_nh);
178
cc5f0eb2 179 ip_fib_metrics_put(f6i->fib6_metrics);
a64efe14
DA
180 kfree(f6i);
181}
9b0a8da8 182EXPORT_SYMBOL_GPL(fib6_info_destroy_rcu);
a64efe14 183
81eb8447 184static struct fib6_node *node_alloc(struct net *net)
1da177e4
LT
185{
186 struct fib6_node *fn;
187
c3762229 188 fn = kmem_cache_zalloc(fib6_node_kmem, GFP_ATOMIC);
81eb8447
WW
189 if (fn)
190 net->ipv6.rt6_stats->fib_nodes++;
1da177e4
LT
191
192 return fn;
193}
194
81eb8447 195static void node_free_immediate(struct net *net, struct fib6_node *fn)
c5cff856
WW
196{
197 kmem_cache_free(fib6_node_kmem, fn);
81eb8447 198 net->ipv6.rt6_stats->fib_nodes--;
c5cff856
WW
199}
200
81eb8447 201static void node_free(struct net *net, struct fib6_node *fn)
c5cff856 202{
85e48bcf 203 kfree_rcu(fn, rcu);
81eb8447 204 net->ipv6.rt6_stats->fib_nodes--;
c5cff856
WW
205}
206
ba1cc08d
SD
207static void fib6_free_table(struct fib6_table *table)
208{
209 inetpeer_invalidate_tree(&table->tb6_peers);
210 kfree(table);
211}
212
58f09b78 213static void fib6_link_table(struct net *net, struct fib6_table *tb)
1b43af54
PM
214{
215 unsigned int h;
216
375216ad
TG
217 /*
218 * Initialize table lock at a single place to give lockdep a key,
219 * tables aren't visible prior to being linked to the list.
220 */
66f5d6ce 221 spin_lock_init(&tb->tb6_lock);
a33bc5c1 222 h = tb->tb6_id & (FIB6_TABLE_HASHSZ - 1);
1b43af54
PM
223
224 /*
225 * No protection necessary, this is the only list mutatation
226 * operation, tables never disappear once they exist.
227 */
58f09b78 228 hlist_add_head_rcu(&tb->tb6_hlist, &net->ipv6.fib_table_hash[h]);
1b43af54 229}
c71099ac 230
1b43af54 231#ifdef CONFIG_IPV6_MULTIPLE_TABLES
e0b85590 232
8ed67789 233static struct fib6_table *fib6_alloc_table(struct net *net, u32 id)
c71099ac
TG
234{
235 struct fib6_table *table;
236
237 table = kzalloc(sizeof(*table), GFP_ATOMIC);
507c9b1e 238 if (table) {
c71099ac 239 table->tb6_id = id;
66f5d6ce 240 rcu_assign_pointer(table->tb6_root.leaf,
421842ed 241 net->ipv6.fib6_null_entry);
c71099ac 242 table->tb6_root.fn_flags = RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
8e773277 243 inet_peer_base_init(&table->tb6_peers);
5eb902b8 244 INIT_HLIST_HEAD(&table->tb6_gc_hlist);
c71099ac
TG
245 }
246
247 return table;
248}
249
58f09b78 250struct fib6_table *fib6_new_table(struct net *net, u32 id)
c71099ac 251{
834d9784 252 struct fib6_table *tb, *new_tb;
c71099ac
TG
253
254 if (id == 0)
255 id = RT6_TABLE_MAIN;
834d9784 256
58f09b78 257 tb = fib6_get_table(net, id);
c71099ac
TG
258 if (tb)
259 return tb;
260
834d9784
KI
261 new_tb = fib6_alloc_table(net, id);
262 if (!new_tb)
263 return NULL;
264
265 spin_lock_bh(&net->ipv6.fib_table_hash_lock);
266
267 tb = fib6_get_table(net, id);
268 if (unlikely(tb)) {
269 spin_unlock_bh(&net->ipv6.fib_table_hash_lock);
270 kfree(new_tb);
271 return tb;
272 }
c71099ac 273
834d9784
KI
274 fib6_link_table(net, new_tb);
275
276 spin_unlock_bh(&net->ipv6.fib_table_hash_lock);
277
278 return new_tb;
c71099ac 279}
b3b4663c 280EXPORT_SYMBOL_GPL(fib6_new_table);
c71099ac 281
58f09b78 282struct fib6_table *fib6_get_table(struct net *net, u32 id)
c71099ac 283{
58f09b78 284 struct hlist_head *head;
f1a8d107 285 struct fib6_table *tb;
c71099ac 286
f1a8d107 287 if (!id)
c71099ac 288 id = RT6_TABLE_MAIN;
f1a8d107
KI
289
290 head = &net->ipv6.fib_table_hash[id & (FIB6_TABLE_HASHSZ - 1)];
291
292 /* See comment in fib6_link_table(). RCU is not required,
293 * but rcu_dereference_raw() is used to avoid data-race.
294 */
295 hlist_for_each_entry_rcu(tb, head, tb6_hlist, true)
296 if (tb->tb6_id == id)
c71099ac 297 return tb;
c71099ac
TG
298
299 return NULL;
300}
c4850687 301EXPORT_SYMBOL_GPL(fib6_get_table);
c71099ac 302
2c8c1e72 303static void __net_init fib6_tables_init(struct net *net)
c71099ac 304{
58f09b78
DL
305 fib6_link_table(net, net->ipv6.fib6_main_tbl);
306 fib6_link_table(net, net->ipv6.fib6_local_tbl);
c71099ac 307}
c71099ac
TG
308#else
309
58f09b78 310struct fib6_table *fib6_new_table(struct net *net, u32 id)
c71099ac 311{
58f09b78 312 return fib6_get_table(net, id);
c71099ac
TG
313}
314
58f09b78 315struct fib6_table *fib6_get_table(struct net *net, u32 id)
c71099ac 316{
58f09b78 317 return net->ipv6.fib6_main_tbl;
c71099ac
TG
318}
319
4c9483b2 320struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,
b75cc8f9 321 const struct sk_buff *skb,
58f09b78 322 int flags, pol_lookup_t lookup)
c71099ac 323{
ab997ad4 324 struct rt6_info *rt;
325
55cced4f
BV
326 rt = pol_lookup_func(lookup,
327 net, net->ipv6.fib6_main_tbl, fl6, skb, flags);
07f61557 328 if (rt->dst.error == -EAGAIN) {
d64a1f57 329 ip6_rt_put_flags(rt, flags);
ab997ad4 330 rt = net->ipv6.ip6_null_entry;
7b09c2d0 331 if (!(flags & RT6_LOOKUP_F_DST_NOREF))
d64a1f57 332 dst_hold(&rt->dst);
ab997ad4 333 }
334
335 return &rt->dst;
c71099ac
TG
336}
337
138118ec 338/* called with rcu lock held; no reference taken on fib6_info */
effda4dd
DA
339int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6,
340 struct fib6_result *res, int flags)
138118ec 341{
effda4dd
DA
342 return fib6_table_lookup(net, net->ipv6.fib6_main_tbl, oif, fl6,
343 res, flags);
138118ec
DA
344}
345
2c8c1e72 346static void __net_init fib6_tables_init(struct net *net)
c71099ac 347{
58f09b78 348 fib6_link_table(net, net->ipv6.fib6_main_tbl);
c71099ac
TG
349}
350
351#endif
352
e60ea454 353unsigned int fib6_tables_seq_read(const struct net *net)
e1ee0a5b
IS
354{
355 unsigned int h, fib_seq = 0;
356
357 rcu_read_lock();
358 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
e60ea454
ED
359 const struct hlist_head *head = &net->ipv6.fib_table_hash[h];
360 const struct fib6_table *tb;
e1ee0a5b 361
66f5d6ce 362 hlist_for_each_entry_rcu(tb, head, tb6_hlist)
e60ea454 363 fib_seq += READ_ONCE(tb->fib_seq);
e1ee0a5b
IS
364 }
365 rcu_read_unlock();
366
367 return fib_seq;
368}
369
7c550daf 370static int call_fib6_entry_notifier(struct notifier_block *nb,
e1ee0a5b 371 enum fib_event_type event_type,
b7a59557
JP
372 struct fib6_info *rt,
373 struct netlink_ext_ack *extack)
e1ee0a5b
IS
374{
375 struct fib6_entry_notifier_info info = {
b7a59557 376 .info.extack = extack,
e1ee0a5b
IS
377 .rt = rt,
378 };
379
7c550daf 380 return call_fib6_notifier(nb, event_type, &info.info);
e1ee0a5b
IS
381}
382
9c6ecd3c
IS
383static int call_fib6_multipath_entry_notifier(struct notifier_block *nb,
384 enum fib_event_type event_type,
385 struct fib6_info *rt,
386 unsigned int nsiblings,
387 struct netlink_ext_ack *extack)
388{
389 struct fib6_entry_notifier_info info = {
390 .info.extack = extack,
391 .rt = rt,
392 .nsiblings = nsiblings,
393 };
394
395 return call_fib6_notifier(nb, event_type, &info.info);
396}
397
19a3b7ee
DA
398int call_fib6_entry_notifiers(struct net *net,
399 enum fib_event_type event_type,
400 struct fib6_info *rt,
401 struct netlink_ext_ack *extack)
df77fe4d
IS
402{
403 struct fib6_entry_notifier_info info = {
6c31e5a9 404 .info.extack = extack,
df77fe4d
IS
405 .rt = rt,
406 };
407
e60ea454 408 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
df77fe4d
IS
409 return call_fib6_notifiers(net, event_type, &info.info);
410}
411
d4b96c7b
IS
412int call_fib6_multipath_entry_notifiers(struct net *net,
413 enum fib_event_type event_type,
414 struct fib6_info *rt,
415 unsigned int nsiblings,
416 struct netlink_ext_ack *extack)
417{
418 struct fib6_entry_notifier_info info = {
419 .info.extack = extack,
420 .rt = rt,
421 .nsiblings = nsiblings,
d4b96c7b
IS
422 };
423
e60ea454 424 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
d4b96c7b
IS
425 return call_fib6_notifiers(net, event_type, &info.info);
426}
427
d2f0c9b1
IS
428int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt)
429{
430 struct fib6_entry_notifier_info info = {
431 .rt = rt,
432 .nsiblings = rt->fib6_nsiblings,
433 };
434
e60ea454 435 WRITE_ONCE(rt->fib6_table->fib_seq, rt->fib6_table->fib_seq + 1);
caafb250 436 return call_fib6_notifiers(net, FIB_EVENT_ENTRY_REPLACE, &info.info);
d2f0c9b1
IS
437}
438
e1ee0a5b
IS
439struct fib6_dump_arg {
440 struct net *net;
441 struct notifier_block *nb;
b7a59557 442 struct netlink_ext_ack *extack;
e1ee0a5b
IS
443};
444
55c894f7 445static int fib6_rt_dump(struct fib6_info *rt, struct fib6_dump_arg *arg)
e1ee0a5b 446{
caafb250 447 enum fib_event_type fib_event = FIB_EVENT_ENTRY_REPLACE;
9c6ecd3c
IS
448 int err;
449
450 if (!rt || rt == arg->net->ipv6.fib6_null_entry)
451 return 0;
452
453 if (rt->fib6_nsiblings)
454 err = call_fib6_multipath_entry_notifier(arg->nb, fib_event,
455 rt,
456 rt->fib6_nsiblings,
457 arg->extack);
458 else
459 err = call_fib6_entry_notifier(arg->nb, fib_event, rt,
460 arg->extack);
461
462 return err;
463}
464
e1ee0a5b
IS
465static int fib6_node_dump(struct fib6_walker *w)
466{
caafb250 467 int err;
9c6ecd3c 468
caafb250 469 err = fib6_rt_dump(w->leaf, w->args);
e1ee0a5b 470 w->leaf = NULL;
55c894f7 471 return err;
e1ee0a5b
IS
472}
473
55c894f7
JP
474static int fib6_table_dump(struct net *net, struct fib6_table *tb,
475 struct fib6_walker *w)
e1ee0a5b 476{
55c894f7
JP
477 int err;
478
e1ee0a5b 479 w->root = &tb->tb6_root;
66f5d6ce 480 spin_lock_bh(&tb->tb6_lock);
55c894f7 481 err = fib6_walk(net, w);
66f5d6ce 482 spin_unlock_bh(&tb->tb6_lock);
55c894f7 483 return err;
e1ee0a5b
IS
484}
485
486/* Called with rcu_read_lock() */
b7a59557
JP
487int fib6_tables_dump(struct net *net, struct notifier_block *nb,
488 struct netlink_ext_ack *extack)
e1ee0a5b
IS
489{
490 struct fib6_dump_arg arg;
491 struct fib6_walker *w;
492 unsigned int h;
55c894f7 493 int err = 0;
e1ee0a5b
IS
494
495 w = kzalloc(sizeof(*w), GFP_ATOMIC);
496 if (!w)
497 return -ENOMEM;
498
499 w->func = fib6_node_dump;
500 arg.net = net;
501 arg.nb = nb;
b7a59557 502 arg.extack = extack;
e1ee0a5b
IS
503 w->args = &arg;
504
505 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
506 struct hlist_head *head = &net->ipv6.fib_table_hash[h];
507 struct fib6_table *tb;
508
55c894f7
JP
509 hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
510 err = fib6_table_dump(net, tb, w);
6f199552 511 if (err)
55c894f7
JP
512 goto out;
513 }
e1ee0a5b
IS
514 }
515
55c894f7 516out:
e1ee0a5b
IS
517 kfree(w);
518
6f199552
VB
519 /* The tree traversal function should never return a positive value. */
520 return err > 0 ? -EINVAL : err;
e1ee0a5b
IS
521}
522
94b2cfe0 523static int fib6_dump_node(struct fib6_walker *w)
1b43af54
PM
524{
525 int res;
8d1c802b 526 struct fib6_info *rt;
1b43af54 527
66f5d6ce 528 for_each_fib6_walker_rt(w) {
1e47b483 529 res = rt6_dump_route(rt, w->args, w->skip_in_node);
bf9a8a06 530 if (res >= 0) {
1b43af54
PM
531 /* Frame is full, suspend walking */
532 w->leaf = rt;
1e47b483
SB
533
534 /* We'll restart from this node, so if some routes were
535 * already dumped, skip them next time.
536 */
537 w->skip_in_node += res;
538
1b43af54
PM
539 return 1;
540 }
1e47b483 541 w->skip_in_node = 0;
beb1afac
DA
542
543 /* Multipath routes are dumped in one route with the
544 * RTA_MULTIPATH attribute. Jump 'rt' to point to the
545 * last sibling of this route (no need to dump the
546 * sibling routes again)
547 */
93c2fb25
DA
548 if (rt->fib6_nsiblings)
549 rt = list_last_entry(&rt->fib6_siblings,
8d1c802b 550 struct fib6_info,
93c2fb25 551 fib6_siblings);
1b43af54
PM
552 }
553 w->leaf = NULL;
554 return 0;
555}
556
557static void fib6_dump_end(struct netlink_callback *cb)
558{
9a03cd8f 559 struct net *net = sock_net(cb->skb->sk);
94b2cfe0 560 struct fib6_walker *w = (void *)cb->args[2];
1b43af54
PM
561
562 if (w) {
7891cc81
HX
563 if (cb->args[4]) {
564 cb->args[4] = 0;
9a03cd8f 565 fib6_walker_unlink(net, w);
7891cc81 566 }
1b43af54
PM
567 cb->args[2] = 0;
568 kfree(w);
569 }
437de07c 570 cb->done = (void *)cb->args[3];
1b43af54
PM
571 cb->args[1] = 3;
572}
573
574static int fib6_dump_done(struct netlink_callback *cb)
575{
576 fib6_dump_end(cb);
577 return cb->done ? cb->done(cb) : 0;
578}
579
580static int fib6_dump_table(struct fib6_table *table, struct sk_buff *skb,
581 struct netlink_callback *cb)
582{
9a03cd8f 583 struct net *net = sock_net(skb->sk);
94b2cfe0 584 struct fib6_walker *w;
1b43af54
PM
585 int res;
586
587 w = (void *)cb->args[2];
588 w->root = &table->tb6_root;
589
590 if (cb->args[4] == 0) {
2bec5a36
PM
591 w->count = 0;
592 w->skip = 0;
1e47b483 593 w->skip_in_node = 0;
2bec5a36 594
66f5d6ce 595 spin_lock_bh(&table->tb6_lock);
9a03cd8f 596 res = fib6_walk(net, w);
66f5d6ce 597 spin_unlock_bh(&table->tb6_lock);
2bec5a36 598 if (res > 0) {
1b43af54 599 cb->args[4] = 1;
aafc2e32 600 cb->args[5] = READ_ONCE(w->root->fn_sernum);
2bec5a36 601 }
1b43af54 602 } else {
aafc2e32
ED
603 int sernum = READ_ONCE(w->root->fn_sernum);
604 if (cb->args[5] != sernum) {
2bec5a36 605 /* Begin at the root if the tree changed */
aafc2e32 606 cb->args[5] = sernum;
2bec5a36
PM
607 w->state = FWS_INIT;
608 w->node = w->root;
609 w->skip = w->count;
1e47b483 610 w->skip_in_node = 0;
2bec5a36
PM
611 } else
612 w->skip = 0;
613
66f5d6ce 614 spin_lock_bh(&table->tb6_lock);
1b43af54 615 res = fib6_walk_continue(w);
66f5d6ce 616 spin_unlock_bh(&table->tb6_lock);
7891cc81 617 if (res <= 0) {
9a03cd8f 618 fib6_walker_unlink(net, w);
7891cc81 619 cb->args[4] = 0;
1b43af54 620 }
1b43af54 621 }
7891cc81 622
1b43af54
PM
623 return res;
624}
625
c127ea2c 626static int inet6_dump_fib(struct sk_buff *skb, struct netlink_callback *cb)
1b43af54 627{
22e36ea9
ED
628 struct rt6_rtnl_dump_arg arg = {
629 .filter.dump_exceptions = true,
630 .filter.dump_routes = true,
5fc68320 631 .filter.rtnl_held = false,
22e36ea9 632 };
e8ba330a 633 const struct nlmsghdr *nlh = cb->nlh;
3b1e0a65 634 struct net *net = sock_net(skb->sk);
1b43af54 635 unsigned int e = 0, s_e;
5fc68320 636 struct hlist_head *head;
94b2cfe0 637 struct fib6_walker *w;
1b43af54 638 struct fib6_table *tb;
5fc68320
ED
639 unsigned int h, s_h;
640 int err = 0;
1b43af54 641
5fc68320 642 rcu_read_lock();
e8ba330a 643 if (cb->strict_check) {
effe6792 644 err = ip_valid_fib_dump_req(net, nlh, &arg.filter, cb);
e8ba330a 645 if (err < 0)
5fc68320 646 goto unlock;
13e38901
DA
647 } else if (nlmsg_len(nlh) >= sizeof(struct rtmsg)) {
648 struct rtmsg *rtm = nlmsg_data(nlh);
e8ba330a 649
ef11209d
SB
650 if (rtm->rtm_flags & RTM_F_PREFIX)
651 arg.filter.flags = RTM_F_PREFIX;
13e38901 652 }
1b43af54
PM
653
654 w = (void *)cb->args[2];
507c9b1e 655 if (!w) {
1b43af54
PM
656 /* New dump:
657 *
d21d4060 658 * 1. allocate and initialize walker.
1b43af54
PM
659 */
660 w = kzalloc(sizeof(*w), GFP_ATOMIC);
5fc68320
ED
661 if (!w) {
662 err = -ENOMEM;
663 goto unlock;
664 }
1b43af54
PM
665 w->func = fib6_dump_node;
666 cb->args[2] = (long)w;
d21d4060
KI
667
668 /* 2. hook callback destructor.
669 */
670 cb->args[3] = (long)cb->done;
671 cb->done = fib6_dump_done;
672
1b43af54
PM
673 }
674
675 arg.skb = skb;
676 arg.cb = cb;
191cd582 677 arg.net = net;
1b43af54
PM
678 w->args = &arg;
679
13e38901
DA
680 if (arg.filter.table_id) {
681 tb = fib6_get_table(net, arg.filter.table_id);
682 if (!tb) {
41b4bd98 683 if (rtnl_msg_family(cb->nlh) != PF_INET6)
5fc68320 684 goto unlock;
ae677bbb 685
13e38901 686 NL_SET_ERR_MSG_MOD(cb->extack, "FIB table does not exist");
5fc68320
ED
687 err = -ENOENT;
688 goto unlock;
13e38901
DA
689 }
690
73155879 691 if (!cb->args[0]) {
5fc68320
ED
692 err = fib6_dump_table(tb, skb, cb);
693 if (!err)
73155879
DA
694 cb->args[0] = 1;
695 }
5fc68320 696 goto unlock;
13e38901
DA
697 }
698
699 s_h = cb->args[0];
700 s_e = cb->args[1];
701
a33bc5c1 702 for (h = s_h; h < FIB6_TABLE_HASHSZ; h++, s_e = 0) {
1b43af54 703 e = 0;
58f09b78 704 head = &net->ipv6.fib_table_hash[h];
b67bfe0d 705 hlist_for_each_entry_rcu(tb, head, tb6_hlist) {
1b43af54
PM
706 if (e < s_e)
707 goto next;
5fc68320
ED
708 err = fib6_dump_table(tb, skb, cb);
709 if (err != 0)
710 goto out;
1b43af54
PM
711next:
712 e++;
713 }
714 }
5fc68320 715out:
1b43af54
PM
716 cb->args[1] = e;
717 cb->args[0] = h;
5fc68320
ED
718
719unlock:
720 rcu_read_unlock();
721 if (err <= 0)
1b43af54 722 fib6_dump_end(cb);
5fc68320 723 return err;
1b43af54 724}
1da177e4 725
8d1c802b 726void fib6_metric_set(struct fib6_info *f6i, int metric, u32 val)
d4ead6b3
DA
727{
728 if (!f6i)
729 return;
730
731 if (f6i->fib6_metrics == &dst_default_metrics) {
732 struct dst_metrics *p = kzalloc(sizeof(*p), GFP_ATOMIC);
733
734 if (!p)
735 return;
736
737 refcount_set(&p->refcnt, 1);
738 f6i->fib6_metrics = p;
739 }
740
741 f6i->fib6_metrics->metrics[metric - 1] = val;
742}
743
1da177e4
LT
744/*
745 * Routing Table
746 *
747 * return the appropriate node for a routing tree "add" operation
748 * by either creating and inserting or by returning an existing
749 * node.
750 */
751
81eb8447
WW
752static struct fib6_node *fib6_add_1(struct net *net,
753 struct fib6_table *table,
66f5d6ce
WW
754 struct fib6_node *root,
755 struct in6_addr *addr, int plen,
756 int offset, int allow_create,
757 int replace_required,
758 struct netlink_ext_ack *extack)
1da177e4
LT
759{
760 struct fib6_node *fn, *in, *ln;
761 struct fib6_node *pn = NULL;
762 struct rt6key *key;
763 int bit;
1ab1457c 764 __be32 dir = 0;
1da177e4 765
1da177e4
LT
766 /* insert node in tree */
767
768 fn = root;
769
770 do {
8d1c802b 771 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
66f5d6ce
WW
772 lockdep_is_held(&table->tb6_lock));
773 key = (struct rt6key *)((u8 *)leaf + offset);
1da177e4
LT
774
775 /*
776 * Prefix match
777 */
778 if (plen < fn->fn_bit ||
4a287eba 779 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit)) {
14df015b
MV
780 if (!allow_create) {
781 if (replace_required) {
d5d531cb
DA
782 NL_SET_ERR_MSG(extack,
783 "Can not replace route - no match found");
f3213831 784 pr_warn("Can't replace route, no match found\n");
14df015b
MV
785 return ERR_PTR(-ENOENT);
786 }
f3213831 787 pr_warn("NLM_F_CREATE should be set when creating new route\n");
14df015b 788 }
1da177e4 789 goto insert_above;
4a287eba 790 }
1ab1457c 791
1da177e4
LT
792 /*
793 * Exact match ?
794 */
1ab1457c 795
1da177e4
LT
796 if (plen == fn->fn_bit) {
797 /* clean up an intermediate node */
507c9b1e 798 if (!(fn->fn_flags & RTN_RTINFO)) {
66f5d6ce 799 RCU_INIT_POINTER(fn->leaf, NULL);
93531c67 800 fib6_info_release(leaf);
4512c43e
WW
801 /* remove null_entry in the root node */
802 } else if (fn->fn_flags & RTN_TL_ROOT &&
803 rcu_access_pointer(fn->leaf) ==
421842ed 804 net->ipv6.fib6_null_entry) {
4512c43e 805 RCU_INIT_POINTER(fn->leaf, NULL);
1da177e4 806 }
1ab1457c 807
1da177e4
LT
808 return fn;
809 }
810
811 /*
812 * We have more bits to go
813 */
1ab1457c 814
1da177e4 815 /* Try to walk down on tree. */
1da177e4
LT
816 dir = addr_bit_set(addr, fn->fn_bit);
817 pn = fn;
66f5d6ce
WW
818 fn = dir ?
819 rcu_dereference_protected(fn->right,
820 lockdep_is_held(&table->tb6_lock)) :
821 rcu_dereference_protected(fn->left,
822 lockdep_is_held(&table->tb6_lock));
1da177e4
LT
823 } while (fn);
824
14df015b 825 if (!allow_create) {
4a287eba
MV
826 /* We should not create new node because
827 * NLM_F_REPLACE was specified without NLM_F_CREATE
828 * I assume it is safe to require NLM_F_CREATE when
829 * REPLACE flag is used! Later we may want to remove the
830 * check for replace_required, because according
831 * to netlink specification, NLM_F_CREATE
832 * MUST be specified if new route is created.
833 * That would keep IPv6 consistent with IPv4
834 */
14df015b 835 if (replace_required) {
d5d531cb
DA
836 NL_SET_ERR_MSG(extack,
837 "Can not replace route - no match found");
f3213831 838 pr_warn("Can't replace route, no match found\n");
14df015b
MV
839 return ERR_PTR(-ENOENT);
840 }
f3213831 841 pr_warn("NLM_F_CREATE should be set when creating new route\n");
4a287eba 842 }
1da177e4
LT
843 /*
844 * We walked to the bottom of tree.
845 * Create new leaf node without children.
846 */
847
81eb8447 848 ln = node_alloc(net);
1da177e4 849
507c9b1e 850 if (!ln)
188c517a 851 return ERR_PTR(-ENOMEM);
1da177e4 852 ln->fn_bit = plen;
66f5d6ce 853 RCU_INIT_POINTER(ln->parent, pn);
1da177e4
LT
854
855 if (dir)
66f5d6ce 856 rcu_assign_pointer(pn->right, ln);
1da177e4 857 else
66f5d6ce 858 rcu_assign_pointer(pn->left, ln);
1da177e4
LT
859
860 return ln;
861
862
863insert_above:
864 /*
1ab1457c 865 * split since we don't have a common prefix anymore or
1da177e4
LT
866 * we have a less significant route.
867 * we've to insert an intermediate node on the list
868 * this new node will point to the one we need to create
869 * and the current
870 */
871
66f5d6ce
WW
872 pn = rcu_dereference_protected(fn->parent,
873 lockdep_is_held(&table->tb6_lock));
1da177e4
LT
874
875 /* find 1st bit in difference between the 2 addrs.
876
971f359d 877 See comment in __ipv6_addr_diff: bit may be an invalid value,
1da177e4
LT
878 but if it is >= plen, the value is ignored in any case.
879 */
1ab1457c 880
9225b230 881 bit = __ipv6_addr_diff(addr, &key->addr, sizeof(*addr));
1da177e4 882
1ab1457c
YH
883 /*
884 * (intermediate)[in]
1da177e4
LT
885 * / \
886 * (new leaf node)[ln] (old node)[fn]
887 */
888 if (plen > bit) {
81eb8447
WW
889 in = node_alloc(net);
890 ln = node_alloc(net);
1ab1457c 891
507c9b1e 892 if (!in || !ln) {
1da177e4 893 if (in)
81eb8447 894 node_free_immediate(net, in);
1da177e4 895 if (ln)
81eb8447 896 node_free_immediate(net, ln);
188c517a 897 return ERR_PTR(-ENOMEM);
1da177e4
LT
898 }
899
1ab1457c
YH
900 /*
901 * new intermediate node.
1da177e4
LT
902 * RTN_RTINFO will
903 * be off since that an address that chooses one of
904 * the branches would not match less specific routes
905 * in the other branch
906 */
907
908 in->fn_bit = bit;
909
66f5d6ce 910 RCU_INIT_POINTER(in->parent, pn);
1da177e4 911 in->leaf = fn->leaf;
5ea71528
ED
912 fib6_info_hold(rcu_dereference_protected(in->leaf,
913 lockdep_is_held(&table->tb6_lock)));
1da177e4 914
1da177e4
LT
915 /* update parent pointer */
916 if (dir)
66f5d6ce 917 rcu_assign_pointer(pn->right, in);
1da177e4 918 else
66f5d6ce 919 rcu_assign_pointer(pn->left, in);
1da177e4
LT
920
921 ln->fn_bit = plen;
922
66f5d6ce
WW
923 RCU_INIT_POINTER(ln->parent, in);
924 rcu_assign_pointer(fn->parent, in);
1da177e4 925
1da177e4 926 if (addr_bit_set(addr, bit)) {
66f5d6ce
WW
927 rcu_assign_pointer(in->right, ln);
928 rcu_assign_pointer(in->left, fn);
1da177e4 929 } else {
66f5d6ce
WW
930 rcu_assign_pointer(in->left, ln);
931 rcu_assign_pointer(in->right, fn);
1da177e4
LT
932 }
933 } else { /* plen <= bit */
934
1ab1457c 935 /*
1da177e4
LT
936 * (new leaf node)[ln]
937 * / \
938 * (old node)[fn] NULL
939 */
940
81eb8447 941 ln = node_alloc(net);
1da177e4 942
507c9b1e 943 if (!ln)
188c517a 944 return ERR_PTR(-ENOMEM);
1da177e4
LT
945
946 ln->fn_bit = plen;
947
66f5d6ce 948 RCU_INIT_POINTER(ln->parent, pn);
1da177e4
LT
949
950 if (addr_bit_set(&key->addr, plen))
66f5d6ce 951 RCU_INIT_POINTER(ln->right, fn);
1da177e4 952 else
66f5d6ce
WW
953 RCU_INIT_POINTER(ln->left, fn);
954
955 rcu_assign_pointer(fn->parent, ln);
1da177e4 956
66f5d6ce
WW
957 if (dir)
958 rcu_assign_pointer(pn->right, ln);
959 else
960 rcu_assign_pointer(pn->left, ln);
1da177e4
LT
961 }
962 return ln;
963}
964
7d88d8b5
DA
965static void __fib6_drop_pcpu_from(struct fib6_nh *fib6_nh,
966 const struct fib6_info *match,
967 const struct fib6_table *table)
e715b6d3 968{
5bcaa41b 969 int cpu;
e715b6d3 970
f40b6ae2
DA
971 if (!fib6_nh->rt6i_pcpu)
972 return;
973
b01e1c03 974 rcu_read_lock();
5bcaa41b
DA
975 /* release the reference to this fib entry from
976 * all of its cached pcpu routes
977 */
978 for_each_possible_cpu(cpu) {
979 struct rt6_info **ppcpu_rt;
980 struct rt6_info *pcpu_rt;
e715b6d3 981
f40b6ae2 982 ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu);
b01e1c03
ED
983
984 /* Paired with xchg() in rt6_get_pcpu_route() */
985 pcpu_rt = READ_ONCE(*ppcpu_rt);
7d88d8b5
DA
986
987 /* only dropping the 'from' reference if the cached route
988 * is using 'match'. The cached pcpu_rt->from only changes
989 * from a fib6_info to NULL (ip6_dst_destroy); it can never
990 * change from one fib6_info reference to another
991 */
992 if (pcpu_rt && rcu_access_pointer(pcpu_rt->from) == match) {
a68886a6 993 struct fib6_info *from;
e715b6d3 994
b4cb4a13 995 from = unrcu_pointer(xchg(&pcpu_rt->from, NULL));
a68886a6 996 fib6_info_release(from);
5bcaa41b 997 }
e5fd387a 998 }
b01e1c03 999 rcu_read_unlock();
e5fd387a
MK
1000}
1001
2ab75bfb
DA
1002struct fib6_nh_pcpu_arg {
1003 struct fib6_info *from;
1004 const struct fib6_table *table;
1005};
1006
1007static int fib6_nh_drop_pcpu_from(struct fib6_nh *nh, void *_arg)
1008{
1009 struct fib6_nh_pcpu_arg *arg = _arg;
1010
1011 __fib6_drop_pcpu_from(nh, arg->from, arg->table);
1012 return 0;
1013}
1014
7d88d8b5
DA
1015static void fib6_drop_pcpu_from(struct fib6_info *f6i,
1016 const struct fib6_table *table)
1017{
7d88d8b5
DA
1018 /* Make sure rt6_make_pcpu_route() wont add other percpu routes
1019 * while we are cleaning them here.
1020 */
1021 f6i->fib6_destroying = 1;
1022 mb(); /* paired with the cmpxchg() in rt6_make_pcpu_route() */
1023
2ab75bfb
DA
1024 if (f6i->nh) {
1025 struct fib6_nh_pcpu_arg arg = {
1026 .from = f6i,
1027 .table = table
1028 };
1029
8e5f1bb8
KI
1030 rcu_read_lock();
1031 nexthop_for_each_fib6_nh(f6i->nh, fib6_nh_drop_pcpu_from, &arg);
1032 rcu_read_unlock();
2ab75bfb
DA
1033 } else {
1034 struct fib6_nh *fib6_nh;
1035
1036 fib6_nh = f6i->fib6_nh;
1037 __fib6_drop_pcpu_from(fib6_nh, f6i, table);
1038 }
7d88d8b5
DA
1039}
1040
8d1c802b 1041static void fib6_purge_rt(struct fib6_info *rt, struct fib6_node *fn,
6e9e16e6
HFS
1042 struct net *net)
1043{
93c2fb25 1044 struct fib6_table *table = rt->fib6_table;
66f5d6ce 1045
d8f5c296
ST
1046 /* Flush all cached dst in exception table */
1047 rt6_flush_exceptions(rt);
f40b6ae2 1048 fib6_drop_pcpu_from(rt, table);
61fb0d01 1049
081efd18
KI
1050 if (rt->nh) {
1051 spin_lock(&rt->nh->lock);
1052
1053 if (!list_empty(&rt->nh_list))
1054 list_del_init(&rt->nh_list);
1055
1056 spin_unlock(&rt->nh->lock);
1057 }
f88d8ea6 1058
f05713e0 1059 if (refcount_read(&rt->fib6_ref) != 1) {
6e9e16e6
HFS
1060 /* This route is used as dummy address holder in some split
1061 * nodes. It is not leaked, but it still holds other resources,
1062 * which must be released in time. So, scan ascendant nodes
1063 * and replace dummy references to this route with references
1064 * to still alive ones.
1065 */
1066 while (fn) {
8d1c802b 1067 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
66f5d6ce 1068 lockdep_is_held(&table->tb6_lock));
8d1c802b 1069 struct fib6_info *new_leaf;
66f5d6ce
WW
1070 if (!(fn->fn_flags & RTN_RTINFO) && leaf == rt) {
1071 new_leaf = fib6_find_prefix(net, table, fn);
5ea71528 1072 fib6_info_hold(new_leaf);
93531c67 1073
66f5d6ce 1074 rcu_assign_pointer(fn->leaf, new_leaf);
93531c67 1075 fib6_info_release(rt);
6e9e16e6 1076 }
66f5d6ce
WW
1077 fn = rcu_dereference_protected(fn->parent,
1078 lockdep_is_held(&table->tb6_lock));
6e9e16e6 1079 }
6e9e16e6 1080 }
5eb902b8
KFL
1081
1082 fib6_clean_expires(rt);
1083 fib6_remove_gc_list(rt);
6e9e16e6
HFS
1084}
1085
1da177e4
LT
1086/*
1087 * Insert routing information in a node.
1088 */
1089
8d1c802b 1090static int fib6_add_rt2node(struct fib6_node *fn, struct fib6_info *rt,
accb46b5
KI
1091 struct nl_info *info, struct netlink_ext_ack *extack,
1092 struct list_head *purge_list)
1da177e4 1093{
8d1c802b 1094 struct fib6_info *leaf = rcu_dereference_protected(fn->leaf,
93c2fb25 1095 lockdep_is_held(&rt->fib6_table->tb6_lock));
33bd5ac5 1096 struct fib6_info *iter = NULL;
8d1c802b 1097 struct fib6_info __rcu **ins;
33bd5ac5 1098 struct fib6_info __rcu **fallback_ins = NULL;
507c9b1e
DM
1099 int replace = (info->nlh &&
1100 (info->nlh->nlmsg_flags & NLM_F_REPLACE));
1101 int add = (!info->nlh ||
1102 (info->nlh->nlmsg_flags & NLM_F_CREATE));
4a287eba 1103 int found = 0;
33bd5ac5 1104 bool rt_can_ecmp = rt6_qualify_for_ecmp(rt);
c10c4279 1105 bool notify_sibling_rt = false;
73483c12 1106 u16 nlflags = NLM_F_EXCL;
e5fd387a 1107 int err;
1da177e4 1108
33bd5ac5 1109 if (info->nlh && (info->nlh->nlmsg_flags & NLM_F_APPEND))
1f5e29ce
DA
1110 nlflags |= NLM_F_APPEND;
1111
1da177e4
LT
1112 ins = &fn->leaf;
1113
66f5d6ce 1114 for (iter = leaf; iter;
8fb11a9a 1115 iter = rcu_dereference_protected(iter->fib6_next,
93c2fb25 1116 lockdep_is_held(&rt->fib6_table->tb6_lock))) {
1da177e4
LT
1117 /*
1118 * Search for duplicates
1119 */
1120
93c2fb25 1121 if (iter->fib6_metric == rt->fib6_metric) {
1da177e4
LT
1122 /*
1123 * Same priority level
1124 */
507c9b1e
DM
1125 if (info->nlh &&
1126 (info->nlh->nlmsg_flags & NLM_F_EXCL))
4a287eba 1127 return -EEXIST;
73483c12
GN
1128
1129 nlflags &= ~NLM_F_EXCL;
4a287eba 1130 if (replace) {
33bd5ac5
DA
1131 if (rt_can_ecmp == rt6_qualify_for_ecmp(iter)) {
1132 found++;
1133 break;
1134 }
e404b8c7 1135 fallback_ins = fallback_ins ?: ins;
33bd5ac5 1136 goto next_iter;
4a287eba 1137 }
1da177e4 1138
f06b7549 1139 if (rt6_duplicate_nexthop(iter, rt)) {
93c2fb25
DA
1140 if (rt->fib6_nsiblings)
1141 rt->fib6_nsiblings = 0;
1142 if (!(iter->fib6_flags & RTF_EXPIRES))
1da177e4 1143 return -EEXIST;
5eb902b8 1144 if (!(rt->fib6_flags & RTF_EXPIRES)) {
dade3f6a 1145 fib6_clean_expires(iter);
5eb902b8
KFL
1146 fib6_remove_gc_list(iter);
1147 } else {
dade3f6a 1148 fib6_set_expires(iter, rt->expires);
5eb902b8
KFL
1149 fib6_add_gc_list(iter);
1150 }
15a81b41
DA
1151
1152 if (rt->fib6_pmtu)
1153 fib6_metric_set(iter, RTAX_MTU,
1154 rt->fib6_pmtu);
1da177e4
LT
1155 return -EEXIST;
1156 }
33bd5ac5
DA
1157 /* If we have the same destination and the same metric,
1158 * but not the same gateway, then the route we try to
1159 * add is sibling to this route, increment our counter
1160 * of siblings, and later we will add our route to the
1161 * list.
1162 * Only static routes (which don't have flag
1163 * RTF_EXPIRES) are used for ECMPv6.
1164 *
1165 * To avoid long list, we only had siblings if the
1166 * route have a gateway.
1167 */
1168 if (rt_can_ecmp &&
1169 rt6_qualify_for_ecmp(iter))
1170 rt->fib6_nsiblings++;
1da177e4
LT
1171 }
1172
93c2fb25 1173 if (iter->fib6_metric > rt->fib6_metric)
1da177e4
LT
1174 break;
1175
33bd5ac5 1176next_iter:
8fb11a9a 1177 ins = &iter->fib6_next;
27596472
MK
1178 }
1179
33bd5ac5 1180 if (fallback_ins && !found) {
e404b8c7
BP
1181 /* No matching route with same ecmp-able-ness found, replace
1182 * first matching route
1183 */
33bd5ac5
DA
1184 ins = fallback_ins;
1185 iter = rcu_dereference_protected(*ins,
1186 lockdep_is_held(&rt->fib6_table->tb6_lock));
1187 found++;
1188 }
1189
f11e6659
DM
1190 /* Reset round-robin state, if necessary */
1191 if (ins == &fn->leaf)
1192 fn->rr_ptr = NULL;
1193
51ebd318 1194 /* Link this route to others same route. */
33bd5ac5
DA
1195 if (rt->fib6_nsiblings) {
1196 unsigned int fib6_nsiblings;
8d1c802b 1197 struct fib6_info *sibling, *temp_sibling;
51ebd318 1198
33bd5ac5
DA
1199 /* Find the first route that have the same metric */
1200 sibling = leaf;
c10c4279 1201 notify_sibling_rt = true;
33bd5ac5
DA
1202 while (sibling) {
1203 if (sibling->fib6_metric == rt->fib6_metric &&
1204 rt6_qualify_for_ecmp(sibling)) {
d9ccb18f
OEH
1205 list_add_tail_rcu(&rt->fib6_siblings,
1206 &sibling->fib6_siblings);
33bd5ac5
DA
1207 break;
1208 }
1209 sibling = rcu_dereference_protected(sibling->fib6_next,
1210 lockdep_is_held(&rt->fib6_table->tb6_lock));
c10c4279 1211 notify_sibling_rt = false;
51ebd318
ND
1212 }
1213 /* For each sibling in the list, increment the counter of
1214 * siblings. BUG() if counters does not match, list of siblings
1215 * is broken!
1216 */
33bd5ac5 1217 fib6_nsiblings = 0;
51ebd318 1218 list_for_each_entry_safe(sibling, temp_sibling,
33bd5ac5 1219 &rt->fib6_siblings, fib6_siblings) {
93c2fb25 1220 sibling->fib6_nsiblings++;
33bd5ac5
DA
1221 BUG_ON(sibling->fib6_nsiblings != rt->fib6_nsiblings);
1222 fib6_nsiblings++;
51ebd318 1223 }
33bd5ac5 1224 BUG_ON(fib6_nsiblings != rt->fib6_nsiblings);
8e5f1bb8 1225 rcu_read_lock();
33bd5ac5 1226 rt6_multipath_rebalance(temp_sibling);
8e5f1bb8 1227 rcu_read_unlock();
51ebd318
ND
1228 }
1229
1da177e4
LT
1230 /*
1231 * insert node
1232 */
4a287eba
MV
1233 if (!replace) {
1234 if (!add)
f3213831 1235 pr_warn("NLM_F_CREATE should be set when creating new route\n");
4a287eba
MV
1236
1237add:
73483c12 1238 nlflags |= NLM_F_CREATE;
e715b6d3 1239
caafb250
IS
1240 /* The route should only be notified if it is the first
1241 * route in the node or if it is added as a sibling
1242 * route to the first route in the node.
1243 */
1244 if (!info->skip_notify_kernel &&
1245 (notify_sibling_rt || ins == &fn->leaf)) {
c10c4279
IS
1246 enum fib_event_type fib_event;
1247
1248 if (notify_sibling_rt)
1249 fib_event = FIB_EVENT_ENTRY_APPEND;
1250 else
caafb250 1251 fib_event = FIB_EVENT_ENTRY_REPLACE;
d5382fef 1252 err = call_fib6_entry_notifiers(info->nl_net,
caafb250
IS
1253 fib_event, rt,
1254 extack);
54851aa9
IS
1255 if (err) {
1256 struct fib6_info *sibling, *next_sibling;
1257
1258 /* If the route has siblings, then it first
1259 * needs to be unlinked from them.
1260 */
1261 if (!rt->fib6_nsiblings)
1262 return err;
1263
1264 list_for_each_entry_safe(sibling, next_sibling,
1265 &rt->fib6_siblings,
1266 fib6_siblings)
1267 sibling->fib6_nsiblings--;
1268 rt->fib6_nsiblings = 0;
d9ccb18f 1269 list_del_rcu(&rt->fib6_siblings);
8e5f1bb8 1270 rcu_read_lock();
54851aa9 1271 rt6_multipath_rebalance(next_sibling);
8e5f1bb8 1272 rcu_read_unlock();
d5382fef 1273 return err;
54851aa9 1274 }
d5382fef 1275 }
2233000c 1276
8fb11a9a 1277 rcu_assign_pointer(rt->fib6_next, iter);
5ea71528 1278 fib6_info_hold(rt);
93c2fb25 1279 rcu_assign_pointer(rt->fib6_node, fn);
66f5d6ce 1280 rcu_assign_pointer(*ins, rt);
3b1137fe
DA
1281 if (!info->skip_notify)
1282 inet6_rt_notify(RTM_NEWROUTE, rt, info, nlflags);
4a287eba
MV
1283 info->nl_net->ipv6.rt6_stats->fib_rt_entries++;
1284
507c9b1e 1285 if (!(fn->fn_flags & RTN_RTINFO)) {
4a287eba
MV
1286 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1287 fn->fn_flags |= RTN_RTINFO;
1288 }
1da177e4 1289
4a287eba 1290 } else {
33bd5ac5 1291 int nsiblings;
27596472 1292
4a287eba
MV
1293 if (!found) {
1294 if (add)
1295 goto add;
f3213831 1296 pr_warn("NLM_F_REPLACE set, but no existing node found!\n");
4a287eba
MV
1297 return -ENOENT;
1298 }
e715b6d3 1299
caafb250 1300 if (!info->skip_notify_kernel && ins == &fn->leaf) {
d5382fef
IS
1301 err = call_fib6_entry_notifiers(info->nl_net,
1302 FIB_EVENT_ENTRY_REPLACE,
1303 rt, extack);
1304 if (err)
1305 return err;
1306 }
2233000c 1307
5ea71528 1308 fib6_info_hold(rt);
93c2fb25 1309 rcu_assign_pointer(rt->fib6_node, fn);
33bd5ac5 1310 rt->fib6_next = iter->fib6_next;
66f5d6ce 1311 rcu_assign_pointer(*ins, rt);
3b1137fe
DA
1312 if (!info->skip_notify)
1313 inet6_rt_notify(RTM_NEWROUTE, rt, info, NLM_F_REPLACE);
507c9b1e 1314 if (!(fn->fn_flags & RTN_RTINFO)) {
4a287eba
MV
1315 info->nl_net->ipv6.rt6_stats->fib_route_nodes++;
1316 fn->fn_flags |= RTN_RTINFO;
1317 }
33bd5ac5
DA
1318 nsiblings = iter->fib6_nsiblings;
1319 iter->fib6_node = NULL;
accb46b5 1320 list_add(&iter->purge_link, purge_list);
33bd5ac5
DA
1321 if (rcu_access_pointer(fn->rr_ptr) == iter)
1322 fn->rr_ptr = NULL;
27596472 1323
33bd5ac5 1324 if (nsiblings) {
27596472 1325 /* Replacing an ECMP route, remove all siblings */
33bd5ac5
DA
1326 ins = &rt->fib6_next;
1327 iter = rcu_dereference_protected(*ins,
1328 lockdep_is_held(&rt->fib6_table->tb6_lock));
1329 while (iter) {
1330 if (iter->fib6_metric > rt->fib6_metric)
1331 break;
1332 if (rt6_qualify_for_ecmp(iter)) {
1333 *ins = iter->fib6_next;
1334 iter->fib6_node = NULL;
accb46b5 1335 list_add(&iter->purge_link, purge_list);
33bd5ac5
DA
1336 if (rcu_access_pointer(fn->rr_ptr) == iter)
1337 fn->rr_ptr = NULL;
33bd5ac5
DA
1338 nsiblings--;
1339 info->nl_net->ipv6.rt6_stats->fib_rt_entries--;
1340 } else {
1341 ins = &iter->fib6_next;
1342 }
1343 iter = rcu_dereference_protected(*ins,
1344 lockdep_is_held(&rt->fib6_table->tb6_lock));
27596472 1345 }
33bd5ac5 1346 WARN_ON(nsiblings != 0);
27596472 1347 }
1da177e4
LT
1348 }
1349
1350 return 0;
1351}
1352
081efd18
KI
1353static int fib6_add_rt2node_nh(struct fib6_node *fn, struct fib6_info *rt,
1354 struct nl_info *info, struct netlink_ext_ack *extack,
1355 struct list_head *purge_list)
1356{
1357 int err;
1358
1359 spin_lock(&rt->nh->lock);
1360
1361 if (rt->nh->dead) {
1362 NL_SET_ERR_MSG(extack, "Nexthop has been deleted");
1363 err = -EINVAL;
1364 } else {
1365 err = fib6_add_rt2node(fn, rt, info, extack, purge_list);
1366 if (!err)
1367 list_add(&rt->nh_list, &rt->nh->f6i_list);
1368 }
1369
1370 spin_unlock(&rt->nh->lock);
1371
1372 return err;
1373}
1374
8d1c802b 1375static void fib6_start_gc(struct net *net, struct fib6_info *rt)
1da177e4 1376{
417f28bb 1377 if (!timer_pending(&net->ipv6.ip6_fib_timer) &&
93c2fb25 1378 (rt->fib6_flags & RTF_EXPIRES))
417f28bb 1379 mod_timer(&net->ipv6.ip6_fib_timer,
847499ce 1380 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1da177e4
LT
1381}
1382
63152fc0 1383void fib6_force_start_gc(struct net *net)
1da177e4 1384{
417f28bb
SH
1385 if (!timer_pending(&net->ipv6.ip6_fib_timer))
1386 mod_timer(&net->ipv6.ip6_fib_timer,
847499ce 1387 jiffies + net->ipv6.sysctl.ip6_rt_gc_interval);
1da177e4
LT
1388}
1389
8d1c802b 1390static void __fib6_update_sernum_upto_root(struct fib6_info *rt,
4a8e56ee 1391 int sernum)
bbd63f06 1392{
93c2fb25
DA
1393 struct fib6_node *fn = rcu_dereference_protected(rt->fib6_node,
1394 lockdep_is_held(&rt->fib6_table->tb6_lock));
bbd63f06 1395
446e7f21 1396 /* paired with smp_rmb() in fib6_get_cookie_safe() */
bbd63f06
WW
1397 smp_wmb();
1398 while (fn) {
aafc2e32 1399 WRITE_ONCE(fn->fn_sernum, sernum);
66f5d6ce 1400 fn = rcu_dereference_protected(fn->parent,
93c2fb25 1401 lockdep_is_held(&rt->fib6_table->tb6_lock));
bbd63f06
WW
1402 }
1403}
1404
8d1c802b 1405void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt)
4a8e56ee
IS
1406{
1407 __fib6_update_sernum_upto_root(rt, fib6_new_sernum(net));
1408}
1409
cdaa16a4
DA
1410/* allow ipv4 to update sernum via ipv6_stub */
1411void fib6_update_sernum_stub(struct net *net, struct fib6_info *f6i)
1412{
1413 spin_lock_bh(&f6i->fib6_table->tb6_lock);
1414 fib6_update_sernum_upto_root(net, f6i);
1415 spin_unlock_bh(&f6i->fib6_table->tb6_lock);
1416}
1417
1da177e4
LT
1418/*
1419 * Add routing information to the routing tree.
1420 * <destination addr>/<source addr>
1421 * with source addr info in sub-trees
66f5d6ce 1422 * Need to own table->tb6_lock
1da177e4
LT
1423 */
1424
8d1c802b 1425int fib6_add(struct fib6_node *root, struct fib6_info *rt,
d4ead6b3 1426 struct nl_info *info, struct netlink_ext_ack *extack)
1da177e4 1427{
93c2fb25 1428 struct fib6_table *table = rt->fib6_table;
accb46b5 1429 LIST_HEAD(purge_list);
74043489
AB
1430 struct fib6_node *fn;
1431#ifdef CONFIG_IPV6_SUBTREES
1432 struct fib6_node *pn = NULL;
1433#endif
1da177e4 1434 int err = -ENOMEM;
4a287eba
MV
1435 int allow_create = 1;
1436 int replace_required = 0;
507c9b1e
DM
1437
1438 if (info->nlh) {
1439 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4a287eba 1440 allow_create = 0;
507c9b1e 1441 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4a287eba
MV
1442 replace_required = 1;
1443 }
1444 if (!allow_create && !replace_required)
f3213831 1445 pr_warn("RTM_NEWROUTE with no NLM_F_CREATE or NLM_F_REPLACE\n");
1da177e4 1446
81eb8447 1447 fn = fib6_add_1(info->nl_net, table, root,
93c2fb25
DA
1448 &rt->fib6_dst.addr, rt->fib6_dst.plen,
1449 offsetof(struct fib6_info, fib6_dst), allow_create,
bbd63f06 1450 replace_required, extack);
4a287eba
MV
1451 if (IS_ERR(fn)) {
1452 err = PTR_ERR(fn);
ae7b4e1f 1453 fn = NULL;
1da177e4 1454 goto out;
188c517a 1455 }
1da177e4 1456
74043489 1457#ifdef CONFIG_IPV6_SUBTREES
66729e18
YH
1458 pn = fn;
1459
93c2fb25 1460 if (rt->fib6_src.plen) {
1da177e4
LT
1461 struct fib6_node *sn;
1462
66f5d6ce 1463 if (!rcu_access_pointer(fn->subtree)) {
1da177e4
LT
1464 struct fib6_node *sfn;
1465
1466 /*
1467 * Create subtree.
1468 *
1469 * fn[main tree]
1470 * |
1471 * sfn[subtree root]
1472 * \
1473 * sn[new leaf node]
1474 */
1475
1476 /* Create subtree root node */
81eb8447 1477 sfn = node_alloc(info->nl_net);
507c9b1e 1478 if (!sfn)
348a4002 1479 goto failure;
1da177e4 1480
5ea71528 1481 fib6_info_hold(info->nl_net->ipv6.fib6_null_entry);
66f5d6ce 1482 rcu_assign_pointer(sfn->leaf,
421842ed 1483 info->nl_net->ipv6.fib6_null_entry);
1da177e4 1484 sfn->fn_flags = RTN_ROOT;
1da177e4
LT
1485
1486 /* Now add the first leaf node to new subtree */
1487
81eb8447 1488 sn = fib6_add_1(info->nl_net, table, sfn,
93c2fb25
DA
1489 &rt->fib6_src.addr, rt->fib6_src.plen,
1490 offsetof(struct fib6_info, fib6_src),
bbd63f06 1491 allow_create, replace_required, extack);
1da177e4 1492
f950c0ec 1493 if (IS_ERR(sn)) {
1da177e4 1494 /* If it is failed, discard just allocated
348a4002 1495 root, and then (in failure) stale node
1da177e4
LT
1496 in main tree.
1497 */
81eb8447 1498 node_free_immediate(info->nl_net, sfn);
188c517a 1499 err = PTR_ERR(sn);
348a4002 1500 goto failure;
1da177e4
LT
1501 }
1502
1503 /* Now link new subtree to main tree */
66f5d6ce
WW
1504 rcu_assign_pointer(sfn->parent, fn);
1505 rcu_assign_pointer(fn->subtree, sfn);
1da177e4 1506 } else {
81eb8447 1507 sn = fib6_add_1(info->nl_net, table, FIB6_SUBTREE(fn),
93c2fb25
DA
1508 &rt->fib6_src.addr, rt->fib6_src.plen,
1509 offsetof(struct fib6_info, fib6_src),
bbd63f06 1510 allow_create, replace_required, extack);
1da177e4 1511
4a287eba
MV
1512 if (IS_ERR(sn)) {
1513 err = PTR_ERR(sn);
348a4002 1514 goto failure;
188c517a 1515 }
1da177e4
LT
1516 }
1517
66f5d6ce 1518 if (!rcu_access_pointer(fn->leaf)) {
591ff9ea
WW
1519 if (fn->fn_flags & RTN_TL_ROOT) {
1520 /* put back null_entry for root node */
1521 rcu_assign_pointer(fn->leaf,
421842ed 1522 info->nl_net->ipv6.fib6_null_entry);
591ff9ea 1523 } else {
5ea71528 1524 fib6_info_hold(rt);
591ff9ea
WW
1525 rcu_assign_pointer(fn->leaf, rt);
1526 }
66729e18 1527 }
1da177e4
LT
1528 fn = sn;
1529 }
1530#endif
1531
081efd18
KI
1532 if (rt->nh)
1533 err = fib6_add_rt2node_nh(fn, rt, info, extack, &purge_list);
1534 else
1535 err = fib6_add_rt2node(fn, rt, info, extack, &purge_list);
bbd63f06 1536 if (!err) {
accb46b5
KI
1537 struct fib6_info *iter, *next;
1538
1539 list_for_each_entry_safe(iter, next, &purge_list, purge_link) {
1540 list_del(&iter->purge_link);
1541 fib6_purge_rt(iter, fn, info->nl_net);
1542 fib6_info_release(iter);
1543 }
1544
e87b5052 1545 __fib6_update_sernum_upto_root(rt, fib6_new_sernum(info->nl_net));
5eb902b8
KFL
1546
1547 if (rt->fib6_flags & RTF_EXPIRES)
1548 fib6_add_gc_list(rt);
1549
63152fc0 1550 fib6_start_gc(info->nl_net, rt);
bbd63f06 1551 }
1da177e4
LT
1552
1553out:
66729e18
YH
1554 if (err) {
1555#ifdef CONFIG_IPV6_SUBTREES
1556 /*
1557 * If fib6_add_1 has cleared the old leaf pointer in the
1558 * super-tree leaf node we have to find a new one for it.
1559 */
7bbfe00e 1560 if (pn != fn) {
8d1c802b 1561 struct fib6_info *pn_leaf =
7bbfe00e
WW
1562 rcu_dereference_protected(pn->leaf,
1563 lockdep_is_held(&table->tb6_lock));
1564 if (pn_leaf == rt) {
1565 pn_leaf = NULL;
1566 RCU_INIT_POINTER(pn->leaf, NULL);
93531c67 1567 fib6_info_release(rt);
66729e18 1568 }
7bbfe00e
WW
1569 if (!pn_leaf && !(pn->fn_flags & RTN_RTINFO)) {
1570 pn_leaf = fib6_find_prefix(info->nl_net, table,
1571 pn);
75475bb5 1572 if (!pn_leaf)
7bbfe00e 1573 pn_leaf =
421842ed 1574 info->nl_net->ipv6.fib6_null_entry;
93531c67 1575 fib6_info_hold(pn_leaf);
7bbfe00e
WW
1576 rcu_assign_pointer(pn->leaf, pn_leaf);
1577 }
66729e18
YH
1578 }
1579#endif
348a4002 1580 goto failure;
b9b33e7c
PA
1581 } else if (fib6_requires_src(rt)) {
1582 fib6_routes_require_src_inc(info->nl_net);
66729e18 1583 }
1da177e4
LT
1584 return err;
1585
348a4002 1586failure:
4512c43e
WW
1587 /* fn->leaf could be NULL and fib6_repair_tree() needs to be called if:
1588 * 1. fn is an intermediate node and we failed to add the new
1589 * route to it in both subtree creation failure and fib6_add_rt2node()
1590 * failure case.
1591 * 2. fn is the root node in the table and we fail to add the first
1592 * default route to it.
1da177e4 1593 */
4512c43e
WW
1594 if (fn &&
1595 (!(fn->fn_flags & (RTN_RTINFO|RTN_ROOT)) ||
1596 (fn->fn_flags & RTN_TL_ROOT &&
1597 !rcu_access_pointer(fn->leaf))))
66f5d6ce 1598 fib6_repair_tree(info->nl_net, table, fn);
1da177e4 1599 return err;
1da177e4
LT
1600}
1601
1602/*
1603 * Routing tree lookup
1604 *
1605 */
1606
1607struct lookup_args {
8d1c802b 1608 int offset; /* key offset on fib6_info */
b71d1d42 1609 const struct in6_addr *addr; /* search key */
1da177e4
LT
1610};
1611
6454743b
DA
1612static struct fib6_node *fib6_node_lookup_1(struct fib6_node *root,
1613 struct lookup_args *args)
1da177e4
LT
1614{
1615 struct fib6_node *fn;
e69a4adc 1616 __be32 dir;
1da177e4 1617
825e288e
YH
1618 if (unlikely(args->offset == 0))
1619 return NULL;
1620
1da177e4
LT
1621 /*
1622 * Descend on a tree
1623 */
1624
1625 fn = root;
1626
1627 for (;;) {
1628 struct fib6_node *next;
1629
1630 dir = addr_bit_set(args->addr, fn->fn_bit);
1631
66f5d6ce
WW
1632 next = dir ? rcu_dereference(fn->right) :
1633 rcu_dereference(fn->left);
1da177e4
LT
1634
1635 if (next) {
1636 fn = next;
1637 continue;
1638 }
1da177e4
LT
1639 break;
1640 }
1641
507c9b1e 1642 while (fn) {
66f5d6ce
WW
1643 struct fib6_node *subtree = FIB6_SUBTREE(fn);
1644
1645 if (subtree || fn->fn_flags & RTN_RTINFO) {
8d1c802b 1646 struct fib6_info *leaf = rcu_dereference(fn->leaf);
1da177e4
LT
1647 struct rt6key *key;
1648
8d1040e8
WW
1649 if (!leaf)
1650 goto backtrack;
1651
1652 key = (struct rt6key *) ((u8 *)leaf + args->offset);
1da177e4 1653
3fc5e044
YH
1654 if (ipv6_prefix_equal(&key->addr, args->addr, key->plen)) {
1655#ifdef CONFIG_IPV6_SUBTREES
66f5d6ce 1656 if (subtree) {
3e3be275 1657 struct fib6_node *sfn;
6454743b
DA
1658 sfn = fib6_node_lookup_1(subtree,
1659 args + 1);
3e3be275
HFS
1660 if (!sfn)
1661 goto backtrack;
1662 fn = sfn;
1663 }
3fc5e044 1664#endif
3e3be275 1665 if (fn->fn_flags & RTN_RTINFO)
3fc5e044
YH
1666 return fn;
1667 }
1da177e4 1668 }
3e3be275 1669backtrack:
3fc5e044
YH
1670 if (fn->fn_flags & RTN_ROOT)
1671 break;
1672
66f5d6ce 1673 fn = rcu_dereference(fn->parent);
1da177e4
LT
1674 }
1675
1676 return NULL;
1677}
1678
66f5d6ce
WW
1679/* called with rcu_read_lock() held
1680 */
6454743b
DA
1681struct fib6_node *fib6_node_lookup(struct fib6_node *root,
1682 const struct in6_addr *daddr,
1683 const struct in6_addr *saddr)
1da177e4 1684{
1da177e4 1685 struct fib6_node *fn;
825e288e
YH
1686 struct lookup_args args[] = {
1687 {
93c2fb25 1688 .offset = offsetof(struct fib6_info, fib6_dst),
825e288e
YH
1689 .addr = daddr,
1690 },
1da177e4 1691#ifdef CONFIG_IPV6_SUBTREES
825e288e 1692 {
93c2fb25 1693 .offset = offsetof(struct fib6_info, fib6_src),
825e288e
YH
1694 .addr = saddr,
1695 },
1da177e4 1696#endif
825e288e
YH
1697 {
1698 .offset = 0, /* sentinel */
1699 }
1700 };
1da177e4 1701
6454743b 1702 fn = fib6_node_lookup_1(root, daddr ? args : args + 1);
507c9b1e 1703 if (!fn || fn->fn_flags & RTN_TL_ROOT)
1da177e4
LT
1704 fn = root;
1705
1706 return fn;
1707}
1708
1709/*
1710 * Get node with specified destination prefix (and source prefix,
1711 * if subtrees are used)
38fbeeee
WW
1712 * exact_match == true means we try to find fn with exact match of
1713 * the passed in prefix addr
1714 * exact_match == false means we try to find fn with longest prefix
1715 * match of the passed in prefix addr. This is useful for finding fn
1716 * for cached route as it will be stored in the exception table under
1717 * the node with longest prefix length.
1da177e4
LT
1718 */
1719
1720
437de07c
WY
1721static struct fib6_node *fib6_locate_1(struct fib6_node *root,
1722 const struct in6_addr *addr,
38fbeeee
WW
1723 int plen, int offset,
1724 bool exact_match)
1da177e4 1725{
38fbeeee 1726 struct fib6_node *fn, *prev = NULL;
1da177e4
LT
1727
1728 for (fn = root; fn ; ) {
8d1c802b 1729 struct fib6_info *leaf = rcu_dereference(fn->leaf);
8d1040e8
WW
1730 struct rt6key *key;
1731
1732 /* This node is being deleted */
1733 if (!leaf) {
1734 if (plen <= fn->fn_bit)
1735 goto out;
1736 else
1737 goto next;
1738 }
1739
1740 key = (struct rt6key *)((u8 *)leaf + offset);
1da177e4
LT
1741
1742 /*
1743 * Prefix match
1744 */
1745 if (plen < fn->fn_bit ||
1746 !ipv6_prefix_equal(&key->addr, addr, fn->fn_bit))
38fbeeee 1747 goto out;
1da177e4
LT
1748
1749 if (plen == fn->fn_bit)
1750 return fn;
1751
40cb35d5
SB
1752 if (fn->fn_flags & RTN_RTINFO)
1753 prev = fn;
38fbeeee 1754
8d1040e8 1755next:
1da177e4
LT
1756 /*
1757 * We have more bits to go
1758 */
1759 if (addr_bit_set(addr, fn->fn_bit))
66f5d6ce 1760 fn = rcu_dereference(fn->right);
1da177e4 1761 else
66f5d6ce 1762 fn = rcu_dereference(fn->left);
1da177e4 1763 }
38fbeeee
WW
1764out:
1765 if (exact_match)
1766 return NULL;
1767 else
1768 return prev;
1da177e4
LT
1769}
1770
437de07c
WY
1771struct fib6_node *fib6_locate(struct fib6_node *root,
1772 const struct in6_addr *daddr, int dst_len,
38fbeeee
WW
1773 const struct in6_addr *saddr, int src_len,
1774 bool exact_match)
1da177e4
LT
1775{
1776 struct fib6_node *fn;
1777
1778 fn = fib6_locate_1(root, daddr, dst_len,
93c2fb25 1779 offsetof(struct fib6_info, fib6_dst),
38fbeeee 1780 exact_match);
1da177e4
LT
1781
1782#ifdef CONFIG_IPV6_SUBTREES
1783 if (src_len) {
547b792c 1784 WARN_ON(saddr == NULL);
0e80193b
WW
1785 if (fn) {
1786 struct fib6_node *subtree = FIB6_SUBTREE(fn);
1787
1788 if (subtree) {
1789 fn = fib6_locate_1(subtree, saddr, src_len,
93c2fb25 1790 offsetof(struct fib6_info, fib6_src),
38fbeeee 1791 exact_match);
0e80193b
WW
1792 }
1793 }
1da177e4
LT
1794 }
1795#endif
1796
507c9b1e 1797 if (fn && fn->fn_flags & RTN_RTINFO)
1da177e4
LT
1798 return fn;
1799
1800 return NULL;
1801}
1802
1803
1804/*
1805 * Deletion
1806 *
1807 */
1808
8d1c802b 1809static struct fib6_info *fib6_find_prefix(struct net *net,
66f5d6ce
WW
1810 struct fib6_table *table,
1811 struct fib6_node *fn)
1da177e4 1812{
66f5d6ce
WW
1813 struct fib6_node *child_left, *child_right;
1814
507c9b1e 1815 if (fn->fn_flags & RTN_ROOT)
421842ed 1816 return net->ipv6.fib6_null_entry;
1da177e4 1817
507c9b1e 1818 while (fn) {
66f5d6ce
WW
1819 child_left = rcu_dereference_protected(fn->left,
1820 lockdep_is_held(&table->tb6_lock));
1821 child_right = rcu_dereference_protected(fn->right,
1822 lockdep_is_held(&table->tb6_lock));
1823 if (child_left)
1824 return rcu_dereference_protected(child_left->leaf,
1825 lockdep_is_held(&table->tb6_lock));
1826 if (child_right)
1827 return rcu_dereference_protected(child_right->leaf,
1828 lockdep_is_held(&table->tb6_lock));
1da177e4 1829
7fc33165 1830 fn = FIB6_SUBTREE(fn);
1da177e4
LT
1831 }
1832 return NULL;
1833}
1834
1835/*
1836 * Called to trim the tree of intermediate nodes when possible. "fn"
1837 * is the node we want to try and remove.
66f5d6ce 1838 * Need to own table->tb6_lock
1da177e4
LT
1839 */
1840
8ed67789 1841static struct fib6_node *fib6_repair_tree(struct net *net,
66f5d6ce
WW
1842 struct fib6_table *table,
1843 struct fib6_node *fn)
1da177e4
LT
1844{
1845 int children;
1846 int nstate;
66f5d6ce 1847 struct fib6_node *child;
94b2cfe0 1848 struct fib6_walker *w;
1da177e4
LT
1849 int iter = 0;
1850
4512c43e
WW
1851 /* Set fn->leaf to null_entry for root node. */
1852 if (fn->fn_flags & RTN_TL_ROOT) {
421842ed 1853 rcu_assign_pointer(fn->leaf, net->ipv6.fib6_null_entry);
4512c43e
WW
1854 return fn;
1855 }
1856
1da177e4 1857 for (;;) {
66f5d6ce
WW
1858 struct fib6_node *fn_r = rcu_dereference_protected(fn->right,
1859 lockdep_is_held(&table->tb6_lock));
1860 struct fib6_node *fn_l = rcu_dereference_protected(fn->left,
1861 lockdep_is_held(&table->tb6_lock));
1862 struct fib6_node *pn = rcu_dereference_protected(fn->parent,
1863 lockdep_is_held(&table->tb6_lock));
1864 struct fib6_node *pn_r = rcu_dereference_protected(pn->right,
1865 lockdep_is_held(&table->tb6_lock));
1866 struct fib6_node *pn_l = rcu_dereference_protected(pn->left,
1867 lockdep_is_held(&table->tb6_lock));
8d1c802b 1868 struct fib6_info *fn_leaf = rcu_dereference_protected(fn->leaf,
66f5d6ce 1869 lockdep_is_held(&table->tb6_lock));
8d1c802b 1870 struct fib6_info *pn_leaf = rcu_dereference_protected(pn->leaf,
66f5d6ce 1871 lockdep_is_held(&table->tb6_lock));
8d1c802b 1872 struct fib6_info *new_fn_leaf;
66f5d6ce 1873
20df28fb 1874 pr_debug("fixing tree: plen=%d iter=%d\n", fn->fn_bit, iter);
1da177e4
LT
1875 iter++;
1876
547b792c
IJ
1877 WARN_ON(fn->fn_flags & RTN_RTINFO);
1878 WARN_ON(fn->fn_flags & RTN_TL_ROOT);
66f5d6ce 1879 WARN_ON(fn_leaf);
1da177e4
LT
1880
1881 children = 0;
1882 child = NULL;
dee84779
JP
1883 if (fn_r) {
1884 child = fn_r;
1885 children |= 1;
1886 }
1887 if (fn_l) {
1888 child = fn_l;
1889 children |= 2;
1890 }
1da177e4 1891
7fc33165 1892 if (children == 3 || FIB6_SUBTREE(fn)
1da177e4
LT
1893#ifdef CONFIG_IPV6_SUBTREES
1894 /* Subtree root (i.e. fn) may have one child */
507c9b1e 1895 || (children && fn->fn_flags & RTN_ROOT)
1da177e4
LT
1896#endif
1897 ) {
66f5d6ce 1898 new_fn_leaf = fib6_find_prefix(net, table, fn);
1da177e4 1899#if RT6_DEBUG >= 2
66f5d6ce
WW
1900 if (!new_fn_leaf) {
1901 WARN_ON(!new_fn_leaf);
421842ed 1902 new_fn_leaf = net->ipv6.fib6_null_entry;
1da177e4
LT
1903 }
1904#endif
93531c67 1905 fib6_info_hold(new_fn_leaf);
66f5d6ce
WW
1906 rcu_assign_pointer(fn->leaf, new_fn_leaf);
1907 return pn;
1da177e4
LT
1908 }
1909
1da177e4 1910#ifdef CONFIG_IPV6_SUBTREES
7fc33165 1911 if (FIB6_SUBTREE(pn) == fn) {
547b792c 1912 WARN_ON(!(fn->fn_flags & RTN_ROOT));
66f5d6ce 1913 RCU_INIT_POINTER(pn->subtree, NULL);
1da177e4
LT
1914 nstate = FWS_L;
1915 } else {
547b792c 1916 WARN_ON(fn->fn_flags & RTN_ROOT);
1da177e4 1917#endif
66f5d6ce
WW
1918 if (pn_r == fn)
1919 rcu_assign_pointer(pn->right, child);
1920 else if (pn_l == fn)
1921 rcu_assign_pointer(pn->left, child);
1da177e4 1922#if RT6_DEBUG >= 2
547b792c
IJ
1923 else
1924 WARN_ON(1);
1da177e4
LT
1925#endif
1926 if (child)
66f5d6ce 1927 rcu_assign_pointer(child->parent, pn);
1da177e4
LT
1928 nstate = FWS_R;
1929#ifdef CONFIG_IPV6_SUBTREES
1930 }
1931#endif
1932
9a03cd8f
MK
1933 read_lock(&net->ipv6.fib6_walker_lock);
1934 FOR_WALKERS(net, w) {
507c9b1e 1935 if (!child) {
2b760fcf 1936 if (w->node == fn) {
20df28fb
BL
1937 pr_debug("W %p adjusted by delnode 1, s=%d/%d\n",
1938 w, w->state, nstate);
1da177e4
LT
1939 w->node = pn;
1940 w->state = nstate;
1941 }
1942 } else {
1da177e4
LT
1943 if (w->node == fn) {
1944 w->node = child;
1945 if (children&2) {
20df28fb
BL
1946 pr_debug("W %p adjusted by delnode 2, s=%d\n",
1947 w, w->state);
8db46f1d 1948 w->state = w->state >= FWS_R ? FWS_U : FWS_INIT;
1da177e4 1949 } else {
20df28fb
BL
1950 pr_debug("W %p adjusted by delnode 2, s=%d\n",
1951 w, w->state);
8db46f1d 1952 w->state = w->state >= FWS_C ? FWS_U : FWS_INIT;
1da177e4
LT
1953 }
1954 }
1955 }
1956 }
9a03cd8f 1957 read_unlock(&net->ipv6.fib6_walker_lock);
1da177e4 1958
81eb8447 1959 node_free(net, fn);
507c9b1e 1960 if (pn->fn_flags & RTN_RTINFO || FIB6_SUBTREE(pn))
1da177e4
LT
1961 return pn;
1962
66f5d6ce 1963 RCU_INIT_POINTER(pn->leaf, NULL);
93531c67 1964 fib6_info_release(pn_leaf);
1da177e4
LT
1965 fn = pn;
1966 }
1967}
1968
66f5d6ce 1969static void fib6_del_route(struct fib6_table *table, struct fib6_node *fn,
8d1c802b 1970 struct fib6_info __rcu **rtp, struct nl_info *info)
1da177e4 1971{
d2f0c9b1 1972 struct fib6_info *leaf, *replace_rt = NULL;
94b2cfe0 1973 struct fib6_walker *w;
8d1c802b 1974 struct fib6_info *rt = rcu_dereference_protected(*rtp,
66f5d6ce 1975 lockdep_is_held(&table->tb6_lock));
c572872f 1976 struct net *net = info->nl_net;
d2f0c9b1 1977 bool notify_del = false;
1da177e4 1978
d2f0c9b1
IS
1979 /* If the deleted route is the first in the node and it is not part of
1980 * a multipath route, then we need to replace it with the next route
1981 * in the node, if exists.
1982 */
1983 leaf = rcu_dereference_protected(fn->leaf,
1984 lockdep_is_held(&table->tb6_lock));
1985 if (leaf == rt && !rt->fib6_nsiblings) {
1986 if (rcu_access_pointer(rt->fib6_next))
1987 replace_rt = rcu_dereference_protected(rt->fib6_next,
1988 lockdep_is_held(&table->tb6_lock));
1989 else
1990 notify_del = true;
1991 }
1992
1da177e4 1993 /* Unlink it */
8fb11a9a 1994 *rtp = rt->fib6_next;
93c2fb25 1995 rt->fib6_node = NULL;
c572872f
BT
1996 net->ipv6.rt6_stats->fib_rt_entries--;
1997 net->ipv6.rt6_stats->fib_discarded_routes++;
1da177e4 1998
f11e6659 1999 /* Reset round-robin state, if necessary */
66f5d6ce 2000 if (rcu_access_pointer(fn->rr_ptr) == rt)
f11e6659
DM
2001 fn->rr_ptr = NULL;
2002
51ebd318 2003 /* Remove this entry from other siblings */
93c2fb25 2004 if (rt->fib6_nsiblings) {
8d1c802b 2005 struct fib6_info *sibling, *next_sibling;
51ebd318 2006
d2f0c9b1
IS
2007 /* The route is deleted from a multipath route. If this
2008 * multipath route is the first route in the node, then we need
2009 * to emit a delete notification. Otherwise, we need to skip
2010 * the notification.
2011 */
2012 if (rt->fib6_metric == leaf->fib6_metric &&
2013 rt6_qualify_for_ecmp(leaf))
2014 notify_del = true;
51ebd318 2015 list_for_each_entry_safe(sibling, next_sibling,
93c2fb25
DA
2016 &rt->fib6_siblings, fib6_siblings)
2017 sibling->fib6_nsiblings--;
2018 rt->fib6_nsiblings = 0;
d9ccb18f 2019 list_del_rcu(&rt->fib6_siblings);
d7dedee1 2020 rt6_multipath_rebalance(next_sibling);
51ebd318
ND
2021 }
2022
1da177e4 2023 /* Adjust walkers */
9a03cd8f
MK
2024 read_lock(&net->ipv6.fib6_walker_lock);
2025 FOR_WALKERS(net, w) {
1da177e4 2026 if (w->state == FWS_C && w->leaf == rt) {
20df28fb 2027 pr_debug("walker %p adjusted by delroute\n", w);
8fb11a9a 2028 w->leaf = rcu_dereference_protected(rt->fib6_next,
66f5d6ce 2029 lockdep_is_held(&table->tb6_lock));
507c9b1e 2030 if (!w->leaf)
1da177e4
LT
2031 w->state = FWS_U;
2032 }
2033 }
9a03cd8f 2034 read_unlock(&net->ipv6.fib6_walker_lock);
1da177e4 2035
4512c43e
WW
2036 /* If it was last route, call fib6_repair_tree() to:
2037 * 1. For root node, put back null_entry as how the table was created.
2038 * 2. For other nodes, expunge its radix tree node.
2039 */
66f5d6ce 2040 if (!rcu_access_pointer(fn->leaf)) {
4512c43e
WW
2041 if (!(fn->fn_flags & RTN_TL_ROOT)) {
2042 fn->fn_flags &= ~RTN_RTINFO;
2043 net->ipv6.rt6_stats->fib_route_nodes--;
2044 }
66f5d6ce 2045 fn = fib6_repair_tree(net, table, fn);
1da177e4
LT
2046 }
2047
6e9e16e6 2048 fib6_purge_rt(rt, fn, net);
1da177e4 2049
d2f0c9b1
IS
2050 if (!info->skip_notify_kernel) {
2051 if (notify_del)
caafb250 2052 call_fib6_entry_notifiers(net, FIB_EVENT_ENTRY_DEL,
d2f0c9b1
IS
2053 rt, NULL);
2054 else if (replace_rt)
2055 call_fib6_entry_notifiers_replace(net, replace_rt);
d2f0c9b1 2056 }
16a16cd3
DA
2057 if (!info->skip_notify)
2058 inet6_rt_notify(RTM_DELROUTE, rt, info, 0);
d5382fef 2059
93531c67 2060 fib6_info_release(rt);
1da177e4
LT
2061}
2062
66f5d6ce 2063/* Need to own table->tb6_lock */
8d1c802b 2064int fib6_del(struct fib6_info *rt, struct nl_info *info)
1da177e4 2065{
8ed67789 2066 struct net *net = info->nl_net;
8d1c802b
DA
2067 struct fib6_info __rcu **rtp;
2068 struct fib6_info __rcu **rtp_next;
843d926b
ED
2069 struct fib6_table *table;
2070 struct fib6_node *fn;
2071
2072 if (rt == net->ipv6.fib6_null_entry)
2073 return -ENOENT;
1da177e4 2074
843d926b
ED
2075 table = rt->fib6_table;
2076 fn = rcu_dereference_protected(rt->fib6_node,
2077 lockdep_is_held(&table->tb6_lock));
2078 if (!fn)
1da177e4
LT
2079 return -ENOENT;
2080
547b792c 2081 WARN_ON(!(fn->fn_flags & RTN_RTINFO));
1da177e4 2082
1da177e4
LT
2083 /*
2084 * Walk the leaf entries looking for ourself
2085 */
2086
66f5d6ce 2087 for (rtp = &fn->leaf; *rtp; rtp = rtp_next) {
8d1c802b 2088 struct fib6_info *cur = rcu_dereference_protected(*rtp,
66f5d6ce
WW
2089 lockdep_is_held(&table->tb6_lock));
2090 if (rt == cur) {
b9b33e7c
PA
2091 if (fib6_requires_src(cur))
2092 fib6_routes_require_src_dec(info->nl_net);
66f5d6ce 2093 fib6_del_route(table, fn, rtp, info);
1da177e4
LT
2094 return 0;
2095 }
8fb11a9a 2096 rtp_next = &cur->fib6_next;
1da177e4
LT
2097 }
2098 return -ENOENT;
2099}
2100
2101/*
2102 * Tree traversal function.
2103 *
2104 * Certainly, it is not interrupt safe.
2105 * However, it is internally reenterable wrt itself and fib6_add/fib6_del.
2106 * It means, that we can modify tree during walking
2107 * and use this function for garbage collection, clone pruning,
1ab1457c 2108 * cleaning tree when a device goes down etc. etc.
1da177e4
LT
2109 *
2110 * It guarantees that every node will be traversed,
2111 * and that it will be traversed only once.
2112 *
2113 * Callback function w->func may return:
2114 * 0 -> continue walking.
2115 * positive value -> walking is suspended (used by tree dumps,
2116 * and probably by gc, if it will be split to several slices)
2117 * negative value -> terminate walking.
2118 *
2119 * The function itself returns:
2120 * 0 -> walk is complete.
2121 * >0 -> walk is incomplete (i.e. suspended)
2122 * <0 -> walk is terminated by an error.
66f5d6ce
WW
2123 *
2124 * This function is called with tb6_lock held.
1da177e4
LT
2125 */
2126
94b2cfe0 2127static int fib6_walk_continue(struct fib6_walker *w)
1da177e4 2128{
66f5d6ce 2129 struct fib6_node *fn, *pn, *left, *right;
1da177e4 2130
2b760fcf
WW
2131 /* w->root should always be table->tb6_root */
2132 WARN_ON_ONCE(!(w->root->fn_flags & RTN_TL_ROOT));
2133
1da177e4
LT
2134 for (;;) {
2135 fn = w->node;
507c9b1e 2136 if (!fn)
1da177e4
LT
2137 return 0;
2138
1da177e4
LT
2139 switch (w->state) {
2140#ifdef CONFIG_IPV6_SUBTREES
2141 case FWS_S:
7fc33165
YH
2142 if (FIB6_SUBTREE(fn)) {
2143 w->node = FIB6_SUBTREE(fn);
1da177e4
LT
2144 continue;
2145 }
2146 w->state = FWS_L;
a8eceea8 2147 fallthrough;
1ab1457c 2148#endif
1da177e4 2149 case FWS_L:
66f5d6ce
WW
2150 left = rcu_dereference_protected(fn->left, 1);
2151 if (left) {
2152 w->node = left;
1da177e4
LT
2153 w->state = FWS_INIT;
2154 continue;
2155 }
2156 w->state = FWS_R;
a8eceea8 2157 fallthrough;
1da177e4 2158 case FWS_R:
66f5d6ce
WW
2159 right = rcu_dereference_protected(fn->right, 1);
2160 if (right) {
2161 w->node = right;
1da177e4
LT
2162 w->state = FWS_INIT;
2163 continue;
2164 }
2165 w->state = FWS_C;
66f5d6ce 2166 w->leaf = rcu_dereference_protected(fn->leaf, 1);
a8eceea8 2167 fallthrough;
1da177e4 2168 case FWS_C:
507c9b1e 2169 if (w->leaf && fn->fn_flags & RTN_RTINFO) {
2bec5a36
PM
2170 int err;
2171
fa809e2f
ED
2172 if (w->skip) {
2173 w->skip--;
1c265854 2174 goto skip;
2bec5a36
PM
2175 }
2176
2177 err = w->func(w);
1da177e4
LT
2178 if (err)
2179 return err;
2bec5a36
PM
2180
2181 w->count++;
1da177e4
LT
2182 continue;
2183 }
1c265854 2184skip:
1da177e4 2185 w->state = FWS_U;
a8eceea8 2186 fallthrough;
1da177e4
LT
2187 case FWS_U:
2188 if (fn == w->root)
2189 return 0;
66f5d6ce
WW
2190 pn = rcu_dereference_protected(fn->parent, 1);
2191 left = rcu_dereference_protected(pn->left, 1);
2192 right = rcu_dereference_protected(pn->right, 1);
1da177e4
LT
2193 w->node = pn;
2194#ifdef CONFIG_IPV6_SUBTREES
7fc33165 2195 if (FIB6_SUBTREE(pn) == fn) {
547b792c 2196 WARN_ON(!(fn->fn_flags & RTN_ROOT));
1da177e4
LT
2197 w->state = FWS_L;
2198 continue;
2199 }
2200#endif
66f5d6ce 2201 if (left == fn) {
1da177e4
LT
2202 w->state = FWS_R;
2203 continue;
2204 }
66f5d6ce 2205 if (right == fn) {
1da177e4 2206 w->state = FWS_C;
66f5d6ce 2207 w->leaf = rcu_dereference_protected(w->node->leaf, 1);
1da177e4
LT
2208 continue;
2209 }
2210#if RT6_DEBUG >= 2
547b792c 2211 WARN_ON(1);
1da177e4
LT
2212#endif
2213 }
2214 }
2215}
2216
9a03cd8f 2217static int fib6_walk(struct net *net, struct fib6_walker *w)
1da177e4
LT
2218{
2219 int res;
2220
2221 w->state = FWS_INIT;
2222 w->node = w->root;
2223
9a03cd8f 2224 fib6_walker_link(net, w);
1da177e4
LT
2225 res = fib6_walk_continue(w);
2226 if (res <= 0)
9a03cd8f 2227 fib6_walker_unlink(net, w);
1da177e4
LT
2228 return res;
2229}
2230
94b2cfe0 2231static int fib6_clean_node(struct fib6_walker *w)
1da177e4
LT
2232{
2233 int res;
8d1c802b 2234 struct fib6_info *rt;
94b2cfe0 2235 struct fib6_cleaner *c = container_of(w, struct fib6_cleaner, w);
ec7d43c2
BT
2236 struct nl_info info = {
2237 .nl_net = c->net,
7c6bb7d2 2238 .skip_notify = c->skip_notify,
ec7d43c2 2239 };
1da177e4 2240
327571cb 2241 if (c->sernum != FIB6_NO_SERNUM_CHANGE &&
aafc2e32
ED
2242 READ_ONCE(w->node->fn_sernum) != c->sernum)
2243 WRITE_ONCE(w->node->fn_sernum, c->sernum);
327571cb
HFS
2244
2245 if (!c->func) {
2246 WARN_ON_ONCE(c->sernum == FIB6_NO_SERNUM_CHANGE);
2247 w->leaf = NULL;
2248 return 0;
2249 }
2250
66f5d6ce 2251 for_each_fib6_walker_rt(w) {
1da177e4 2252 res = c->func(rt, c->arg);
b5cb5a75 2253 if (res == -1) {
1da177e4 2254 w->leaf = rt;
528c4ceb 2255 res = fib6_del(rt, &info);
1da177e4
LT
2256 if (res) {
2257#if RT6_DEBUG >= 2
91df42be 2258 pr_debug("%s: del failed: rt=%p@%p err=%d\n",
4e587ea7 2259 __func__, rt,
93c2fb25 2260 rcu_access_pointer(rt->fib6_node),
4e587ea7 2261 res);
1da177e4
LT
2262#endif
2263 continue;
2264 }
2265 return 0;
b5cb5a75 2266 } else if (res == -2) {
93c2fb25 2267 if (WARN_ON(!rt->fib6_nsiblings))
b5cb5a75 2268 continue;
93c2fb25
DA
2269 rt = list_last_entry(&rt->fib6_siblings,
2270 struct fib6_info, fib6_siblings);
b5cb5a75 2271 continue;
1da177e4 2272 }
547b792c 2273 WARN_ON(res != 0);
1da177e4
LT
2274 }
2275 w->leaf = rt;
2276 return 0;
2277}
2278
2279/*
2280 * Convenient frontend to tree walker.
1ab1457c 2281 *
1da177e4 2282 * func is called on each route.
b5cb5a75
IS
2283 * It may return -2 -> skip multipath route.
2284 * -1 -> delete this route.
1da177e4 2285 * 0 -> continue walking
1da177e4
LT
2286 */
2287
ec7d43c2 2288static void fib6_clean_tree(struct net *net, struct fib6_node *root,
8d1c802b 2289 int (*func)(struct fib6_info *, void *arg),
7c6bb7d2 2290 int sernum, void *arg, bool skip_notify)
1da177e4 2291{
94b2cfe0 2292 struct fib6_cleaner c;
1da177e4
LT
2293
2294 c.w.root = root;
2295 c.w.func = fib6_clean_node;
2bec5a36
PM
2296 c.w.count = 0;
2297 c.w.skip = 0;
1e47b483 2298 c.w.skip_in_node = 0;
1da177e4 2299 c.func = func;
327571cb 2300 c.sernum = sernum;
1da177e4 2301 c.arg = arg;
ec7d43c2 2302 c.net = net;
7c6bb7d2 2303 c.skip_notify = skip_notify;
1da177e4 2304
9a03cd8f 2305 fib6_walk(net, &c.w);
1da177e4
LT
2306}
2307
327571cb 2308static void __fib6_clean_all(struct net *net,
8d1c802b 2309 int (*func)(struct fib6_info *, void *),
7c6bb7d2 2310 int sernum, void *arg, bool skip_notify)
c71099ac 2311{
c71099ac 2312 struct fib6_table *table;
58f09b78 2313 struct hlist_head *head;
1b43af54 2314 unsigned int h;
c71099ac 2315
1b43af54 2316 rcu_read_lock();
a33bc5c1 2317 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
f3db4851 2318 head = &net->ipv6.fib_table_hash[h];
b67bfe0d 2319 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
66f5d6ce 2320 spin_lock_bh(&table->tb6_lock);
ec7d43c2 2321 fib6_clean_tree(net, &table->tb6_root,
7c6bb7d2 2322 func, sernum, arg, skip_notify);
66f5d6ce 2323 spin_unlock_bh(&table->tb6_lock);
c71099ac
TG
2324 }
2325 }
1b43af54 2326 rcu_read_unlock();
c71099ac
TG
2327}
2328
8d1c802b 2329void fib6_clean_all(struct net *net, int (*func)(struct fib6_info *, void *),
327571cb
HFS
2330 void *arg)
2331{
7c6bb7d2
DA
2332 __fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, false);
2333}
2334
2335void fib6_clean_all_skip_notify(struct net *net,
2336 int (*func)(struct fib6_info *, void *),
2337 void *arg)
2338{
2339 __fib6_clean_all(net, func, FIB6_NO_SERNUM_CHANGE, arg, true);
327571cb
HFS
2340}
2341
705f1c86
HFS
2342static void fib6_flush_trees(struct net *net)
2343{
812918c4 2344 int new_sernum = fib6_new_sernum(net);
705f1c86 2345
7c6bb7d2 2346 __fib6_clean_all(net, NULL, new_sernum, NULL, false);
705f1c86
HFS
2347}
2348
1da177e4
LT
2349/*
2350 * Garbage collection
2351 */
2352
5eb902b8 2353static int fib6_age(struct fib6_info *rt, struct fib6_gc_args *gc_args)
1da177e4
LT
2354{
2355 unsigned long now = jiffies;
2356
2357 /*
2358 * check addrconf expiration here.
2359 * Routes are expired even if they are in use.
1da177e4
LT
2360 */
2361
dade3f6a 2362 if (rt->fib6_flags & RTF_EXPIRES && rt->expires) {
14895687 2363 if (time_after(now, rt->expires)) {
20df28fb 2364 pr_debug("expiring %p\n", rt);
1da177e4
LT
2365 return -1;
2366 }
3570df91 2367 gc_args->more++;
1da177e4
LT
2368 }
2369
c757faa8
WW
2370 /* Also age clones in the exception table.
2371 * Note, that clones are aged out
2372 * only if they are not in use now.
2373 */
2374 rt6_age_exceptions(rt, gc_args, now);
2375
1da177e4
LT
2376 return 0;
2377}
2378
5eb902b8
KFL
2379static void fib6_gc_table(struct net *net,
2380 struct fib6_table *tb6,
2381 struct fib6_gc_args *gc_args)
2382{
2383 struct fib6_info *rt;
2384 struct hlist_node *n;
2385 struct nl_info info = {
2386 .nl_net = net,
2387 .skip_notify = false,
2388 };
2389
2390 hlist_for_each_entry_safe(rt, n, &tb6->tb6_gc_hlist, gc_link)
2391 if (fib6_age(rt, gc_args) == -1)
2392 fib6_del(rt, &info);
2393}
2394
2395static void fib6_gc_all(struct net *net, struct fib6_gc_args *gc_args)
2396{
2397 struct fib6_table *table;
2398 struct hlist_head *head;
2399 unsigned int h;
2400
2401 rcu_read_lock();
2402 for (h = 0; h < FIB6_TABLE_HASHSZ; h++) {
2403 head = &net->ipv6.fib_table_hash[h];
2404 hlist_for_each_entry_rcu(table, head, tb6_hlist) {
2405 spin_lock_bh(&table->tb6_lock);
2406
2407 fib6_gc_table(net, table, gc_args);
2408
2409 spin_unlock_bh(&table->tb6_lock);
2410 }
2411 }
2412 rcu_read_unlock();
2413}
2414
2ac3ac8f 2415void fib6_run_gc(unsigned long expires, struct net *net, bool force)
1da177e4 2416{
3570df91 2417 struct fib6_gc_args gc_args;
49a18d86
MK
2418 unsigned long now;
2419
2ac3ac8f 2420 if (force) {
3dc94f93
MK
2421 spin_lock_bh(&net->ipv6.fib6_gc_lock);
2422 } else if (!spin_trylock_bh(&net->ipv6.fib6_gc_lock)) {
2ac3ac8f
MK
2423 mod_timer(&net->ipv6.ip6_fib_timer, jiffies + HZ);
2424 return;
1da177e4 2425 }
2ac3ac8f
MK
2426 gc_args.timeout = expires ? (int)expires :
2427 net->ipv6.sysctl.ip6_rt_gc_interval;
db916649 2428 gc_args.more = 0;
f3db4851 2429
5eb902b8 2430 fib6_gc_all(net, &gc_args);
49a18d86
MK
2431 now = jiffies;
2432 net->ipv6.ip6_rt_last_gc = now;
1da177e4
LT
2433
2434 if (gc_args.more)
c8a45222 2435 mod_timer(&net->ipv6.ip6_fib_timer,
49a18d86 2436 round_jiffies(now
c8a45222 2437 + net->ipv6.sysctl.ip6_rt_gc_interval));
417f28bb 2438 else
8fa7292f 2439 timer_delete(&net->ipv6.ip6_fib_timer);
3dc94f93 2440 spin_unlock_bh(&net->ipv6.fib6_gc_lock);
1da177e4
LT
2441}
2442
86cb30ec 2443static void fib6_gc_timer_cb(struct timer_list *t)
5b7c931d 2444{
41cb0855 2445 struct net *arg = timer_container_of(arg, t, ipv6.ip6_fib_timer);
86cb30ec
KC
2446
2447 fib6_run_gc(0, arg, true);
5b7c931d
DL
2448}
2449
2c8c1e72 2450static int __net_init fib6_net_init(struct net *net)
1da177e4 2451{
10da66f7 2452 size_t size = sizeof(struct hlist_head) * FIB6_TABLE_HASHSZ;
16ab6d7d
IS
2453 int err;
2454
2455 err = fib6_notifier_init(net);
2456 if (err)
2457 return err;
10da66f7 2458
ed13923f
IS
2459 /* Default to 3-tuple */
2460 net->ipv6.sysctl.multipath_hash_fields =
2461 FIB_MULTIPATH_HASH_FIELD_DEFAULT_MASK;
2462
3dc94f93 2463 spin_lock_init(&net->ipv6.fib6_gc_lock);
9a03cd8f
MK
2464 rwlock_init(&net->ipv6.fib6_walker_lock);
2465 INIT_LIST_HEAD(&net->ipv6.fib6_walkers);
86cb30ec 2466 timer_setup(&net->ipv6.ip6_fib_timer, fib6_gc_timer_cb, 0);
63152fc0 2467
c572872f
BT
2468 net->ipv6.rt6_stats = kzalloc(sizeof(*net->ipv6.rt6_stats), GFP_KERNEL);
2469 if (!net->ipv6.rt6_stats)
67db5ca7 2470 goto out_notifier;
c572872f 2471
10da66f7
ED
2472 /* Avoid false sharing : Use at least a full cache line */
2473 size = max_t(size_t, size, L1_CACHE_BYTES);
2474
2475 net->ipv6.fib_table_hash = kzalloc(size, GFP_KERNEL);
58f09b78 2476 if (!net->ipv6.fib_table_hash)
c572872f 2477 goto out_rt6_stats;
e0b85590 2478
834d9784
KI
2479 spin_lock_init(&net->ipv6.fib_table_hash_lock);
2480
58f09b78
DL
2481 net->ipv6.fib6_main_tbl = kzalloc(sizeof(*net->ipv6.fib6_main_tbl),
2482 GFP_KERNEL);
2483 if (!net->ipv6.fib6_main_tbl)
e0b85590
DL
2484 goto out_fib_table_hash;
2485
58f09b78 2486 net->ipv6.fib6_main_tbl->tb6_id = RT6_TABLE_MAIN;
66f5d6ce 2487 rcu_assign_pointer(net->ipv6.fib6_main_tbl->tb6_root.leaf,
421842ed 2488 net->ipv6.fib6_null_entry);
58f09b78
DL
2489 net->ipv6.fib6_main_tbl->tb6_root.fn_flags =
2490 RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
8e773277 2491 inet_peer_base_init(&net->ipv6.fib6_main_tbl->tb6_peers);
5eb902b8 2492 INIT_HLIST_HEAD(&net->ipv6.fib6_main_tbl->tb6_gc_hlist);
e0b85590
DL
2493
2494#ifdef CONFIG_IPV6_MULTIPLE_TABLES
58f09b78
DL
2495 net->ipv6.fib6_local_tbl = kzalloc(sizeof(*net->ipv6.fib6_local_tbl),
2496 GFP_KERNEL);
2497 if (!net->ipv6.fib6_local_tbl)
e0b85590 2498 goto out_fib6_main_tbl;
58f09b78 2499 net->ipv6.fib6_local_tbl->tb6_id = RT6_TABLE_LOCAL;
66f5d6ce 2500 rcu_assign_pointer(net->ipv6.fib6_local_tbl->tb6_root.leaf,
421842ed 2501 net->ipv6.fib6_null_entry);
58f09b78
DL
2502 net->ipv6.fib6_local_tbl->tb6_root.fn_flags =
2503 RTN_ROOT | RTN_TL_ROOT | RTN_RTINFO;
8e773277 2504 inet_peer_base_init(&net->ipv6.fib6_local_tbl->tb6_peers);
5eb902b8 2505 INIT_HLIST_HEAD(&net->ipv6.fib6_local_tbl->tb6_gc_hlist);
e0b85590 2506#endif
58f09b78 2507 fib6_tables_init(net);
f845ab6b 2508
417f28bb 2509 return 0;
d63bddbe 2510
e0b85590 2511#ifdef CONFIG_IPV6_MULTIPLE_TABLES
e0b85590 2512out_fib6_main_tbl:
58f09b78 2513 kfree(net->ipv6.fib6_main_tbl);
e0b85590 2514#endif
e0b85590 2515out_fib_table_hash:
58f09b78 2516 kfree(net->ipv6.fib_table_hash);
c572872f
BT
2517out_rt6_stats:
2518 kfree(net->ipv6.rt6_stats);
67db5ca7 2519out_notifier:
16ab6d7d 2520 fib6_notifier_exit(net);
417f28bb 2521 return -ENOMEM;
8db46f1d 2522}
58f09b78
DL
2523
2524static void fib6_net_exit(struct net *net)
2525{
ba1cc08d
SD
2526 unsigned int i;
2527
8fa7292f 2528 timer_delete_sync(&net->ipv6.ip6_fib_timer);
417f28bb 2529
32a805ba 2530 for (i = 0; i < FIB6_TABLE_HASHSZ; i++) {
ba1cc08d
SD
2531 struct hlist_head *head = &net->ipv6.fib_table_hash[i];
2532 struct hlist_node *tmp;
2533 struct fib6_table *tb;
2534
2535 hlist_for_each_entry_safe(tb, tmp, head, tb6_hlist) {
2536 hlist_del(&tb->tb6_hlist);
2537 fib6_free_table(tb);
2538 }
2539 }
2540
58f09b78 2541 kfree(net->ipv6.fib_table_hash);
c572872f 2542 kfree(net->ipv6.rt6_stats);
16ab6d7d 2543 fib6_notifier_exit(net);
58f09b78
DL
2544}
2545
2546static struct pernet_operations fib6_net_ops = {
2547 .init = fib6_net_init,
2548 .exit = fib6_net_exit,
2549};
2550
a37b0e4e
KI
2551static const struct rtnl_msg_handler fib6_rtnl_msg_handlers[] __initconst_or_module = {
2552 {.owner = THIS_MODULE, .protocol = PF_INET6, .msgtype = RTM_GETROUTE,
2553 .dumpit = inet6_dump_fib,
2554 .flags = RTNL_FLAG_DUMP_UNLOCKED | RTNL_FLAG_DUMP_SPLIT_NLM_DONE},
2555};
2556
58f09b78
DL
2557int __init fib6_init(void)
2558{
2559 int ret = -ENOMEM;
63152fc0 2560
072f88ca
KC
2561 fib6_node_kmem = KMEM_CACHE(fib6_node,
2562 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT);
58f09b78
DL
2563 if (!fib6_node_kmem)
2564 goto out;
2565
2566 ret = register_pernet_subsys(&fib6_net_ops);
2567 if (ret)
c572872f 2568 goto out_kmem_cache_create;
e8803b6c 2569
a37b0e4e 2570 ret = rtnl_register_many(fib6_rtnl_msg_handlers);
e8803b6c
DM
2571 if (ret)
2572 goto out_unregister_subsys;
705f1c86
HFS
2573
2574 __fib6_flush_trees = fib6_flush_trees;
58f09b78
DL
2575out:
2576 return ret;
2577
e8803b6c
DM
2578out_unregister_subsys:
2579 unregister_pernet_subsys(&fib6_net_ops);
d63bddbe
DL
2580out_kmem_cache_create:
2581 kmem_cache_destroy(fib6_node_kmem);
2582 goto out;
1da177e4
LT
2583}
2584
2585void fib6_gc_cleanup(void)
2586{
58f09b78 2587 unregister_pernet_subsys(&fib6_net_ops);
1da177e4
LT
2588 kmem_cache_destroy(fib6_node_kmem);
2589}
8d2ca1d7
HFS
2590
2591#ifdef CONFIG_PROC_FS
138d0be3 2592static int ipv6_route_native_seq_show(struct seq_file *seq, void *v)
8d2ca1d7 2593{
8d1c802b 2594 struct fib6_info *rt = v;
8d2ca1d7 2595 struct ipv6_route_iter *iter = seq->private;
f88d8ea6 2596 struct fib6_nh *fib6_nh = rt->fib6_nh;
2b2450ca 2597 unsigned int flags = rt->fib6_flags;
5e670d84 2598 const struct net_device *dev;
8d2ca1d7 2599
f88d8ea6 2600 if (rt->nh)
ef1148d4 2601 fib6_nh = nexthop_fib6_nh(rt->nh);
f88d8ea6 2602
93c2fb25 2603 seq_printf(seq, "%pi6 %02x ", &rt->fib6_dst.addr, rt->fib6_dst.plen);
8d2ca1d7
HFS
2604
2605#ifdef CONFIG_IPV6_SUBTREES
93c2fb25 2606 seq_printf(seq, "%pi6 %02x ", &rt->fib6_src.addr, rt->fib6_src.plen);
8d2ca1d7
HFS
2607#else
2608 seq_puts(seq, "00000000000000000000000000000000 00 ");
2609#endif
f88d8ea6 2610 if (fib6_nh->fib_nh_gw_family) {
2b2450ca 2611 flags |= RTF_GATEWAY;
f88d8ea6 2612 seq_printf(seq, "%pi6", &fib6_nh->fib_nh_gw6);
2b2450ca 2613 } else {
8d2ca1d7 2614 seq_puts(seq, "00000000000000000000000000000000");
2b2450ca 2615 }
8d2ca1d7 2616
f88d8ea6 2617 dev = fib6_nh->fib_nh_dev;
8d2ca1d7 2618 seq_printf(seq, " %08x %08x %08x %08x %8s\n",
f05713e0 2619 rt->fib6_metric, refcount_read(&rt->fib6_ref), 0,
2b2450ca 2620 flags, dev ? dev->name : "");
8d2ca1d7
HFS
2621 iter->w.leaf = NULL;
2622 return 0;
2623}
2624
94b2cfe0 2625static int ipv6_route_yield(struct fib6_walker *w)
8d2ca1d7
HFS
2626{
2627 struct ipv6_route_iter *iter = w->args;
2628
2629 if (!iter->skip)
2630 return 1;
2631
2632 do {
66f5d6ce 2633 iter->w.leaf = rcu_dereference_protected(
8fb11a9a 2634 iter->w.leaf->fib6_next,
66f5d6ce 2635 lockdep_is_held(&iter->tbl->tb6_lock));
8d2ca1d7
HFS
2636 iter->skip--;
2637 if (!iter->skip && iter->w.leaf)
2638 return 1;
2639 } while (iter->w.leaf);
2640
2641 return 0;
2642}
2643
9a03cd8f
MK
2644static void ipv6_route_seq_setup_walk(struct ipv6_route_iter *iter,
2645 struct net *net)
8d2ca1d7
HFS
2646{
2647 memset(&iter->w, 0, sizeof(iter->w));
2648 iter->w.func = ipv6_route_yield;
2649 iter->w.root = &iter->tbl->tb6_root;
2650 iter->w.state = FWS_INIT;
2651 iter->w.node = iter->w.root;
2652 iter->w.args = iter;
aafc2e32 2653 iter->sernum = READ_ONCE(iter->w.root->fn_sernum);
8d2ca1d7 2654 INIT_LIST_HEAD(&iter->w.lh);
9a03cd8f 2655 fib6_walker_link(net, &iter->w);
8d2ca1d7
HFS
2656}
2657
2658static struct fib6_table *ipv6_route_seq_next_table(struct fib6_table *tbl,
2659 struct net *net)
2660{
2661 unsigned int h;
2662 struct hlist_node *node;
2663
2664 if (tbl) {
2665 h = (tbl->tb6_id & (FIB6_TABLE_HASHSZ - 1)) + 1;
ef1148d4 2666 node = rcu_dereference(hlist_next_rcu(&tbl->tb6_hlist));
8d2ca1d7
HFS
2667 } else {
2668 h = 0;
2669 node = NULL;
2670 }
2671
2672 while (!node && h < FIB6_TABLE_HASHSZ) {
ef1148d4 2673 node = rcu_dereference(
8d2ca1d7
HFS
2674 hlist_first_rcu(&net->ipv6.fib_table_hash[h++]));
2675 }
2676 return hlist_entry_safe(node, struct fib6_table, tb6_hlist);
2677}
2678
0a67d3ef
HFS
2679static void ipv6_route_check_sernum(struct ipv6_route_iter *iter)
2680{
aafc2e32
ED
2681 int sernum = READ_ONCE(iter->w.root->fn_sernum);
2682
2683 if (iter->sernum != sernum) {
2684 iter->sernum = sernum;
0a67d3ef
HFS
2685 iter->w.state = FWS_INIT;
2686 iter->w.node = iter->w.root;
2687 WARN_ON(iter->w.skip);
2688 iter->w.skip = iter->w.count;
2689 }
2690}
2691
8d2ca1d7
HFS
2692static void *ipv6_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2693{
2694 int r;
8d1c802b 2695 struct fib6_info *n;
8d2ca1d7
HFS
2696 struct net *net = seq_file_net(seq);
2697 struct ipv6_route_iter *iter = seq->private;
2698
4fc427e0 2699 ++(*pos);
8d2ca1d7
HFS
2700 if (!v)
2701 goto iter_table;
2702
ef1148d4 2703 n = rcu_dereference(((struct fib6_info *)v)->fib6_next);
4fc427e0 2704 if (n)
8d2ca1d7 2705 return n;
8d2ca1d7
HFS
2706
2707iter_table:
0a67d3ef 2708 ipv6_route_check_sernum(iter);
66f5d6ce 2709 spin_lock_bh(&iter->tbl->tb6_lock);
8d2ca1d7 2710 r = fib6_walk_continue(&iter->w);
66f5d6ce 2711 spin_unlock_bh(&iter->tbl->tb6_lock);
8d2ca1d7 2712 if (r > 0) {
8d2ca1d7
HFS
2713 return iter->w.leaf;
2714 } else if (r < 0) {
9a03cd8f 2715 fib6_walker_unlink(net, &iter->w);
8d2ca1d7
HFS
2716 return NULL;
2717 }
9a03cd8f 2718 fib6_walker_unlink(net, &iter->w);
8d2ca1d7
HFS
2719
2720 iter->tbl = ipv6_route_seq_next_table(iter->tbl, net);
2721 if (!iter->tbl)
2722 return NULL;
2723
9a03cd8f 2724 ipv6_route_seq_setup_walk(iter, net);
8d2ca1d7
HFS
2725 goto iter_table;
2726}
2727
2728static void *ipv6_route_seq_start(struct seq_file *seq, loff_t *pos)
ef1148d4 2729 __acquires(RCU)
8d2ca1d7
HFS
2730{
2731 struct net *net = seq_file_net(seq);
2732 struct ipv6_route_iter *iter = seq->private;
2733
ef1148d4 2734 rcu_read_lock();
8d2ca1d7
HFS
2735 iter->tbl = ipv6_route_seq_next_table(NULL, net);
2736 iter->skip = *pos;
2737
2738 if (iter->tbl) {
6617dfd4
YS
2739 loff_t p = 0;
2740
9a03cd8f 2741 ipv6_route_seq_setup_walk(iter, net);
6617dfd4 2742 return ipv6_route_seq_next(seq, NULL, &p);
8d2ca1d7
HFS
2743 } else {
2744 return NULL;
2745 }
2746}
2747
2748static bool ipv6_route_iter_active(struct ipv6_route_iter *iter)
2749{
94b2cfe0 2750 struct fib6_walker *w = &iter->w;
8d2ca1d7
HFS
2751 return w->node && !(w->state == FWS_U && w->node == w->root);
2752}
2753
138d0be3 2754static void ipv6_route_native_seq_stop(struct seq_file *seq, void *v)
ef1148d4 2755 __releases(RCU)
8d2ca1d7 2756{
9a03cd8f 2757 struct net *net = seq_file_net(seq);
8d2ca1d7
HFS
2758 struct ipv6_route_iter *iter = seq->private;
2759
2760 if (ipv6_route_iter_active(iter))
9a03cd8f 2761 fib6_walker_unlink(net, &iter->w);
8d2ca1d7 2762
ef1148d4 2763 rcu_read_unlock();
8d2ca1d7
HFS
2764}
2765
138d0be3 2766#if IS_BUILTIN(CONFIG_IPV6) && defined(CONFIG_BPF_SYSCALL)
138d0be3
YS
2767static int ipv6_route_prog_seq_show(struct bpf_prog *prog,
2768 struct bpf_iter_meta *meta,
2769 void *v)
2770{
2771 struct bpf_iter__ipv6_route ctx;
2772
2773 ctx.meta = meta;
2774 ctx.rt = v;
2775 return bpf_iter_run_prog(prog, &ctx);
2776}
2777
2778static int ipv6_route_seq_show(struct seq_file *seq, void *v)
2779{
2780 struct ipv6_route_iter *iter = seq->private;
2781 struct bpf_iter_meta meta;
2782 struct bpf_prog *prog;
2783 int ret;
2784
2785 meta.seq = seq;
2786 prog = bpf_iter_get_info(&meta, false);
2787 if (!prog)
2788 return ipv6_route_native_seq_show(seq, v);
2789
2790 ret = ipv6_route_prog_seq_show(prog, &meta, v);
2791 iter->w.leaf = NULL;
2792
2793 return ret;
2794}
2795
2796static void ipv6_route_seq_stop(struct seq_file *seq, void *v)
2797{
2798 struct bpf_iter_meta meta;
2799 struct bpf_prog *prog;
2800
2801 if (!v) {
2802 meta.seq = seq;
2803 prog = bpf_iter_get_info(&meta, true);
2804 if (prog)
2805 (void)ipv6_route_prog_seq_show(prog, &meta, v);
2806 }
2807
2808 ipv6_route_native_seq_stop(seq, v);
2809}
2810#else
2811static int ipv6_route_seq_show(struct seq_file *seq, void *v)
2812{
2813 return ipv6_route_native_seq_show(seq, v);
2814}
2815
2816static void ipv6_route_seq_stop(struct seq_file *seq, void *v)
2817{
2818 ipv6_route_native_seq_stop(seq, v);
2819}
2820#endif
2821
c3506372 2822const struct seq_operations ipv6_route_seq_ops = {
8d2ca1d7
HFS
2823 .start = ipv6_route_seq_start,
2824 .next = ipv6_route_seq_next,
2825 .stop = ipv6_route_seq_stop,
2826 .show = ipv6_route_seq_show
2827};
8d2ca1d7 2828#endif /* CONFIG_PROC_FS */