projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
inet: Remove ->get_peer() method.
[linux-2.6-block.git] / net / ipv4 / route.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * ROUTE - implementation of the IP router.
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 * Linus Torvalds, <Linus.Torvalds@helsinki.fi>
12 * Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 *
14 * Fixes:
15 * Alan Cox : Verify area fixes.
16 * Alan Cox : cli() protects routing changes
17 * Rui Oliveira : ICMP routing table updates
18 * (rco@di.uminho.pt) Routing table insertion and update
19 * Linus Torvalds : Rewrote bits to be sensible
20 * Alan Cox : Added BSD route gw semantics
e905a9ed 21 * Alan Cox : Super /proc >4K
1da177e4
LT		 22 * Alan Cox : MTU in route table
23 * Alan Cox : MSS actually. Also added the window
24 * clamper.
25 * Sam Lantinga : Fixed route matching in rt_del()
26 * Alan Cox : Routing cache support.
27 * Alan Cox : Removed compatibility cruft.
28 * Alan Cox : RTF_REJECT support.
29 * Alan Cox : TCP irtt support.
30 * Jonathan Naylor : Added Metric support.
31 * Miquel van Smoorenburg : BSD API fixes.
32 * Miquel van Smoorenburg : Metrics.
33 * Alan Cox : Use __u32 properly
34 * Alan Cox : Aligned routing errors more closely with BSD
35 * our system is still very different.
36 * Alan Cox : Faster /proc handling
37 * Alexey Kuznetsov : Massive rework to support tree based routing,
38 * routing caches and better behaviour.
e905a9ed 39 *
1da177e4
LT		 40 * Olaf Erb : irtt wasn't being copied right.
41 * Bjorn Ekwall : Kerneld route support.
42 * Alan Cox : Multicast fixed (I hope)
43 * Pavel Krauz : Limited broadcast fixed
44 * Mike McLagan : Routing by source
45 * Alexey Kuznetsov : End of old history. Split to fib.c and
46 * route.c and rewritten from scratch.
47 * Andi Kleen : Load-limit warning messages.
48 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
49 * Vitaly E. Lavrov : Race condition in ip_route_input_slow.
50 * Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
51 * Vladimir V. Ivanov : IP rule info (flowid) is really useful.
52 * Marc Boucher : routing by fwmark
53 * Robert Olsson : Added rt_cache statistics
54 * Arnaldo C. Melo : Convert proc stuff to seq_file
bb1d23b0 55 * Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
cef2685e
IS		 56 * Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
57 * Ilia Sotnikov : Removed TOS from hash calculations
1da177e4
LT		 58 *
59 * This program is free software; you can redistribute it and/or
60 * modify it under the terms of the GNU General Public License
61 * as published by the Free Software Foundation; either version
62 * 2 of the License, or (at your option) any later version.
63 */
64
afd46503
JP		 65#define pr_fmt(fmt) "IPv4: " fmt
66
1da177e4
LT		 67#include <linux/module.h>
68#include <asm/uaccess.h>
1da177e4
LT		 69#include <linux/bitops.h>
70#include <linux/types.h>
71#include <linux/kernel.h>
1da177e4 72#include <linux/mm.h>
424c4b70 73#include <linux/bootmem.h>
1da177e4
LT		 74#include <linux/string.h>
75#include <linux/socket.h>
76#include <linux/sockios.h>
77#include <linux/errno.h>
78#include <linux/in.h>
79#include <linux/inet.h>
80#include <linux/netdevice.h>
81#include <linux/proc_fs.h>
82#include <linux/init.h>
39c90ece 83#include <linux/workqueue.h>
1da177e4 84#include <linux/skbuff.h>
1da177e4
LT		 85#include <linux/inetdevice.h>
86#include <linux/igmp.h>
87#include <linux/pkt_sched.h>
88#include <linux/mroute.h>
89#include <linux/netfilter_ipv4.h>
90#include <linux/random.h>
91#include <linux/jhash.h>
92#include <linux/rcupdate.h>
93#include <linux/times.h>
5a0e3ad6 94#include <linux/slab.h>
b9eda06f 95#include <linux/prefetch.h>
352e512c 96#include <net/dst.h>
457c4cbc 97#include <net/net_namespace.h>
1da177e4
LT		 98#include <net/protocol.h>
99#include <net/ip.h>
100#include <net/route.h>
101#include <net/inetpeer.h>
102#include <net/sock.h>
103#include <net/ip_fib.h>
104#include <net/arp.h>
105#include <net/tcp.h>
106#include <net/icmp.h>
107#include <net/xfrm.h>
8d71740c 108#include <net/netevent.h>
63f3444f 109#include <net/rtnetlink.h>
1da177e4
LT		 110#ifdef CONFIG_SYSCTL
111#include <linux/sysctl.h>
7426a564 112#include <linux/kmemleak.h>
1da177e4 113#endif
6e5714ea 114#include <net/secure_seq.h>
1da177e4 115
68a5e3dd 116#define RT_FL_TOS(oldflp4) \
f61759e6 117 ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
1da177e4
LT		 118
119#define IP_MAX_MTU 0xFFF0
120
121#define RT_GC_TIMEOUT (300*HZ)
122
1da177e4 123static int ip_rt_max_size;
817bc4db 124static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
9f28a2fc 125static int ip_rt_gc_interval __read_mostly = 60 * HZ;
817bc4db
SH		 126static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
127static int ip_rt_redirect_number __read_mostly = 9;
128static int ip_rt_redirect_load __read_mostly = HZ / 50;
129static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
130static int ip_rt_error_cost __read_mostly = HZ;
131static int ip_rt_error_burst __read_mostly = 5 * HZ;
132static int ip_rt_gc_elasticity __read_mostly = 8;
133static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
134static int ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
135static int ip_rt_min_advmss __read_mostly = 256;
1080d709 136static int rt_chain_length_max __read_mostly = 20;
1da177e4 137
9f28a2fc
ED		 138static struct delayed_work expires_work;
139static unsigned long expires_ljiffies;
140
1da177e4
LT		 141/*
142 * Interface to generic destination cache.
143 */
144
145static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
0dbaee3b 146static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
ebb762f2 147static unsigned int ipv4_mtu(const struct dst_entry *dst);
1da177e4 148static void ipv4_dst_destroy(struct dst_entry *dst);
1da177e4
LT		 149static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
150static void ipv4_link_failure(struct sk_buff *skb);
151static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
569d3645 152static int rt_garbage_collect(struct dst_ops *ops);
1da177e4 153
72cdd1d9
ED		 154static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
155 int how)
156{
157}
1da177e4 158
62fa8a84
DM		 159static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
160{
06582540
DM		 161 struct rtable *rt = (struct rtable *) dst;
162 struct inet_peer *peer;
163 u32 *p = NULL;
164
fbfe95a4 165 peer = rt_get_peer_create(rt, rt->rt_dst);
06582540 166 if (peer) {
62fa8a84
DM		 167 u32 *old_p = __DST_METRICS_PTR(old);
168 unsigned long prev, new;
169
06582540
DM		 170 p = peer->metrics;
171 if (inet_metrics_new(peer))
172 memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
62fa8a84
DM		 173
174 new = (unsigned long) p;
175 prev = cmpxchg(&dst->_metrics, old, new);
176
177 if (prev != old) {
62fa8a84
DM		 178 p = __DST_METRICS_PTR(prev);
179 if (prev & DST_METRICS_READ_ONLY)
180 p = NULL;
181 } else {
62fa8a84
DM		 182 if (rt->fi) {
183 fib_info_put(rt->fi);
184 rt->fi = NULL;
185 }
186 }
187 }
188 return p;
189}
190
f894cbf8
DM		 191static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
192 struct sk_buff *skb,
193 const void *daddr);
d3aaeb38 194
1da177e4
LT		 195static struct dst_ops ipv4_dst_ops = {
196 .family = AF_INET,
09640e63 197 .protocol = cpu_to_be16(ETH_P_IP),
1da177e4
LT		 198 .gc = rt_garbage_collect,
199 .check = ipv4_dst_check,
0dbaee3b 200 .default_advmss = ipv4_default_advmss,
ebb762f2 201 .mtu = ipv4_mtu,
62fa8a84 202 .cow_metrics = ipv4_cow_metrics,
1da177e4
LT		 203 .destroy = ipv4_dst_destroy,
204 .ifdown = ipv4_dst_ifdown,
205 .negative_advice = ipv4_negative_advice,
206 .link_failure = ipv4_link_failure,
207 .update_pmtu = ip_rt_update_pmtu,
1ac06e03 208 .local_out = __ip_local_out,
d3aaeb38 209 .neigh_lookup = ipv4_neigh_lookup,
1da177e4
LT		 210};
211
212#define ECN_OR_COST(class) TC_PRIO_##class
213
4839c52b 214const __u8 ip_tos2prio[16] = {
1da177e4 215 TC_PRIO_BESTEFFORT,
4a2b9c37 216 ECN_OR_COST(BESTEFFORT),
1da177e4
LT		 217 TC_PRIO_BESTEFFORT,
218 ECN_OR_COST(BESTEFFORT),
219 TC_PRIO_BULK,
220 ECN_OR_COST(BULK),
221 TC_PRIO_BULK,
222 ECN_OR_COST(BULK),
223 TC_PRIO_INTERACTIVE,
224 ECN_OR_COST(INTERACTIVE),
225 TC_PRIO_INTERACTIVE,
226 ECN_OR_COST(INTERACTIVE),
227 TC_PRIO_INTERACTIVE_BULK,
228 ECN_OR_COST(INTERACTIVE_BULK),
229 TC_PRIO_INTERACTIVE_BULK,
230 ECN_OR_COST(INTERACTIVE_BULK)
231};
d4a96865 232EXPORT_SYMBOL(ip_tos2prio);
1da177e4
LT		 233
234/*
235 * Route cache.
236 */
237
238/* The locking scheme is rather straight forward:
239 *
240 * 1) Read-Copy Update protects the buckets of the central route hash.
241 * 2) Only writers remove entries, and they hold the lock
242 * as they look at rtable reference counts.
243 * 3) Only readers acquire references to rtable entries,
244 * they do so with atomic increments and with the
245 * lock held.
246 */
247
248struct rt_hash_bucket {
1c31720a 249 struct rtable __rcu *chain;
22c047cc 250};
1080d709 251
8a25d5de
IM		 252#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
253 defined(CONFIG_PROVE_LOCKING)
22c047cc
ED		 254/*
255 * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
256 * The size of this table is a power of two and depends on the number of CPUS.
62051200 257 * (on lockdep we have a quite big spinlock_t, so keep the size down there)
22c047cc 258 */
62051200
IM		 259#ifdef CONFIG_LOCKDEP
260# define RT_HASH_LOCK_SZ 256
22c047cc 261#else
62051200
IM		 262# if NR_CPUS >= 32
263# define RT_HASH_LOCK_SZ 4096
264# elif NR_CPUS >= 16
265# define RT_HASH_LOCK_SZ 2048
266# elif NR_CPUS >= 8
267# define RT_HASH_LOCK_SZ 1024
268# elif NR_CPUS >= 4
269# define RT_HASH_LOCK_SZ 512
270# else
271# define RT_HASH_LOCK_SZ 256
272# endif
22c047cc
ED		 273#endif
274
275static spinlock_t *rt_hash_locks;
276# define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
1ff1cc20
PE		 277
278static __init void rt_hash_lock_init(void)
279{
280 int i;
281
282 rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
283 GFP_KERNEL);
284 if (!rt_hash_locks)
285 panic("IP: failed to allocate rt_hash_locks\n");
286
287 for (i = 0; i < RT_HASH_LOCK_SZ; i++)
288 spin_lock_init(&rt_hash_locks[i]);
289}
22c047cc
ED		 290#else
291# define rt_hash_lock_addr(slot) NULL
1ff1cc20
PE		 292
293static inline void rt_hash_lock_init(void)
294{
295}
22c047cc 296#endif
1da177e4 297
817bc4db 298static struct rt_hash_bucket *rt_hash_table __read_mostly;
95c96174 299static unsigned int rt_hash_mask __read_mostly;
817bc4db 300static unsigned int rt_hash_log __read_mostly;
1da177e4 301
2f970d83 302static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
27f39c73 303#define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
1da177e4 304
b00180de 305static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
0eae88f3 306 int genid)
1da177e4 307{
0eae88f3 308 return jhash_3words((__force u32)daddr, (__force u32)saddr,
b00180de 309 idx, genid)
29e75252 310 & rt_hash_mask;
1da177e4
LT		 311}
312
e84f84f2
DL		 313static inline int rt_genid(struct net *net)
314{
315 return atomic_read(&net->ipv4.rt_genid);
316}
317
1da177e4
LT		 318#ifdef CONFIG_PROC_FS
319struct rt_cache_iter_state {
a75e936f 320 struct seq_net_private p;
1da177e4 321 int bucket;
29e75252 322 int genid;
1da177e4
LT		 323};
324
1218854a 325static struct rtable *rt_cache_get_first(struct seq_file *seq)
1da177e4 326{
1218854a 327 struct rt_cache_iter_state *st = seq->private;
1da177e4 328 struct rtable *r = NULL;
1da177e4
LT		 329
330 for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
33d480ce 331 if (!rcu_access_pointer(rt_hash_table[st->bucket].chain))
a6272665 332 continue;
1da177e4 333 rcu_read_lock_bh();
a898def2 334 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
29e75252 335 while (r) {
d8d1f30b 336 if (dev_net(r->dst.dev) == seq_file_net(seq) &&
a75e936f 337 r->rt_genid == st->genid)
29e75252 338 return r;
d8d1f30b 339 r = rcu_dereference_bh(r->dst.rt_next);
29e75252 340 }
1da177e4
LT		 341 rcu_read_unlock_bh();
342 }
29e75252 343 return r;
1da177e4
LT		 344}
345
1218854a 346static struct rtable *__rt_cache_get_next(struct seq_file *seq,
642d6318 347 struct rtable *r)
1da177e4 348{
1218854a 349 struct rt_cache_iter_state *st = seq->private;
a6272665 350
1c31720a 351 r = rcu_dereference_bh(r->dst.rt_next);
1da177e4
LT		 352 while (!r) {
353 rcu_read_unlock_bh();
a6272665
ED		 354 do {
355 if (--st->bucket < 0)
356 return NULL;
33d480ce 357 } while (!rcu_access_pointer(rt_hash_table[st->bucket].chain));
1da177e4 358 rcu_read_lock_bh();
1c31720a 359 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
1da177e4 360 }
1c31720a 361 return r;
1da177e4
LT		 362}
363
1218854a 364static struct rtable *rt_cache_get_next(struct seq_file *seq,
642d6318
DL		 365 struct rtable *r)
366{
1218854a
YH		 367 struct rt_cache_iter_state *st = seq->private;
368 while ((r = __rt_cache_get_next(seq, r)) != NULL) {
d8d1f30b 369 if (dev_net(r->dst.dev) != seq_file_net(seq))
a75e936f 370 continue;
642d6318
DL		 371 if (r->rt_genid == st->genid)
372 break;
373 }
374 return r;
375}
376
1218854a 377static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
1da177e4 378{
1218854a 379 struct rtable *r = rt_cache_get_first(seq);
1da177e4
LT		 380
381 if (r)
1218854a 382 while (pos && (r = rt_cache_get_next(seq, r)))
1da177e4
LT		 383 --pos;
384 return pos ? NULL : r;
385}
386
387static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
388{
29e75252 389 struct rt_cache_iter_state *st = seq->private;
29e75252 390 if (*pos)
1218854a 391 return rt_cache_get_idx(seq, *pos - 1);
e84f84f2 392 st->genid = rt_genid(seq_file_net(seq));
29e75252 393 return SEQ_START_TOKEN;
1da177e4
LT		 394}
395
396static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
397{
29e75252 398 struct rtable *r;
1da177e4
LT		 399
400 if (v == SEQ_START_TOKEN)
1218854a 401 r = rt_cache_get_first(seq);
1da177e4 402 else
1218854a 403 r = rt_cache_get_next(seq, v);
1da177e4
LT		 404 ++*pos;
405 return r;
406}
407
408static void rt_cache_seq_stop(struct seq_file *seq, void *v)
409{
410 if (v && v != SEQ_START_TOKEN)
411 rcu_read_unlock_bh();
412}
413
414static int rt_cache_seq_show(struct seq_file *seq, void *v)
415{
416 if (v == SEQ_START_TOKEN)
417 seq_printf(seq, "%-127s\n",
418 "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
419 "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
420 "HHUptod\tSpecDst");
421 else {
422 struct rtable *r = v;
3c521f2b 423 int len;
218fa90f 424
0eae88f3 425 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
794785bf
DM		 426 "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
427 r->dst.dev ? r->dst.dev->name : "*",
428 (__force u32)r->rt_dst,
429 (__force u32)r->rt_gateway,
430 r->rt_flags, atomic_read(&r->dst.__refcnt),
431 r->dst.__use, 0, (__force u32)r->rt_src,
432 dst_metric_advmss(&r->dst) + 40,
433 dst_metric(&r->dst, RTAX_WINDOW), 0,
434 r->rt_key_tos,
435 -1, 0, 0, &len);
5e659e4c
PE		 436
437 seq_printf(seq, "%*s\n", 127 - len, "");
e905a9ed
YH		 438 }
439 return 0;
1da177e4
LT		 440}
441
f690808e 442static const struct seq_operations rt_cache_seq_ops = {
1da177e4
LT		 443 .start = rt_cache_seq_start,
444 .next = rt_cache_seq_next,
445 .stop = rt_cache_seq_stop,
446 .show = rt_cache_seq_show,
447};
448
449static int rt_cache_seq_open(struct inode *inode, struct file *file)
450{
a75e936f 451 return seq_open_net(inode, file, &rt_cache_seq_ops,
cf7732e4 452 sizeof(struct rt_cache_iter_state));
1da177e4
LT		 453}
454
9a32144e 455static const struct file_operations rt_cache_seq_fops = {
1da177e4
LT		 456 .owner = THIS_MODULE,
457 .open = rt_cache_seq_open,
458 .read = seq_read,
459 .llseek = seq_lseek,
a75e936f 460 .release = seq_release_net,
1da177e4
LT		 461};
462
463
464static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
465{
466 int cpu;
467
468 if (*pos == 0)
469 return SEQ_START_TOKEN;
470
0f23174a 471 for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
1da177e4
LT		 472 if (!cpu_possible(cpu))
473 continue;
474 *pos = cpu+1;
2f970d83 475 return &per_cpu(rt_cache_stat, cpu);
1da177e4
LT		 476 }
477 return NULL;
478}
479
480static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
481{
482 int cpu;
483
0f23174a 484 for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
1da177e4
LT		 485 if (!cpu_possible(cpu))
486 continue;
487 *pos = cpu+1;
2f970d83 488 return &per_cpu(rt_cache_stat, cpu);
1da177e4
LT		 489 }
490 return NULL;
e905a9ed 491
1da177e4
LT		 492}
493
494static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
495{
496
497}
498
499static int rt_cpu_seq_show(struct seq_file *seq, void *v)
500{
501 struct rt_cache_stat *st = v;
502
503 if (v == SEQ_START_TOKEN) {
5bec0039 504 seq_printf(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
1da177e4
LT		 505 return 0;
506 }
e905a9ed 507
1da177e4
LT		 508 seq_printf(seq,"%08x %08x %08x %08x %08x %08x %08x %08x "
509 " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
fc66f95c 510 dst_entries_get_slow(&ipv4_dst_ops),
1da177e4
LT		 511 st->in_hit,
512 st->in_slow_tot,
513 st->in_slow_mc,
514 st->in_no_route,
515 st->in_brd,
516 st->in_martian_dst,
517 st->in_martian_src,
518
519 st->out_hit,
520 st->out_slow_tot,
e905a9ed 521 st->out_slow_mc,
1da177e4
LT		 522
523 st->gc_total,
524 st->gc_ignored,
525 st->gc_goal_miss,
526 st->gc_dst_overflow,
527 st->in_hlist_search,
528 st->out_hlist_search
529 );
530 return 0;
531}
532
f690808e 533static const struct seq_operations rt_cpu_seq_ops = {
1da177e4
LT		 534 .start = rt_cpu_seq_start,
535 .next = rt_cpu_seq_next,
536 .stop = rt_cpu_seq_stop,
537 .show = rt_cpu_seq_show,
538};
539
540
541static int rt_cpu_seq_open(struct inode *inode, struct file *file)
542{
543 return seq_open(file, &rt_cpu_seq_ops);
544}
545
9a32144e 546static const struct file_operations rt_cpu_seq_fops = {
1da177e4
LT		 547 .owner = THIS_MODULE,
548 .open = rt_cpu_seq_open,
549 .read = seq_read,
550 .llseek = seq_lseek,
551 .release = seq_release,
552};
553
c7066f70 554#ifdef CONFIG_IP_ROUTE_CLASSID
a661c419 555static int rt_acct_proc_show(struct seq_file *m, void *v)
78c686e9 556{
a661c419
AD		 557 struct ip_rt_acct *dst, *src;
558 unsigned int i, j;
559
560 dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
561 if (!dst)
562 return -ENOMEM;
563
564 for_each_possible_cpu(i) {
565 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
566 for (j = 0; j < 256; j++) {
567 dst[j].o_bytes += src[j].o_bytes;
568 dst[j].o_packets += src[j].o_packets;
569 dst[j].i_bytes += src[j].i_bytes;
570 dst[j].i_packets += src[j].i_packets;
571 }
78c686e9
PE		 572 }
573
a661c419
AD		 574 seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
575 kfree(dst);
576 return 0;
577}
78c686e9 578
a661c419
AD		 579static int rt_acct_proc_open(struct inode *inode, struct file *file)
580{
581 return single_open(file, rt_acct_proc_show, NULL);
78c686e9 582}
a661c419
AD		 583
584static const struct file_operations rt_acct_proc_fops = {
585 .owner = THIS_MODULE,
586 .open = rt_acct_proc_open,
587 .read = seq_read,
588 .llseek = seq_lseek,
589 .release = single_release,
590};
78c686e9 591#endif
107f1634 592
73b38711 593static int __net_init ip_rt_do_proc_init(struct net *net)
107f1634
PE		 594{
595 struct proc_dir_entry *pde;
596
597 pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
598 &rt_cache_seq_fops);
599 if (!pde)
600 goto err1;
601
77020720
WC		 602 pde = proc_create("rt_cache", S_IRUGO,
603 net->proc_net_stat, &rt_cpu_seq_fops);
107f1634
PE		 604 if (!pde)
605 goto err2;
606
c7066f70 607#ifdef CONFIG_IP_ROUTE_CLASSID
a661c419 608 pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
107f1634
PE		 609 if (!pde)
610 goto err3;
611#endif
612 return 0;
613
c7066f70 614#ifdef CONFIG_IP_ROUTE_CLASSID
107f1634
PE		 615err3:
616 remove_proc_entry("rt_cache", net->proc_net_stat);
617#endif
618err2:
619 remove_proc_entry("rt_cache", net->proc_net);
620err1:
621 return -ENOMEM;
622}
73b38711
DL		 623
624static void __net_exit ip_rt_do_proc_exit(struct net *net)
625{
626 remove_proc_entry("rt_cache", net->proc_net_stat);
627 remove_proc_entry("rt_cache", net->proc_net);
c7066f70 628#ifdef CONFIG_IP_ROUTE_CLASSID
73b38711 629 remove_proc_entry("rt_acct", net->proc_net);
0a931acf 630#endif
73b38711
DL		 631}
632
633static struct pernet_operations ip_rt_proc_ops __net_initdata = {
634 .init = ip_rt_do_proc_init,
635 .exit = ip_rt_do_proc_exit,
636};
637
638static int __init ip_rt_proc_init(void)
639{
640 return register_pernet_subsys(&ip_rt_proc_ops);
641}
642
107f1634 643#else
73b38711 644static inline int ip_rt_proc_init(void)
107f1634
PE		 645{
646 return 0;
647}
1da177e4 648#endif /* CONFIG_PROC_FS */
e905a9ed 649
5969f71d 650static inline void rt_free(struct rtable *rt)
1da177e4 651{
d8d1f30b 652 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
1da177e4
LT		 653}
654
5969f71d 655static inline void rt_drop(struct rtable *rt)
1da177e4 656{
1da177e4 657 ip_rt_put(rt);
d8d1f30b 658 call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
1da177e4
LT		 659}
660
5969f71d 661static inline int rt_fast_clean(struct rtable *rth)
1da177e4
LT		 662{
663 /* Kill broadcast/multicast entries very aggresively, if they
664 collide in hash table with more useful entries */
665 return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
c7537967 666 rt_is_input_route(rth) && rth->dst.rt_next;
1da177e4
LT		 667}
668
5969f71d 669static inline int rt_valuable(struct rtable *rth)
1da177e4
LT		 670{
671 return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
97bab73f 672 (rt_has_peer(rth) && rt_peer_ptr(rth)->pmtu_expires);
1da177e4
LT		 673}
674
675static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
676{
677 unsigned long age;
678 int ret = 0;
679
d8d1f30b 680 if (atomic_read(&rth->dst.__refcnt))
1da177e4
LT		 681 goto out;
682
d8d1f30b 683 age = jiffies - rth->dst.lastuse;
1da177e4
LT		 684 if ((age <= tmo1 && !rt_fast_clean(rth)) ||
685 (age <= tmo2 && rt_valuable(rth)))
686 goto out;
687 ret = 1;
688out: return ret;
689}
690
691/* Bits of score are:
692 * 31: very valuable
693 * 30: not quite useless
694 * 29..0: usage counter
695 */
696static inline u32 rt_score(struct rtable *rt)
697{
d8d1f30b 698 u32 score = jiffies - rt->dst.lastuse;
1da177e4
LT		 699
700 score = ~score & ~(3<<30);
701
702 if (rt_valuable(rt))
703 score |= (1<<31);
704
c7537967 705 if (rt_is_output_route(rt) ||
1da177e4
LT		 706 !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
707 score |= (1<<30);
708
709 return score;
710}
711
1080d709
NH		 712static inline bool rt_caching(const struct net *net)
713{
714 return net->ipv4.current_rt_cache_rebuild_count <=
715 net->ipv4.sysctl_rt_cache_rebuild_count;
716}
717
5e2b61f7
DM		 718static inline bool compare_hash_inputs(const struct rtable *rt1,
719 const struct rtable *rt2)
1080d709 720{
5e2b61f7
DM		 721 return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
722 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
97a80410 723 (rt1->rt_route_iif ^ rt2->rt_route_iif)) == 0);
1080d709
NH		 724}
725
5e2b61f7 726static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
1da177e4 727{
5e2b61f7
DM		 728 return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
729 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
730 (rt1->rt_mark ^ rt2->rt_mark) |
475949d8 731 (rt1->rt_key_tos ^ rt2->rt_key_tos) |
d547f727 732 (rt1->rt_route_iif ^ rt2->rt_route_iif) |
97a80410 733 (rt1->rt_oif ^ rt2->rt_oif)) == 0;
1da177e4
LT		 734}
735
b5921910
DL		 736static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
737{
d8d1f30b 738 return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
b5921910
DL		 739}
740
e84f84f2
DL		 741static inline int rt_is_expired(struct rtable *rth)
742{
d8d1f30b 743 return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
e84f84f2
DL		 744}
745
beb659bd
ED		 746/*
747 * Perform a full scan of hash table and free all entries.
748 * Can be called by a softirq or a process.
749 * In the later case, we want to be reschedule if necessary
750 */
6561a3b1 751static void rt_do_flush(struct net *net, int process_context)
beb659bd
ED		 752{
753 unsigned int i;
754 struct rtable *rth, *next;
755
756 for (i = 0; i <= rt_hash_mask; i++) {
6561a3b1
DM		 757 struct rtable __rcu **pprev;
758 struct rtable *list;
759
beb659bd
ED		 760 if (process_context && need_resched())
761 cond_resched();
33d480ce 762 rth = rcu_access_pointer(rt_hash_table[i].chain);
beb659bd
ED		 763 if (!rth)
764 continue;
765
766 spin_lock_bh(rt_hash_lock_addr(i));
32cb5b4e 767
6561a3b1
DM		 768 list = NULL;
769 pprev = &rt_hash_table[i].chain;
770 rth = rcu_dereference_protected(*pprev,
1c31720a 771 lockdep_is_held(rt_hash_lock_addr(i)));
32cb5b4e 772
6561a3b1
DM		 773 while (rth) {
774 next = rcu_dereference_protected(rth->dst.rt_next,
1c31720a 775 lockdep_is_held(rt_hash_lock_addr(i)));
6561a3b1
DM		 776
777 if (!net ||
778 net_eq(dev_net(rth->dst.dev), net)) {
779 rcu_assign_pointer(*pprev, next);
780 rcu_assign_pointer(rth->dst.rt_next, list);
781 list = rth;
32cb5b4e 782 } else {
6561a3b1 783 pprev = &rth->dst.rt_next;
32cb5b4e 784 }
6561a3b1 785 rth = next;
32cb5b4e 786 }
6561a3b1 787
beb659bd
ED		 788 spin_unlock_bh(rt_hash_lock_addr(i));
789
6561a3b1
DM		 790 for (; list; list = next) {
791 next = rcu_dereference_protected(list->dst.rt_next, 1);
792 rt_free(list);
beb659bd
ED		 793 }
794 }
795}
796
1080d709
NH		 797/*
798 * While freeing expired entries, we compute average chain length
799 * and standard deviation, using fixed-point arithmetic.
800 * This to have an estimation of rt_chain_length_max
801 * rt_chain_length_max = max(elasticity, AVG + 4*SD)
802 * We use 3 bits for frational part, and 29 (or 61) for magnitude.
803 */
804
805#define FRACT_BITS 3
806#define ONE (1UL << FRACT_BITS)
807
98376387
ED		 808/*
809 * Given a hash chain and an item in this hash chain,
810 * find if a previous entry has the same hash_inputs
811 * (but differs on tos, mark or oif)
812 * Returns 0 if an alias is found.
813 * Returns ONE if rth has no alias before itself.
814 */
815static int has_noalias(const struct rtable *head, const struct rtable *rth)
816{
817 const struct rtable *aux = head;
818
819 while (aux != rth) {
5e2b61f7 820 if (compare_hash_inputs(aux, rth))
98376387 821 return 0;
1c31720a 822 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
98376387
ED		 823 }
824 return ONE;
825}
826
9f28a2fc
ED		 827static void rt_check_expire(void)
828{
829 static unsigned int rover;
830 unsigned int i = rover, goal;
831 struct rtable *rth;
832 struct rtable __rcu **rthp;
833 unsigned long samples = 0;
834 unsigned long sum = 0, sum2 = 0;
835 unsigned long delta;
836 u64 mult;
837
838 delta = jiffies - expires_ljiffies;
839 expires_ljiffies = jiffies;
840 mult = ((u64)delta) << rt_hash_log;
841 if (ip_rt_gc_timeout > 1)
842 do_div(mult, ip_rt_gc_timeout);
843 goal = (unsigned int)mult;
844 if (goal > rt_hash_mask)
845 goal = rt_hash_mask + 1;
846 for (; goal > 0; goal--) {
847 unsigned long tmo = ip_rt_gc_timeout;
848 unsigned long length;
849
850 i = (i + 1) & rt_hash_mask;
851 rthp = &rt_hash_table[i].chain;
852
853 if (need_resched())
854 cond_resched();
855
856 samples++;
857
858 if (rcu_dereference_raw(*rthp) == NULL)
859 continue;
860 length = 0;
861 spin_lock_bh(rt_hash_lock_addr(i));
862 while ((rth = rcu_dereference_protected(*rthp,
863 lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
864 prefetch(rth->dst.rt_next);
df67e6c9
DM		 865 if (rt_is_expired(rth) ||
866 rt_may_expire(rth, tmo, ip_rt_gc_timeout)) {
9f28a2fc
ED		 867 *rthp = rth->dst.rt_next;
868 rt_free(rth);
869 continue;
870 }
df67e6c9
DM		 871
872 /* We only count entries on a chain with equal
873 * hash inputs once so that entries for
874 * different QOS levels, and other non-hash
875 * input attributes don't unfairly skew the
876 * length computation
877 */
878 tmo >>= 1;
879 rthp = &rth->dst.rt_next;
880 length += has_noalias(rt_hash_table[i].chain, rth);
9f28a2fc
ED		 881 }
882 spin_unlock_bh(rt_hash_lock_addr(i));
883 sum += length;
884 sum2 += length*length;
885 }
886 if (samples) {
887 unsigned long avg = sum / samples;
888 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
889 rt_chain_length_max = max_t(unsigned long,
890 ip_rt_gc_elasticity,
891 (avg + 4*sd) >> FRACT_BITS);
892 }
893 rover = i;
894}
895
896/*
897 * rt_worker_func() is run in process context.
898 * we call rt_check_expire() to scan part of the hash table
899 */
900static void rt_worker_func(struct work_struct *work)
901{
902 rt_check_expire();
903 schedule_delayed_work(&expires_work, ip_rt_gc_interval);
904}
905
29e75252 906/*
25985edc 907 * Perturbation of rt_genid by a small quantity [1..256]
29e75252
ED		 908 * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
909 * many times (2^24) without giving recent rt_genid.
910 * Jenkins hash is strong enough that litle changes of rt_genid are OK.
1da177e4 911 */
86c657f6 912static void rt_cache_invalidate(struct net *net)
1da177e4 913{
29e75252 914 unsigned char shuffle;
1da177e4 915
29e75252 916 get_random_bytes(&shuffle, sizeof(shuffle));
e84f84f2 917 atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
b48c80ec 918 inetpeer_invalidate_family(AF_INET);
1da177e4
LT		 919}
920
29e75252
ED		 921/*
922 * delay < 0 : invalidate cache (fast : entries will be deleted later)
923 * delay >= 0 : invalidate & flush cache (can be long)
924 */
76e6ebfb 925void rt_cache_flush(struct net *net, int delay)
1da177e4 926{
86c657f6 927 rt_cache_invalidate(net);
29e75252 928 if (delay >= 0)
6561a3b1 929 rt_do_flush(net, !in_softirq());
1da177e4
LT		 930}
931
a5ee1551 932/* Flush previous cache invalidated entries from the cache */
6561a3b1 933void rt_cache_flush_batch(struct net *net)
a5ee1551 934{
6561a3b1 935 rt_do_flush(net, !in_softirq());
a5ee1551
EB		 936}
937
1080d709
NH		 938static void rt_emergency_hash_rebuild(struct net *net)
939{
e87cc472 940 net_warn_ratelimited("Route hash chain too long!\n");
3ee94372 941 rt_cache_invalidate(net);
1080d709
NH		 942}
943
1da177e4
LT		 944/*
945 Short description of GC goals.
946
947 We want to build algorithm, which will keep routing cache
948 at some equilibrium point, when number of aged off entries
949 is kept approximately equal to newly generated ones.
950
951 Current expiration strength is variable "expire".
952 We try to adjust it dynamically, so that if networking
953 is idle expires is large enough to keep enough of warm entries,
954 and when load increases it reduces to limit cache size.
955 */
956
569d3645 957static int rt_garbage_collect(struct dst_ops *ops)
1da177e4
LT		 958{
959 static unsigned long expire = RT_GC_TIMEOUT;
960 static unsigned long last_gc;
961 static int rover;
962 static int equilibrium;
1c31720a
ED		 963 struct rtable *rth;
964 struct rtable __rcu **rthp;
1da177e4
LT		 965 unsigned long now = jiffies;
966 int goal;
fc66f95c 967 int entries = dst_entries_get_fast(&ipv4_dst_ops);
1da177e4
LT		 968
969 /*
970 * Garbage collection is pretty expensive,
971 * do not make it too frequently.
972 */
973
974 RT_CACHE_STAT_INC(gc_total);
975
976 if (now - last_gc < ip_rt_gc_min_interval &&
fc66f95c 977 entries < ip_rt_max_size) {
1da177e4
LT		 978 RT_CACHE_STAT_INC(gc_ignored);
979 goto out;
980 }
981
fc66f95c 982 entries = dst_entries_get_slow(&ipv4_dst_ops);
1da177e4 983 /* Calculate number of entries, which we want to expire now. */
fc66f95c 984 goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
1da177e4
LT		 985 if (goal <= 0) {
986 if (equilibrium < ipv4_dst_ops.gc_thresh)
987 equilibrium = ipv4_dst_ops.gc_thresh;
fc66f95c 988 goal = entries - equilibrium;
1da177e4 989 if (goal > 0) {
b790cedd 990 equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
fc66f95c 991 goal = entries - equilibrium;
1da177e4
LT		 992 }
993 } else {
994 /* We are in dangerous area. Try to reduce cache really
995 * aggressively.
996 */
b790cedd 997 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
fc66f95c 998 equilibrium = entries - goal;
1da177e4
LT		 999 }
1000
1001 if (now - last_gc >= ip_rt_gc_min_interval)
1002 last_gc = now;
1003
1004 if (goal <= 0) {
1005 equilibrium += goal;
1006 goto work_done;
1007 }
1008
1009 do {
1010 int i, k;
1011
1012 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
1013 unsigned long tmo = expire;
1014
1015 k = (k + 1) & rt_hash_mask;
1016 rthp = &rt_hash_table[k].chain;
22c047cc 1017 spin_lock_bh(rt_hash_lock_addr(k));
1c31720a
ED		 1018 while ((rth = rcu_dereference_protected(*rthp,
1019 lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
e84f84f2 1020 if (!rt_is_expired(rth) &&
29e75252 1021 !rt_may_expire(rth, tmo, expire)) {
1da177e4 1022 tmo >>= 1;
d8d1f30b 1023 rthp = &rth->dst.rt_next;
1da177e4
LT		 1024 continue;
1025 }
d8d1f30b 1026 *rthp = rth->dst.rt_next;
1da177e4
LT		 1027 rt_free(rth);
1028 goal--;
1da177e4 1029 }
22c047cc 1030 spin_unlock_bh(rt_hash_lock_addr(k));
1da177e4
LT		 1031 if (goal <= 0)
1032 break;
1033 }
1034 rover = k;
1035
1036 if (goal <= 0)
1037 goto work_done;
1038
1039 /* Goal is not achieved. We stop process if:
1040
1041 - if expire reduced to zero. Otherwise, expire is halfed.
1042 - if table is not full.
1043 - if we are called from interrupt.
1044 - jiffies check is just fallback/debug loop breaker.
1045 We will not spin here for long time in any case.
1046 */
1047
1048 RT_CACHE_STAT_INC(gc_goal_miss);
1049
1050 if (expire == 0)
1051 break;
1052
1053 expire >>= 1;
1da177e4 1054
fc66f95c 1055 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1da177e4
LT		 1056 goto out;
1057 } while (!in_softirq() && time_before_eq(jiffies, now));
1058
fc66f95c
ED		 1059 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1060 goto out;
1061 if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
1da177e4 1062 goto out;
e87cc472 1063 net_warn_ratelimited("dst cache overflow\n");
1da177e4
LT		 1064 RT_CACHE_STAT_INC(gc_dst_overflow);
1065 return 1;
1066
1067work_done:
1068 expire += ip_rt_gc_min_interval;
1069 if (expire > ip_rt_gc_timeout ||
fc66f95c
ED		 1070 dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1071 dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
1da177e4 1072 expire = ip_rt_gc_timeout;
1da177e4
LT		 1073out: return 0;
1074}
1075
98376387
ED		 1076/*
1077 * Returns number of entries in a hash chain that have different hash_inputs
1078 */
1079static int slow_chain_length(const struct rtable *head)
1080{
1081 int length = 0;
1082 const struct rtable *rth = head;
1083
1084 while (rth) {
1085 length += has_noalias(head, rth);
1c31720a 1086 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
98376387
ED		 1087 }
1088 return length >> FRACT_BITS;
1089}
1090
f894cbf8
DM		 1091static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
1092 struct sk_buff *skb,
1093 const void *daddr)
3769cffb 1094{
d3aaeb38
DM		 1095 struct net_device *dev = dst->dev;
1096 const __be32 *pkey = daddr;
39232973 1097 const struct rtable *rt;
3769cffb
DM		 1098 struct neighbour *n;
1099
39232973 1100 rt = (const struct rtable *) dst;
a263b309 1101 if (rt->rt_gateway)
39232973 1102 pkey = (const __be32 *) &rt->rt_gateway;
f894cbf8
DM		 1103 else if (skb)
1104 pkey = &ip_hdr(skb)->daddr;
d3aaeb38 1105
80703d26 1106 n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
d3aaeb38
DM		 1107 if (n)
1108 return n;
32092ecf 1109 return neigh_create(&arp_tbl, pkey, dev);
d3aaeb38
DM		 1110}
1111
95c96174 1112static struct rtable *rt_intern_hash(unsigned int hash, struct rtable *rt,
b23dd4fe 1113 struct sk_buff *skb, int ifindex)
1da177e4 1114{
1c31720a
ED		 1115 struct rtable *rth, *cand;
1116 struct rtable __rcu **rthp, **candp;
1da177e4 1117 unsigned long now;
1da177e4
LT		 1118 u32 min_score;
1119 int chain_length;
1da177e4
LT		 1120
1121restart:
1122 chain_length = 0;
1123 min_score = ~(u32)0;
1124 cand = NULL;
1125 candp = NULL;
1126 now = jiffies;
1127
7586eceb 1128 if (!rt_caching(dev_net(rt->dst.dev)) || (rt->dst.flags & DST_NOCACHE)) {
73e42897
NH		 1129 /*
1130 * If we're not caching, just tell the caller we
1131 * were successful and don't touch the route. The
1132 * caller hold the sole reference to the cache entry, and
1133 * it will be released when the caller is done with it.
1134 * If we drop it here, the callers have no way to resolve routes
1135 * when we're not caching. Instead, just point *rp at rt, so
1136 * the caller gets a single use out of the route
b6280b47
NH		 1137 * Note that we do rt_free on this new route entry, so that
1138 * once its refcount hits zero, we are still able to reap it
1139 * (Thanks Alexey)
27b75c95
ED		 1140 * Note: To avoid expensive rcu stuff for this uncached dst,
1141 * we set DST_NOCACHE so that dst_release() can free dst without
1142 * waiting a grace period.
73e42897 1143 */
b6280b47 1144
c7d4426a 1145 rt->dst.flags |= DST_NOCACHE;
b6280b47 1146 goto skip_hashing;
1080d709
NH		 1147 }
1148
1da177e4
LT		 1149 rthp = &rt_hash_table[hash].chain;
1150
22c047cc 1151 spin_lock_bh(rt_hash_lock_addr(hash));
1c31720a
ED		 1152 while ((rth = rcu_dereference_protected(*rthp,
1153 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
e84f84f2 1154 if (rt_is_expired(rth)) {
d8d1f30b 1155 *rthp = rth->dst.rt_next;
29e75252
ED		 1156 rt_free(rth);
1157 continue;
1158 }
5e2b61f7 1159 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
1da177e4 1160 /* Put it first */
d8d1f30b 1161 *rthp = rth->dst.rt_next;
1da177e4
LT		 1162 /*
1163 * Since lookup is lockfree, the deletion
1164 * must be visible to another weakly ordered CPU before
1165 * the insertion at the start of the hash chain.
1166 */
d8d1f30b 1167 rcu_assign_pointer(rth->dst.rt_next,
1da177e4
LT		 1168 rt_hash_table[hash].chain);
1169 /*
1170 * Since lookup is lockfree, the update writes
1171 * must be ordered for consistency on SMP.
1172 */
1173 rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1174
d8d1f30b 1175 dst_use(&rth->dst, now);
22c047cc 1176 spin_unlock_bh(rt_hash_lock_addr(hash));
1da177e4
LT		 1177
1178 rt_drop(rt);
b23dd4fe 1179 if (skb)
d8d1f30b 1180 skb_dst_set(skb, &rth->dst);
b23dd4fe 1181 return rth;
1da177e4
LT		 1182 }
1183
d8d1f30b 1184 if (!atomic_read(&rth->dst.__refcnt)) {
1da177e4
LT		 1185 u32 score = rt_score(rth);
1186
1187 if (score <= min_score) {
1188 cand = rth;
1189 candp = rthp;
1190 min_score = score;
1191 }
1192 }
1193
1194 chain_length++;
1195
d8d1f30b 1196 rthp = &rth->dst.rt_next;
1da177e4
LT		 1197 }
1198
1199 if (cand) {
1200 /* ip_rt_gc_elasticity used to be average length of chain
1201 * length, when exceeded gc becomes really aggressive.
1202 *
1203 * The second limit is less certain. At the moment it allows
1204 * only 2 entries per bucket. We will see.
1205 */
1206 if (chain_length > ip_rt_gc_elasticity) {
d8d1f30b 1207 *candp = cand->dst.rt_next;
1da177e4
LT		 1208 rt_free(cand);
1209 }
1080d709 1210 } else {
98376387
ED		 1211 if (chain_length > rt_chain_length_max &&
1212 slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
d8d1f30b 1213 struct net *net = dev_net(rt->dst.dev);
1080d709 1214 int num = ++net->ipv4.current_rt_cache_rebuild_count;
b35ecb5d 1215 if (!rt_caching(net)) {
058bd4d2 1216 pr_warn("%s: %d rebuilds is over limit, route caching disabled\n",
d8d1f30b 1217 rt->dst.dev->name, num);
1080d709 1218 }
b35ecb5d 1219 rt_emergency_hash_rebuild(net);
6a2bad70
PE		 1220 spin_unlock_bh(rt_hash_lock_addr(hash));
1221
5e2b61f7 1222 hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
6a2bad70
PE		 1223 ifindex, rt_genid(net));
1224 goto restart;
1080d709 1225 }
1da177e4
LT		 1226 }
1227
d8d1f30b 1228 rt->dst.rt_next = rt_hash_table[hash].chain;
1080d709 1229
00269b54
ED		 1230 /*
1231 * Since lookup is lockfree, we must make sure
25985edc 1232 * previous writes to rt are committed to memory
00269b54
ED		 1233 * before making rt visible to other CPUS.
1234 */
1ddbcb00 1235 rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1080d709 1236
22c047cc 1237 spin_unlock_bh(rt_hash_lock_addr(hash));
73e42897 1238
b6280b47 1239skip_hashing:
b23dd4fe 1240 if (skb)
d8d1f30b 1241 skb_dst_set(skb, &rt->dst);
b23dd4fe 1242 return rt;
1da177e4
LT		 1243}
1244
6431cbc2
DM		 1245static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1246
1247static u32 rt_peer_genid(void)
1248{
1249 return atomic_read(&__rt_peer_genid);
1250}
1251
a48eff12 1252void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
1da177e4 1253{
97bab73f 1254 struct inet_peer_base *base;
1da177e4
LT		 1255 struct inet_peer *peer;
1256
97bab73f
DM		 1257 base = inetpeer_base_ptr(rt->_peer);
1258 if (!base)
1259 return;
1260
1261 peer = inet_getpeer_v4(base, daddr, create);
7b34ca2a
DM		 1262 if (peer) {
1263 if (!rt_set_peer(rt, peer))
1264 inet_putpeer(peer);
1265 else
1266 rt->rt_peer_genid = rt_peer_genid();
1267 }
1da177e4
LT		 1268}
1269
1270/*
1271 * Peer allocation may fail only in serious out-of-memory conditions. However
1272 * we still can generate some output.
1273 * Random ID selection looks a bit dangerous because we have no chances to
1274 * select ID being unique in a reasonable period of time.
1275 * But broken packet identifier may be better than no packet at all.
1276 */
1277static void ip_select_fb_ident(struct iphdr *iph)
1278{
1279 static DEFINE_SPINLOCK(ip_fb_id_lock);
1280 static u32 ip_fallback_id;
1281 u32 salt;
1282
1283 spin_lock_bh(&ip_fb_id_lock);
e448515c 1284 salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1da177e4
LT		 1285 iph->id = htons(salt & 0xFFFF);
1286 ip_fallback_id = salt;
1287 spin_unlock_bh(&ip_fb_id_lock);
1288}
1289
1290void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1291{
1292 struct rtable *rt = (struct rtable *) dst;
1293
e688a604 1294 if (rt && !(rt->dst.flags & DST_NOPEER)) {
fbfe95a4 1295 struct inet_peer *peer = rt_get_peer_create(rt, rt->rt_dst);
1da177e4
LT		 1296
1297 /* If peer is attached to destination, it is never detached,
1298 so that we need not to grab a lock to dereference it.
1299 */
fbfe95a4
DM		 1300 if (peer) {
1301 iph->id = htons(inet_getid(peer, more));
1da177e4
LT		 1302 return;
1303 }
e688a604 1304 } else if (!rt)
91df42be 1305 pr_debug("rt_bind_peer(0) @%p\n", __builtin_return_address(0));
1da177e4
LT		 1306
1307 ip_select_fb_ident(iph);
1308}
4bc2f18b 1309EXPORT_SYMBOL(__ip_select_ident);
1da177e4 1310
95c96174 1311static void rt_del(unsigned int hash, struct rtable *rt)
1da177e4 1312{
1c31720a
ED		 1313 struct rtable __rcu **rthp;
1314 struct rtable *aux;
1da177e4 1315
29e75252 1316 rthp = &rt_hash_table[hash].chain;
22c047cc 1317 spin_lock_bh(rt_hash_lock_addr(hash));
1da177e4 1318 ip_rt_put(rt);
1c31720a
ED		 1319 while ((aux = rcu_dereference_protected(*rthp,
1320 lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
e84f84f2 1321 if (aux == rt || rt_is_expired(aux)) {
d8d1f30b 1322 *rthp = aux->dst.rt_next;
29e75252
ED		 1323 rt_free(aux);
1324 continue;
1da177e4 1325 }
d8d1f30b 1326 rthp = &aux->dst.rt_next;
29e75252 1327 }
22c047cc 1328 spin_unlock_bh(rt_hash_lock_addr(hash));
1da177e4
LT		 1329}
1330
de398fb8 1331static void check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
9cc20b26
ED		 1332{
1333 struct rtable *rt = (struct rtable *) dst;
1334 __be32 orig_gw = rt->rt_gateway;
f187bc6e 1335 struct neighbour *n;
9cc20b26
ED		 1336
1337 dst_confirm(&rt->dst);
1338
1339 rt->rt_gateway = peer->redirect_learned.a4;
1340
f894cbf8 1341 n = ipv4_neigh_lookup(&rt->dst, NULL, &rt->rt_gateway);
f187bc6e 1342 if (!n) {
de398fb8
DM		 1343 rt->rt_gateway = orig_gw;
1344 return;
1345 }
de398fb8
DM		 1346 if (!(n->nud_state & NUD_VALID)) {
1347 neigh_event_send(n, NULL);
9cc20b26
ED		 1348 } else {
1349 rt->rt_flags |= RTCF_REDIRECTED;
1350 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
1351 }
f187bc6e 1352 neigh_release(n);
9cc20b26
ED		 1353}
1354
ed7865a4 1355/* called in rcu_read_lock() section */
f7655229
AV		 1356void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1357 __be32 saddr, struct net_device *dev)
1da177e4 1358{
7cc9150e 1359 int s, i;
ed7865a4 1360 struct in_device *in_dev = __in_dev_get_rcu(dev);
7cc9150e
FL		 1361 __be32 skeys[2] = { saddr, 0 };
1362 int ikeys[2] = { dev->ifindex, 0 };
f39925db 1363 struct inet_peer *peer;
317805b8 1364 struct net *net;
1da177e4 1365
1da177e4
LT		 1366 if (!in_dev)
1367 return;
1368
c346dca1 1369 net = dev_net(dev);
9d4fb27d
JP		 1370 if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1371 ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1372 ipv4_is_zeronet(new_gw))
1da177e4
LT		 1373 goto reject_redirect;
1374
1375 if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1376 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1377 goto reject_redirect;
1378 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1379 goto reject_redirect;
1380 } else {
317805b8 1381 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1da177e4
LT		 1382 goto reject_redirect;
1383 }
1384
7cc9150e
FL		 1385 for (s = 0; s < 2; s++) {
1386 for (i = 0; i < 2; i++) {
9cc20b26
ED		 1387 unsigned int hash;
1388 struct rtable __rcu **rthp;
1389 struct rtable *rt;
1390
1391 hash = rt_hash(daddr, skeys[s], ikeys[i], rt_genid(net));
1392
1393 rthp = &rt_hash_table[hash].chain;
1394
1395 while ((rt = rcu_dereference(*rthp)) != NULL) {
1396 rthp = &rt->dst.rt_next;
1397
1398 if (rt->rt_key_dst != daddr ||
1399 rt->rt_key_src != skeys[s] ||
1400 rt->rt_oif != ikeys[i] ||
1401 rt_is_input_route(rt) ||
1402 rt_is_expired(rt) ||
1403 !net_eq(dev_net(rt->dst.dev), net) ||
1404 rt->dst.error ||
1405 rt->dst.dev != dev ||
1406 rt->rt_gateway != old_gw)
1407 continue;
e905a9ed 1408
fbfe95a4 1409 peer = rt_get_peer_create(rt, rt->rt_dst);
9cc20b26 1410 if (peer) {
ac3f48de 1411 if (peer->redirect_learned.a4 != new_gw) {
9cc20b26
ED		 1412 peer->redirect_learned.a4 = new_gw;
1413 atomic_inc(&__rt_peer_genid);
1414 }
1415 check_peer_redir(&rt->dst, peer);
1416 }
7cc9150e 1417 }
7cc9150e 1418 }
1da177e4 1419 }
1da177e4
LT		 1420 return;
1421
1422reject_redirect:
1423#ifdef CONFIG_IP_ROUTE_VERBOSE
e87cc472
JP		 1424 if (IN_DEV_LOG_MARTIANS(in_dev))
1425 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
1426 " Advised path = %pI4 -> %pI4\n",
1427 &old_gw, dev->name, &new_gw,
1428 &saddr, &daddr);
1da177e4 1429#endif
ed7865a4 1430 ;
1da177e4
LT		 1431}
1432
fe6fe792
ED		 1433static bool peer_pmtu_expired(struct inet_peer *peer)
1434{
1435 unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1436
1437 return orig &&
1438 time_after_eq(jiffies, orig) &&
1439 cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1440}
1441
1442static bool peer_pmtu_cleaned(struct inet_peer *peer)
1443{
1444 unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1445
1446 return orig &&
1447 cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1448}
1449
1da177e4
LT		 1450static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1451{
ee6b9673 1452 struct rtable *rt = (struct rtable *)dst;
1da177e4
LT		 1453 struct dst_entry *ret = dst;
1454
1455 if (rt) {
d11a4dc1 1456 if (dst->obsolete > 0) {
1da177e4
LT		 1457 ip_rt_put(rt);
1458 ret = NULL;
2c8cec5c 1459 } else if (rt->rt_flags & RTCF_REDIRECTED) {
95c96174 1460 unsigned int hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
5e2b61f7 1461 rt->rt_oif,
e84f84f2 1462 rt_genid(dev_net(dst->dev)));
1da177e4
LT		 1463 rt_del(hash, rt);
1464 ret = NULL;
97bab73f
DM		 1465 } else if (rt_has_peer(rt)) {
1466 struct inet_peer *peer = rt_peer_ptr(rt);
1467 if (peer_pmtu_expired(peer))
1468 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1da177e4
LT		 1469 }
1470 }
1471 return ret;
1472}
1473
1474/*
1475 * Algorithm:
1476 * 1. The first ip_rt_redirect_number redirects are sent
1477 * with exponential backoff, then we stop sending them at all,
1478 * assuming that the host ignores our redirects.
1479 * 2. If we did not see packets requiring redirects
1480 * during ip_rt_redirect_silence, we assume that the host
1481 * forgot redirected route and start to send redirects again.
1482 *
1483 * This algorithm is much cheaper and more intelligent than dumb load limiting
1484 * in icmp.c.
1485 *
1486 * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1487 * and "frag. need" (breaks PMTU discovery) in icmp.c.
1488 */
1489
1490void ip_rt_send_redirect(struct sk_buff *skb)
1491{
511c3f92 1492 struct rtable *rt = skb_rtable(skb);
30038fc6 1493 struct in_device *in_dev;
92d86829 1494 struct inet_peer *peer;
30038fc6 1495 int log_martians;
1da177e4 1496
30038fc6 1497 rcu_read_lock();
d8d1f30b 1498 in_dev = __in_dev_get_rcu(rt->dst.dev);
30038fc6
ED		 1499 if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1500 rcu_read_unlock();
1da177e4 1501 return;
30038fc6
ED		 1502 }
1503 log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1504 rcu_read_unlock();
1da177e4 1505
fbfe95a4 1506 peer = rt_get_peer_create(rt, rt->rt_dst);
92d86829
DM		 1507 if (!peer) {
1508 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1509 return;
1510 }
1511
1da177e4
LT		 1512 /* No redirected packets during ip_rt_redirect_silence;
1513 * reset the algorithm.
1514 */
92d86829
DM		 1515 if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1516 peer->rate_tokens = 0;
1da177e4
LT		 1517
1518 /* Too many ignored redirects; do not send anything
d8d1f30b 1519 * set dst.rate_last to the last seen redirected packet.
1da177e4 1520 */
92d86829
DM		 1521 if (peer->rate_tokens >= ip_rt_redirect_number) {
1522 peer->rate_last = jiffies;
30038fc6 1523 return;
1da177e4
LT		 1524 }
1525
1526 /* Check for load limit; set rate_last to the latest sent
1527 * redirect.
1528 */
92d86829 1529 if (peer->rate_tokens == 0 ||
14fb8a76 1530 time_after(jiffies,
92d86829
DM		 1531 (peer->rate_last +
1532 (ip_rt_redirect_load << peer->rate_tokens)))) {
1da177e4 1533 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
92d86829
DM		 1534 peer->rate_last = jiffies;
1535 ++peer->rate_tokens;
1da177e4 1536#ifdef CONFIG_IP_ROUTE_VERBOSE
30038fc6 1537 if (log_martians &&
e87cc472
JP		 1538 peer->rate_tokens == ip_rt_redirect_number)
1539 net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
1540 &ip_hdr(skb)->saddr, rt->rt_iif,
1541 &rt->rt_dst, &rt->rt_gateway);
1da177e4
LT		 1542#endif
1543 }
1da177e4
LT		 1544}
1545
1546static int ip_error(struct sk_buff *skb)
1547{
251da413 1548 struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
511c3f92 1549 struct rtable *rt = skb_rtable(skb);
92d86829 1550 struct inet_peer *peer;
1da177e4 1551 unsigned long now;
251da413 1552 struct net *net;
92d86829 1553 bool send;
1da177e4
LT		 1554 int code;
1555
251da413
DM		 1556 net = dev_net(rt->dst.dev);
1557 if (!IN_DEV_FORWARD(in_dev)) {
1558 switch (rt->dst.error) {
1559 case EHOSTUNREACH:
1560 IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
1561 break;
1562
1563 case ENETUNREACH:
1564 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1565 break;
1566 }
1567 goto out;
1568 }
1569
d8d1f30b 1570 switch (rt->dst.error) {
4500ebf8
JP		 1571 case EINVAL:
1572 default:
1573 goto out;
1574 case EHOSTUNREACH:
1575 code = ICMP_HOST_UNREACH;
1576 break;
1577 case ENETUNREACH:
1578 code = ICMP_NET_UNREACH;
251da413 1579 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
4500ebf8
JP		 1580 break;
1581 case EACCES:
1582 code = ICMP_PKT_FILTERED;
1583 break;
1da177e4
LT		 1584 }
1585
fbfe95a4 1586 peer = rt_get_peer_create(rt, rt->rt_dst);
92d86829
DM		 1587
1588 send = true;
1589 if (peer) {
1590 now = jiffies;
1591 peer->rate_tokens += now - peer->rate_last;
1592 if (peer->rate_tokens > ip_rt_error_burst)
1593 peer->rate_tokens = ip_rt_error_burst;
1594 peer->rate_last = now;
1595 if (peer->rate_tokens >= ip_rt_error_cost)
1596 peer->rate_tokens -= ip_rt_error_cost;
1597 else
1598 send = false;
1da177e4 1599 }
92d86829
DM		 1600 if (send)
1601 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1da177e4
LT		 1602
1603out: kfree_skb(skb);
1604 return 0;
e905a9ed 1605}
1da177e4 1606
2c8cec5c
DM		 1607static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1608{
fe6fe792 1609 unsigned long expires = ACCESS_ONCE(peer->pmtu_expires);
2c8cec5c 1610
fe6fe792
ED		 1611 if (!expires)
1612 return;
46af3180 1613 if (time_before(jiffies, expires)) {
2c8cec5c
DM		 1614 u32 orig_dst_mtu = dst_mtu(dst);
1615 if (peer->pmtu_learned < orig_dst_mtu) {
1616 if (!peer->pmtu_orig)
1617 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1618 dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1619 }
1620 } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1621 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1622}
1623
1da177e4
LT		 1624static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1625{
2c8cec5c
DM		 1626 struct rtable *rt = (struct rtable *) dst;
1627 struct inet_peer *peer;
1628
1629 dst_confirm(dst);
1630
fbfe95a4 1631 peer = rt_get_peer_create(rt, rt->rt_dst);
2c8cec5c 1632 if (peer) {
fe6fe792
ED		 1633 unsigned long pmtu_expires = ACCESS_ONCE(peer->pmtu_expires);
1634
2c8cec5c 1635 if (mtu < ip_rt_min_pmtu)
1da177e4 1636 mtu = ip_rt_min_pmtu;
fe6fe792 1637 if (!pmtu_expires || mtu < peer->pmtu_learned) {
46af3180
HS		 1638
1639 pmtu_expires = jiffies + ip_rt_mtu_expires;
1640 if (!pmtu_expires)
1641 pmtu_expires = 1UL;
1642
2c8cec5c 1643 peer->pmtu_learned = mtu;
46af3180 1644 peer->pmtu_expires = pmtu_expires;
2c8cec5c
DM		 1645
1646 atomic_inc(&__rt_peer_genid);
1647 rt->rt_peer_genid = rt_peer_genid();
1da177e4 1648 }
46af3180 1649 check_peer_pmtu(dst, peer);
1da177e4
LT		 1650 }
1651}
1652
36393395
DM		 1653void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1654 int oif, u32 mark, u8 protocol, int flow_flags)
1655{
1656 const struct iphdr *iph = (const struct iphdr *)skb->data;
1657 struct flowi4 fl4;
1658 struct rtable *rt;
1659
1660 flowi4_init_output(&fl4, oif, mark, RT_TOS(iph->tos), RT_SCOPE_UNIVERSE,
1661 protocol, flow_flags | FLOWI_FLAG_PRECOW_METRICS,
1662 iph->daddr, iph->saddr, 0, 0);
1663 rt = __ip_route_output_key(net, &fl4);
1664 if (!IS_ERR(rt)) {
1665 ip_rt_update_pmtu(&rt->dst, mtu);
1666 ip_rt_put(rt);
1667 }
1668}
1669EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1670
1671void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1672{
1673 const struct inet_sock *inet = inet_sk(sk);
1674
1675 return ipv4_update_pmtu(skb, sock_net(sk), mtu,
1676 sk->sk_bound_dev_if, sk->sk_mark,
1677 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
1678 inet_sk_flowi_flags(sk));
1679}
1680EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
f39925db 1681
de398fb8 1682static void ipv4_validate_peer(struct rtable *rt)
1da177e4 1683{
6431cbc2 1684 if (rt->rt_peer_genid != rt_peer_genid()) {
fbfe95a4 1685 struct inet_peer *peer = rt_get_peer(rt, rt->rt_dst);
6431cbc2 1686
fe6fe792 1687 if (peer) {
efbc368d 1688 check_peer_pmtu(&rt->dst, peer);
2c8cec5c 1689
fe6fe792 1690 if (peer->redirect_learned.a4 &&
de398fb8
DM		 1691 peer->redirect_learned.a4 != rt->rt_gateway)
1692 check_peer_redir(&rt->dst, peer);
f39925db
DM		 1693 }
1694
6431cbc2
DM		 1695 rt->rt_peer_genid = rt_peer_genid();
1696 }
efbc368d
DM		 1697}
1698
1699static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1700{
1701 struct rtable *rt = (struct rtable *) dst;
1702
1703 if (rt_is_expired(rt))
1704 return NULL;
de398fb8 1705 ipv4_validate_peer(rt);
d11a4dc1 1706 return dst;
1da177e4
LT		 1707}
1708
1709static void ipv4_dst_destroy(struct dst_entry *dst)
1710{
1711 struct rtable *rt = (struct rtable *) dst;
1da177e4 1712
62fa8a84
DM		 1713 if (rt->fi) {
1714 fib_info_put(rt->fi);
1715 rt->fi = NULL;
1716 }
97bab73f
DM		 1717 if (rt_has_peer(rt)) {
1718 struct inet_peer *peer = rt_peer_ptr(rt);
1da177e4
LT		 1719 inet_putpeer(peer);
1720 }
1da177e4
LT		 1721}
1722
1da177e4
LT		 1723
1724static void ipv4_link_failure(struct sk_buff *skb)
1725{
1726 struct rtable *rt;
1727
1728 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1729
511c3f92 1730 rt = skb_rtable(skb);
97bab73f
DM		 1731 if (rt && rt_has_peer(rt)) {
1732 struct inet_peer *peer = rt_peer_ptr(rt);
1733 if (peer_pmtu_cleaned(peer))
1734 dst_metric_set(&rt->dst, RTAX_MTU, peer->pmtu_orig);
1735 }
1da177e4
LT		 1736}
1737
1738static int ip_rt_bug(struct sk_buff *skb)
1739{
91df42be
JP		 1740 pr_debug("%s: %pI4 -> %pI4, %s\n",
1741 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1742 skb->dev ? skb->dev->name : "?");
1da177e4 1743 kfree_skb(skb);
c378a9c0 1744 WARN_ON(1);
1da177e4
LT		 1745 return 0;
1746}
1747
1748/*
1749 We do not cache source address of outgoing interface,
1750 because it is used only by IP RR, TS and SRR options,
1751 so that it out of fast path.
1752
1753 BTW remember: "addr" is allowed to be not aligned
1754 in IP options!
1755 */
1756
8e36360a 1757void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1da177e4 1758{
a61ced5d 1759 __be32 src;
1da177e4 1760
c7537967 1761 if (rt_is_output_route(rt))
c5be24ff 1762 src = ip_hdr(skb)->saddr;
ebc0ffae 1763 else {
8e36360a
DM		 1764 struct fib_result res;
1765 struct flowi4 fl4;
1766 struct iphdr *iph;
1767
1768 iph = ip_hdr(skb);
1769
1770 memset(&fl4, 0, sizeof(fl4));
1771 fl4.daddr = iph->daddr;
1772 fl4.saddr = iph->saddr;
b0fe4a31 1773 fl4.flowi4_tos = RT_TOS(iph->tos);
8e36360a
DM		 1774 fl4.flowi4_oif = rt->dst.dev->ifindex;
1775 fl4.flowi4_iif = skb->dev->ifindex;
1776 fl4.flowi4_mark = skb->mark;
5e2b61f7 1777
ebc0ffae 1778 rcu_read_lock();
68a5e3dd 1779 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
436c3b66 1780 src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
ebc0ffae
ED		 1781 else
1782 src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1da177e4 1783 RT_SCOPE_UNIVERSE);
ebc0ffae
ED		 1784 rcu_read_unlock();
1785 }
1da177e4
LT		 1786 memcpy(addr, &src, 4);
1787}
1788
c7066f70 1789#ifdef CONFIG_IP_ROUTE_CLASSID
1da177e4
LT		 1790static void set_class_tag(struct rtable *rt, u32 tag)
1791{
d8d1f30b
CG		 1792 if (!(rt->dst.tclassid & 0xFFFF))
1793 rt->dst.tclassid |= tag & 0xFFFF;
1794 if (!(rt->dst.tclassid & 0xFFFF0000))
1795 rt->dst.tclassid |= tag & 0xFFFF0000;
1da177e4
LT		 1796}
1797#endif
1798
0dbaee3b
DM		 1799static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1800{
1801 unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1802
1803 if (advmss == 0) {
1804 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1805 ip_rt_min_advmss);
1806 if (advmss > 65535 - 40)
1807 advmss = 65535 - 40;
1808 }
1809 return advmss;
1810}
1811
ebb762f2 1812static unsigned int ipv4_mtu(const struct dst_entry *dst)
d33e4553 1813{
261663b0 1814 const struct rtable *rt = (const struct rtable *) dst;
618f9bc7
SK		 1815 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
1816
261663b0 1817 if (mtu && rt_is_output_route(rt))
618f9bc7
SK		 1818 return mtu;
1819
1820 mtu = dst->dev->mtu;
d33e4553
DM		 1821
1822 if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
d33e4553
DM		 1823
1824 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1825 mtu = 576;
1826 }
1827
1828 if (mtu > IP_MAX_MTU)
1829 mtu = IP_MAX_MTU;
1830
1831 return mtu;
1832}
1833
813b3b5d 1834static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
5e2b61f7 1835 struct fib_info *fi)
a4daad6b 1836{
97bab73f 1837 struct inet_peer_base *base;
0131ba45
DM		 1838 struct inet_peer *peer;
1839 int create = 0;
a4daad6b 1840
0131ba45
DM		 1841 /* If a peer entry exists for this destination, we must hook
1842 * it up in order to get at cached metrics.
1843 */
813b3b5d 1844 if (fl4 && (fl4->flowi4_flags & FLOWI_FLAG_PRECOW_METRICS))
0131ba45
DM		 1845 create = 1;
1846
97bab73f
DM		 1847 base = inetpeer_base_ptr(rt->_peer);
1848 BUG_ON(!base);
1849
1850 peer = inet_getpeer_v4(base, rt->rt_dst, create);
0131ba45 1851 if (peer) {
97bab73f 1852 __rt_set_peer(rt, peer);
3c0afdca 1853 rt->rt_peer_genid = rt_peer_genid();
a4daad6b
DM		 1854 if (inet_metrics_new(peer))
1855 memcpy(peer->metrics, fi->fib_metrics,
1856 sizeof(u32) * RTAX_MAX);
1857 dst_init_metrics(&rt->dst, peer->metrics, false);
2c8cec5c 1858
fe6fe792 1859 check_peer_pmtu(&rt->dst, peer);
ac3f48de 1860
f39925db
DM		 1861 if (peer->redirect_learned.a4 &&
1862 peer->redirect_learned.a4 != rt->rt_gateway) {
1863 rt->rt_gateway = peer->redirect_learned.a4;
1864 rt->rt_flags |= RTCF_REDIRECTED;
1865 }
0131ba45
DM		 1866 } else {
1867 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1868 rt->fi = fi;
1869 atomic_inc(&fi->fib_clntref);
1870 }
1871 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
a4daad6b
DM		 1872 }
1873}
1874
813b3b5d 1875static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
5e2b61f7 1876 const struct fib_result *res,
982721f3 1877 struct fib_info *fi, u16 type, u32 itag)
1da177e4 1878{
defb3519 1879 struct dst_entry *dst = &rt->dst;
1da177e4
LT		 1880
1881 if (fi) {
1882 if (FIB_RES_GW(*res) &&
1883 FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1884 rt->rt_gateway = FIB_RES_GW(*res);
813b3b5d 1885 rt_init_metrics(rt, fl4, fi);
c7066f70 1886#ifdef CONFIG_IP_ROUTE_CLASSID
defb3519 1887 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1da177e4 1888#endif
d33e4553 1889 }
defb3519 1890
defb3519
DM		 1891 if (dst_mtu(dst) > IP_MAX_MTU)
1892 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1da177e4 1893
c7066f70 1894#ifdef CONFIG_IP_ROUTE_CLASSID
1da177e4
LT		 1895#ifdef CONFIG_IP_MULTIPLE_TABLES
1896 set_class_tag(rt, fib_rules_tclass(res));
1897#endif
1898 set_class_tag(rt, itag);
1899#endif
1da177e4
LT		 1900}
1901
5c1e6aa3
DM		 1902static struct rtable *rt_dst_alloc(struct net_device *dev,
1903 bool nopolicy, bool noxfrm)
0c4dcd58 1904{
5c1e6aa3
DM		 1905 return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
1906 DST_HOST |
1907 (nopolicy ? DST_NOPOLICY : 0) |
1908 (noxfrm ? DST_NOXFRM : 0));
0c4dcd58
DM		 1909}
1910
96d36220 1911/* called in rcu_read_lock() section */
9e12bb22 1912static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1da177e4
LT		 1913 u8 tos, struct net_device *dev, int our)
1914{
96d36220 1915 unsigned int hash;
1da177e4 1916 struct rtable *rth;
96d36220 1917 struct in_device *in_dev = __in_dev_get_rcu(dev);
1da177e4 1918 u32 itag = 0;
b5f7e755 1919 int err;
1da177e4
LT		 1920
1921 /* Primary sanity checks. */
1922
1923 if (in_dev == NULL)
1924 return -EINVAL;
1925
1e637c74 1926 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
d0daebc3 1927 skb->protocol != htons(ETH_P_IP))
1da177e4
LT		 1928 goto e_inval;
1929
d0daebc3
TG		 1930 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1931 if (ipv4_is_loopback(saddr))
1932 goto e_inval;
1933
f97c1e0c
JP		 1934 if (ipv4_is_zeronet(saddr)) {
1935 if (!ipv4_is_local_multicast(daddr))
1da177e4 1936 goto e_inval;
b5f7e755 1937 } else {
9e56e380
DM		 1938 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1939 in_dev, &itag);
b5f7e755
ED		 1940 if (err < 0)
1941 goto e_err;
1942 }
4e7b2f14 1943 rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
5c1e6aa3 1944 IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1da177e4
LT		 1945 if (!rth)
1946 goto e_nobufs;
1947
cf911662
DM		 1948#ifdef CONFIG_IP_ROUTE_CLASSID
1949 rth->dst.tclassid = itag;
1950#endif
d8d1f30b 1951 rth->dst.output = ip_rt_bug;
1da177e4 1952
5e2b61f7 1953 rth->rt_key_dst = daddr;
5e2b61f7 1954 rth->rt_key_src = saddr;
cf911662
DM		 1955 rth->rt_genid = rt_genid(dev_net(dev));
1956 rth->rt_flags = RTCF_MULTICAST;
1957 rth->rt_type = RTN_MULTICAST;
475949d8 1958 rth->rt_key_tos = tos;
cf911662 1959 rth->rt_dst = daddr;
1da177e4 1960 rth->rt_src = saddr;
1b86a58f 1961 rth->rt_route_iif = dev->ifindex;
5e2b61f7 1962 rth->rt_iif = dev->ifindex;
5e2b61f7 1963 rth->rt_oif = 0;
cf911662 1964 rth->rt_mark = skb->mark;
1da177e4 1965 rth->rt_gateway = daddr;
cf911662 1966 rth->rt_peer_genid = 0;
97bab73f 1967 rt_init_peer(rth, dev_net(dev)->ipv4.peers);
cf911662 1968 rth->fi = NULL;
1da177e4 1969 if (our) {
d8d1f30b 1970 rth->dst.input= ip_local_deliver;
1da177e4
LT		 1971 rth->rt_flags |= RTCF_LOCAL;
1972 }
1973
1974#ifdef CONFIG_IP_MROUTE
f97c1e0c 1975 if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
d8d1f30b 1976 rth->dst.input = ip_mr_input;
1da177e4
LT		 1977#endif
1978 RT_CACHE_STAT_INC(in_slow_mc);
1979
e84f84f2 1980 hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
b23dd4fe 1981 rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
9aa3c94c 1982 return IS_ERR(rth) ? PTR_ERR(rth) : 0;
1da177e4
LT		 1983
1984e_nobufs:
1da177e4 1985 return -ENOBUFS;
1da177e4 1986e_inval:
96d36220 1987 return -EINVAL;
b5f7e755 1988e_err:
b5f7e755 1989 return err;
1da177e4
LT		 1990}
1991
1992
1993static void ip_handle_martian_source(struct net_device *dev,
1994 struct in_device *in_dev,
1995 struct sk_buff *skb,
9e12bb22
AV		 1996 __be32 daddr,
1997 __be32 saddr)
1da177e4
LT		 1998{
1999 RT_CACHE_STAT_INC(in_martian_src);
2000#ifdef CONFIG_IP_ROUTE_VERBOSE
2001 if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
2002 /*
2003 * RFC1812 recommendation, if source is martian,
2004 * the only hint is MAC header.
2005 */
058bd4d2 2006 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
673d57e7 2007 &daddr, &saddr, dev->name);
98e399f8 2008 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
058bd4d2
JP		 2009 print_hex_dump(KERN_WARNING, "ll header: ",
2010 DUMP_PREFIX_OFFSET, 16, 1,
2011 skb_mac_header(skb),
2012 dev->hard_header_len, true);
1da177e4
LT		 2013 }
2014 }
2015#endif
2016}
2017
47360228 2018/* called in rcu_read_lock() section */
5969f71d 2019static int __mkroute_input(struct sk_buff *skb,
982721f3 2020 const struct fib_result *res,
5969f71d
SH		 2021 struct in_device *in_dev,
2022 __be32 daddr, __be32 saddr, u32 tos,
2023 struct rtable **result)
1da177e4 2024{
1da177e4
LT		 2025 struct rtable *rth;
2026 int err;
2027 struct in_device *out_dev;
47360228 2028 unsigned int flags = 0;
d9c9df8c 2029 u32 itag;
1da177e4
LT		 2030
2031 /* get a working reference to the output device */
47360228 2032 out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1da177e4 2033 if (out_dev == NULL) {
e87cc472 2034 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1da177e4
LT		 2035 return -EINVAL;
2036 }
2037
2038
5c04c819 2039 err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
9e56e380 2040 in_dev->dev, in_dev, &itag);
1da177e4 2041 if (err < 0) {
e905a9ed 2042 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1da177e4 2043 saddr);
e905a9ed 2044
1da177e4
LT		 2045 goto cleanup;
2046 }
2047
2048 if (err)
2049 flags |= RTCF_DIRECTSRC;
2050
51b77cae 2051 if (out_dev == in_dev && err &&
1da177e4
LT		 2052 (IN_DEV_SHARED_MEDIA(out_dev) ||
2053 inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
2054 flags |= RTCF_DOREDIRECT;
2055
2056 if (skb->protocol != htons(ETH_P_IP)) {
2057 /* Not IP (i.e. ARP). Do not create route, if it is
2058 * invalid for proxy arp. DNAT routes are always valid.
65324144
JDB		 2059 *
2060 * Proxy arp feature have been extended to allow, ARP
2061 * replies back to the same interface, to support
2062 * Private VLAN switch technologies. See arp.c.
1da177e4 2063 */
65324144
JDB		 2064 if (out_dev == in_dev &&
2065 IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1da177e4
LT		 2066 err = -EINVAL;
2067 goto cleanup;
2068 }
2069 }
2070
5c1e6aa3
DM		 2071 rth = rt_dst_alloc(out_dev->dev,
2072 IN_DEV_CONF_GET(in_dev, NOPOLICY),
0c4dcd58 2073 IN_DEV_CONF_GET(out_dev, NOXFRM));
1da177e4
LT		 2074 if (!rth) {
2075 err = -ENOBUFS;
2076 goto cleanup;
2077 }
2078
5e2b61f7 2079 rth->rt_key_dst = daddr;
5e2b61f7 2080 rth->rt_key_src = saddr;
cf911662
DM		 2081 rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2082 rth->rt_flags = flags;
2083 rth->rt_type = res->type;
475949d8 2084 rth->rt_key_tos = tos;
cf911662 2085 rth->rt_dst = daddr;
1da177e4 2086 rth->rt_src = saddr;
1b86a58f 2087 rth->rt_route_iif = in_dev->dev->ifindex;
5e2b61f7 2088 rth->rt_iif = in_dev->dev->ifindex;
5e2b61f7 2089 rth->rt_oif = 0;
cf911662
DM		 2090 rth->rt_mark = skb->mark;
2091 rth->rt_gateway = daddr;
cf911662 2092 rth->rt_peer_genid = 0;
8b96d22d 2093 rt_init_peer(rth, &res->table->tb_peers);
cf911662 2094 rth->fi = NULL;
1da177e4 2095
d8d1f30b
CG		 2096 rth->dst.input = ip_forward;
2097 rth->dst.output = ip_output;
1da177e4 2098
5e2b61f7 2099 rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
1da177e4 2100
1da177e4
LT		 2101 *result = rth;
2102 err = 0;
2103 cleanup:
1da177e4 2104 return err;
e905a9ed 2105}
1da177e4 2106
5969f71d
SH		 2107static int ip_mkroute_input(struct sk_buff *skb,
2108 struct fib_result *res,
68a5e3dd 2109 const struct flowi4 *fl4,
5969f71d
SH		 2110 struct in_device *in_dev,
2111 __be32 daddr, __be32 saddr, u32 tos)
1da177e4 2112{
5e73ea1a 2113 struct rtable *rth = NULL;
1da177e4 2114 int err;
95c96174 2115 unsigned int hash;
1da177e4
LT		 2116
2117#ifdef CONFIG_IP_ROUTE_MULTIPATH
ff3fccb3 2118 if (res->fi && res->fi->fib_nhs > 1)
1b7fe593 2119 fib_select_multipath(res);
1da177e4
LT		 2120#endif
2121
2122 /* create a routing cache entry */
2123 err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2124 if (err)
2125 return err;
1da177e4
LT		 2126
2127 /* put it into the cache */
68a5e3dd 2128 hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
d8d1f30b 2129 rt_genid(dev_net(rth->dst.dev)));
68a5e3dd 2130 rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
b23dd4fe
DM		 2131 if (IS_ERR(rth))
2132 return PTR_ERR(rth);
2133 return 0;
1da177e4
LT		 2134}
2135
1da177e4
LT		 2136/*
2137 * NOTE. We drop all the packets that has local source
2138 * addresses, because every properly looped back packet
2139 * must have correct destination already attached by output routine.
2140 *
2141 * Such approach solves two big problems:
2142 * 1. Not simplex devices are handled properly.
2143 * 2. IP spoofing attempts are filtered with 100% of guarantee.
ebc0ffae 2144 * called with rcu_read_lock()
1da177e4
LT		 2145 */
2146
9e12bb22 2147static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
c10237e0 2148 u8 tos, struct net_device *dev)
1da177e4
LT		 2149{
2150 struct fib_result res;
96d36220 2151 struct in_device *in_dev = __in_dev_get_rcu(dev);
68a5e3dd 2152 struct flowi4 fl4;
95c96174 2153 unsigned int flags = 0;
1da177e4 2154 u32 itag = 0;
95c96174
ED		 2155 struct rtable *rth;
2156 unsigned int hash;
1da177e4 2157 int err = -EINVAL;
5e73ea1a 2158 struct net *net = dev_net(dev);
1da177e4
LT		 2159
2160 /* IP on this device is disabled. */
2161
2162 if (!in_dev)
2163 goto out;
2164
2165 /* Check for the most weird martians, which can be not detected
2166 by fib_lookup.
2167 */
2168
d0daebc3 2169 if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1da177e4
LT		 2170 goto martian_source;
2171
27a954bd 2172 if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1da177e4
LT		 2173 goto brd_input;
2174
2175 /* Accept zero addresses only to limited broadcast;
2176 * I even do not know to fix it or not. Waiting for complains :-)
2177 */
f97c1e0c 2178 if (ipv4_is_zeronet(saddr))
1da177e4
LT		 2179 goto martian_source;
2180
d0daebc3 2181 if (ipv4_is_zeronet(daddr))
1da177e4
LT		 2182 goto martian_destination;
2183
d0daebc3
TG		 2184 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) {
2185 if (ipv4_is_loopback(daddr))
2186 goto martian_destination;
2187
2188 if (ipv4_is_loopback(saddr))
2189 goto martian_source;
2190 }
2191
1da177e4
LT		 2192 /*
2193 * Now we are ready to route packet.
2194 */
68a5e3dd
DM		 2195 fl4.flowi4_oif = 0;
2196 fl4.flowi4_iif = dev->ifindex;
2197 fl4.flowi4_mark = skb->mark;
2198 fl4.flowi4_tos = tos;
2199 fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2200 fl4.daddr = daddr;
2201 fl4.saddr = saddr;
2202 err = fib_lookup(net, &fl4, &res);
251da413 2203 if (err != 0)
1da177e4 2204 goto no_route;
1da177e4
LT		 2205
2206 RT_CACHE_STAT_INC(in_slow_tot);
2207
2208 if (res.type == RTN_BROADCAST)
2209 goto brd_input;
2210
2211 if (res.type == RTN_LOCAL) {
5c04c819 2212 err = fib_validate_source(skb, saddr, daddr, tos,
ebc0ffae 2213 net->loopback_dev->ifindex,
9e56e380 2214 dev, in_dev, &itag);
b5f7e755
ED		 2215 if (err < 0)
2216 goto martian_source_keep_err;
2217 if (err)
1da177e4 2218 flags |= RTCF_DIRECTSRC;
1da177e4
LT		 2219 goto local_input;
2220 }
2221
2222 if (!IN_DEV_FORWARD(in_dev))
251da413 2223 goto no_route;
1da177e4
LT		 2224 if (res.type != RTN_UNICAST)
2225 goto martian_destination;
2226
68a5e3dd 2227 err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1da177e4
LT		 2228out: return err;
2229
2230brd_input:
2231 if (skb->protocol != htons(ETH_P_IP))
2232 goto e_inval;
2233
41347dcd 2234 if (!ipv4_is_zeronet(saddr)) {
9e56e380
DM		 2235 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2236 in_dev, &itag);
1da177e4 2237 if (err < 0)
b5f7e755 2238 goto martian_source_keep_err;
1da177e4
LT		 2239 if (err)
2240 flags |= RTCF_DIRECTSRC;
2241 }
2242 flags |= RTCF_BROADCAST;
2243 res.type = RTN_BROADCAST;
2244 RT_CACHE_STAT_INC(in_brd);
2245
2246local_input:
5c1e6aa3
DM		 2247 rth = rt_dst_alloc(net->loopback_dev,
2248 IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1da177e4
LT		 2249 if (!rth)
2250 goto e_nobufs;
2251
cf911662 2252 rth->dst.input= ip_local_deliver;
d8d1f30b 2253 rth->dst.output= ip_rt_bug;
cf911662
DM		 2254#ifdef CONFIG_IP_ROUTE_CLASSID
2255 rth->dst.tclassid = itag;
2256#endif
1da177e4 2257
5e2b61f7 2258 rth->rt_key_dst = daddr;
5e2b61f7 2259 rth->rt_key_src = saddr;
cf911662
DM		 2260 rth->rt_genid = rt_genid(net);
2261 rth->rt_flags = flags|RTCF_LOCAL;
2262 rth->rt_type = res.type;
475949d8 2263 rth->rt_key_tos = tos;
cf911662 2264 rth->rt_dst = daddr;
1da177e4 2265 rth->rt_src = saddr;
1b86a58f 2266 rth->rt_route_iif = dev->ifindex;
5e2b61f7 2267 rth->rt_iif = dev->ifindex;
cf911662
DM		 2268 rth->rt_oif = 0;
2269 rth->rt_mark = skb->mark;
1da177e4 2270 rth->rt_gateway = daddr;
cf911662 2271 rth->rt_peer_genid = 0;
97bab73f 2272 rt_init_peer(rth, net->ipv4.peers);
cf911662 2273 rth->fi = NULL;
1da177e4 2274 if (res.type == RTN_UNREACHABLE) {
d8d1f30b
CG		 2275 rth->dst.input= ip_error;
2276 rth->dst.error= -err;
1da177e4
LT		 2277 rth->rt_flags &= ~RTCF_LOCAL;
2278 }
68a5e3dd
DM		 2279 hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2280 rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
b23dd4fe
DM		 2281 err = 0;
2282 if (IS_ERR(rth))
2283 err = PTR_ERR(rth);
ebc0ffae 2284 goto out;
1da177e4
LT		 2285
2286no_route:
2287 RT_CACHE_STAT_INC(in_no_route);
1da177e4 2288 res.type = RTN_UNREACHABLE;
7f53878d
MC		 2289 if (err == -ESRCH)
2290 err = -ENETUNREACH;
1da177e4
LT		 2291 goto local_input;
2292
2293 /*
2294 * Do not cache martian addresses: they should be logged (RFC1812)
2295 */
2296martian_destination:
2297 RT_CACHE_STAT_INC(in_martian_dst);
2298#ifdef CONFIG_IP_ROUTE_VERBOSE
e87cc472
JP		 2299 if (IN_DEV_LOG_MARTIANS(in_dev))
2300 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2301 &daddr, &saddr, dev->name);
1da177e4 2302#endif
2c2910a4 2303
1da177e4
LT		 2304e_inval:
2305 err = -EINVAL;
ebc0ffae 2306 goto out;
1da177e4
LT		 2307
2308e_nobufs:
2309 err = -ENOBUFS;
ebc0ffae 2310 goto out;
1da177e4
LT		 2311
2312martian_source:
b5f7e755
ED		 2313 err = -EINVAL;
2314martian_source_keep_err:
1da177e4 2315 ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
ebc0ffae 2316 goto out;
1da177e4
LT		 2317}
2318
407eadd9 2319int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
c10237e0 2320 u8 tos, struct net_device *dev, bool noref)
1da177e4 2321{
95c96174
ED		 2322 struct rtable *rth;
2323 unsigned int hash;
1da177e4 2324 int iif = dev->ifindex;
b5921910 2325 struct net *net;
96d36220 2326 int res;
1da177e4 2327
c346dca1 2328 net = dev_net(dev);
1080d709 2329
96d36220
ED		 2330 rcu_read_lock();
2331
1080d709
NH		 2332 if (!rt_caching(net))
2333 goto skip_cache;
2334
1da177e4 2335 tos &= IPTOS_RT_MASK;
e84f84f2 2336 hash = rt_hash(daddr, saddr, iif, rt_genid(net));
1da177e4 2337
1da177e4 2338 for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
d8d1f30b 2339 rth = rcu_dereference(rth->dst.rt_next)) {
5e2b61f7
DM		 2340 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2341 ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
97a80410 2342 (rth->rt_route_iif ^ iif) |
475949d8 2343 (rth->rt_key_tos ^ tos)) == 0 &&
5e2b61f7 2344 rth->rt_mark == skb->mark &&
d8d1f30b 2345 net_eq(dev_net(rth->dst.dev), net) &&
e84f84f2 2346 !rt_is_expired(rth)) {
de398fb8 2347 ipv4_validate_peer(rth);
407eadd9 2348 if (noref) {
d8d1f30b
CG		 2349 dst_use_noref(&rth->dst, jiffies);
2350 skb_dst_set_noref(skb, &rth->dst);
407eadd9 2351 } else {
d8d1f30b
CG		 2352 dst_use(&rth->dst, jiffies);
2353 skb_dst_set(skb, &rth->dst);
407eadd9 2354 }
1da177e4
LT		 2355 RT_CACHE_STAT_INC(in_hit);
2356 rcu_read_unlock();
1da177e4
LT		 2357 return 0;
2358 }
2359 RT_CACHE_STAT_INC(in_hlist_search);
2360 }
1da177e4 2361
1080d709 2362skip_cache:
1da177e4
LT		 2363 /* Multicast recognition logic is moved from route cache to here.
2364 The problem was that too many Ethernet cards have broken/missing
2365 hardware multicast filters :-( As result the host on multicasting
2366 network acquires a lot of useless route cache entries, sort of
2367 SDR messages from all the world. Now we try to get rid of them.
2368 Really, provided software IP multicast filter is organized
2369 reasonably (at least, hashed), it does not result in a slowdown
2370 comparing with route cache reject entries.
2371 Note, that multicast routers are not affected, because
2372 route cache entry is created eventually.
2373 */
f97c1e0c 2374 if (ipv4_is_multicast(daddr)) {
96d36220 2375 struct in_device *in_dev = __in_dev_get_rcu(dev);
1da177e4 2376
96d36220 2377 if (in_dev) {
dbdd9a52
DM		 2378 int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2379 ip_hdr(skb)->protocol);
1da177e4
LT		 2380 if (our
2381#ifdef CONFIG_IP_MROUTE
9d4fb27d
JP		 2382 ||
2383 (!ipv4_is_local_multicast(daddr) &&
2384 IN_DEV_MFORWARD(in_dev))
1da177e4 2385#endif
9d4fb27d 2386 ) {
96d36220
ED		 2387 int res = ip_route_input_mc(skb, daddr, saddr,
2388 tos, dev, our);
1da177e4 2389 rcu_read_unlock();
96d36220 2390 return res;
1da177e4
LT		 2391 }
2392 }
2393 rcu_read_unlock();
2394 return -EINVAL;
2395 }
c10237e0 2396 res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
96d36220
ED		 2397 rcu_read_unlock();
2398 return res;
1da177e4 2399}
407eadd9 2400EXPORT_SYMBOL(ip_route_input_common);
1da177e4 2401
ebc0ffae 2402/* called with rcu_read_lock() */
982721f3 2403static struct rtable *__mkroute_output(const struct fib_result *res,
68a5e3dd 2404 const struct flowi4 *fl4,
813b3b5d 2405 __be32 orig_daddr, __be32 orig_saddr,
f61759e6
JA		 2406 int orig_oif, __u8 orig_rtos,
2407 struct net_device *dev_out,
5ada5527 2408 unsigned int flags)
1da177e4 2409{
982721f3 2410 struct fib_info *fi = res->fi;
5ada5527 2411 struct in_device *in_dev;
982721f3 2412 u16 type = res->type;
5ada5527 2413 struct rtable *rth;
1da177e4 2414
d0daebc3
TG		 2415 in_dev = __in_dev_get_rcu(dev_out);
2416 if (!in_dev)
5ada5527 2417 return ERR_PTR(-EINVAL);
1da177e4 2418
d0daebc3
TG		 2419 if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2420 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2421 return ERR_PTR(-EINVAL);
2422
68a5e3dd 2423 if (ipv4_is_lbcast(fl4->daddr))
982721f3 2424 type = RTN_BROADCAST;
68a5e3dd 2425 else if (ipv4_is_multicast(fl4->daddr))
982721f3 2426 type = RTN_MULTICAST;
68a5e3dd 2427 else if (ipv4_is_zeronet(fl4->daddr))
5ada5527 2428 return ERR_PTR(-EINVAL);
1da177e4
LT		 2429
2430 if (dev_out->flags & IFF_LOOPBACK)
2431 flags |= RTCF_LOCAL;
2432
982721f3 2433 if (type == RTN_BROADCAST) {
1da177e4 2434 flags |= RTCF_BROADCAST | RTCF_LOCAL;
982721f3
DM		 2435 fi = NULL;
2436 } else if (type == RTN_MULTICAST) {
dd28d1a0 2437 flags |= RTCF_MULTICAST | RTCF_LOCAL;
813b3b5d
DM		 2438 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2439 fl4->flowi4_proto))
1da177e4
LT		 2440 flags &= ~RTCF_LOCAL;
2441 /* If multicast route do not exist use
dd28d1a0
ED		 2442 * default one, but do not gateway in this case.
2443 * Yes, it is hack.
1da177e4 2444 */
982721f3
DM		 2445 if (fi && res->prefixlen < 4)
2446 fi = NULL;
1da177e4
LT		 2447 }
2448
5c1e6aa3
DM		 2449 rth = rt_dst_alloc(dev_out,
2450 IN_DEV_CONF_GET(in_dev, NOPOLICY),
0c4dcd58 2451 IN_DEV_CONF_GET(in_dev, NOXFRM));
8391d07b 2452 if (!rth)
5ada5527 2453 return ERR_PTR(-ENOBUFS);
8391d07b 2454
cf911662
DM		 2455 rth->dst.output = ip_output;
2456
813b3b5d
DM		 2457 rth->rt_key_dst = orig_daddr;
2458 rth->rt_key_src = orig_saddr;
cf911662
DM		 2459 rth->rt_genid = rt_genid(dev_net(dev_out));
2460 rth->rt_flags = flags;
2461 rth->rt_type = type;
f61759e6 2462 rth->rt_key_tos = orig_rtos;
68a5e3dd
DM		 2463 rth->rt_dst = fl4->daddr;
2464 rth->rt_src = fl4->saddr;
1b86a58f 2465 rth->rt_route_iif = 0;
813b3b5d
DM		 2466 rth->rt_iif = orig_oif ? : dev_out->ifindex;
2467 rth->rt_oif = orig_oif;
2468 rth->rt_mark = fl4->flowi4_mark;
68a5e3dd 2469 rth->rt_gateway = fl4->daddr;
cf911662 2470 rth->rt_peer_genid = 0;
8b96d22d
DM		 2471 rt_init_peer(rth, (res->table ?
2472 &res->table->tb_peers :
2473 dev_net(dev_out)->ipv4.peers));
cf911662 2474 rth->fi = NULL;
1da177e4
LT		 2475
2476 RT_CACHE_STAT_INC(out_slow_tot);
2477
41347dcd 2478 if (flags & RTCF_LOCAL)
d8d1f30b 2479 rth->dst.input = ip_local_deliver;
1da177e4 2480 if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
e905a9ed 2481 if (flags & RTCF_LOCAL &&
1da177e4 2482 !(dev_out->flags & IFF_LOOPBACK)) {
d8d1f30b 2483 rth->dst.output = ip_mc_output;
1da177e4
LT		 2484 RT_CACHE_STAT_INC(out_slow_mc);
2485 }
2486#ifdef CONFIG_IP_MROUTE
982721f3 2487 if (type == RTN_MULTICAST) {
1da177e4 2488 if (IN_DEV_MFORWARD(in_dev) &&
813b3b5d 2489 !ipv4_is_local_multicast(fl4->daddr)) {
d8d1f30b
CG		 2490 rth->dst.input = ip_mr_input;
2491 rth->dst.output = ip_mc_output;
1da177e4
LT		 2492 }
2493 }
2494#endif
2495 }
2496
813b3b5d 2497 rt_set_nexthop(rth, fl4, res, fi, type, 0);
1da177e4 2498
7586eceb
ED		 2499 if (fl4->flowi4_flags & FLOWI_FLAG_RT_NOCACHE)
2500 rth->dst.flags |= DST_NOCACHE;
2501
5ada5527 2502 return rth;
1da177e4
LT		 2503}
2504
1da177e4
LT		 2505/*
2506 * Major route resolver routine.
0197aa38 2507 * called with rcu_read_lock();
1da177e4
LT		 2508 */
2509
813b3b5d 2510static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
1da177e4 2511{
1da177e4 2512 struct net_device *dev_out = NULL;
f61759e6 2513 __u8 tos = RT_FL_TOS(fl4);
813b3b5d
DM		 2514 unsigned int flags = 0;
2515 struct fib_result res;
5ada5527 2516 struct rtable *rth;
813b3b5d
DM		 2517 __be32 orig_daddr;
2518 __be32 orig_saddr;
2519 int orig_oif;
1da177e4
LT		 2520
2521 res.fi = NULL;
8b96d22d 2522 res.table = NULL;
1da177e4
LT		 2523#ifdef CONFIG_IP_MULTIPLE_TABLES
2524 res.r = NULL;
2525#endif
2526
813b3b5d
DM		 2527 orig_daddr = fl4->daddr;
2528 orig_saddr = fl4->saddr;
2529 orig_oif = fl4->flowi4_oif;
2530
2531 fl4->flowi4_iif = net->loopback_dev->ifindex;
2532 fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2533 fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2534 RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
44713b67 2535
010c2708 2536 rcu_read_lock();
813b3b5d 2537 if (fl4->saddr) {
b23dd4fe 2538 rth = ERR_PTR(-EINVAL);
813b3b5d
DM		 2539 if (ipv4_is_multicast(fl4->saddr) ||
2540 ipv4_is_lbcast(fl4->saddr) ||
2541 ipv4_is_zeronet(fl4->saddr))
1da177e4
LT		 2542 goto out;
2543
1da177e4
LT		 2544 /* I removed check for oif == dev_out->oif here.
2545 It was wrong for two reasons:
1ab35276
DL		 2546 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2547 is assigned to multiple interfaces.
1da177e4
LT		 2548 2. Moreover, we are allowed to send packets with saddr
2549 of another iface. --ANK
2550 */
2551
813b3b5d
DM		 2552 if (fl4->flowi4_oif == 0 &&
2553 (ipv4_is_multicast(fl4->daddr) ||
2554 ipv4_is_lbcast(fl4->daddr))) {
a210d01a 2555 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
813b3b5d 2556 dev_out = __ip_dev_find(net, fl4->saddr, false);
a210d01a
JA		 2557 if (dev_out == NULL)
2558 goto out;
2559
1da177e4
LT		 2560 /* Special hack: user can direct multicasts
2561 and limited broadcast via necessary interface
2562 without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2563 This hack is not just for fun, it allows
2564 vic,vat and friends to work.
2565 They bind socket to loopback, set ttl to zero
2566 and expect that it will work.
2567 From the viewpoint of routing cache they are broken,
2568 because we are not allowed to build multicast path
2569 with loopback source addr (look, routing cache
2570 cannot know, that ttl is zero, so that packet
2571 will not leave this host and route is valid).
2572 Luckily, this hack is good workaround.
2573 */
2574
813b3b5d 2575 fl4->flowi4_oif = dev_out->ifindex;
1da177e4
LT		 2576 goto make_route;
2577 }
a210d01a 2578
813b3b5d 2579 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
a210d01a 2580 /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
813b3b5d 2581 if (!__ip_dev_find(net, fl4->saddr, false))
a210d01a 2582 goto out;
a210d01a 2583 }
1da177e4
LT		 2584 }
2585
2586
813b3b5d
DM		 2587 if (fl4->flowi4_oif) {
2588 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
b23dd4fe 2589 rth = ERR_PTR(-ENODEV);
1da177e4
LT		 2590 if (dev_out == NULL)
2591 goto out;
e5ed6399
HX		 2592
2593 /* RACE: Check return value of inet_select_addr instead. */
fc75fc83 2594 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
b23dd4fe 2595 rth = ERR_PTR(-ENETUNREACH);
fc75fc83
ED		 2596 goto out;
2597 }
813b3b5d
DM		 2598 if (ipv4_is_local_multicast(fl4->daddr) ||
2599 ipv4_is_lbcast(fl4->daddr)) {
2600 if (!fl4->saddr)
2601 fl4->saddr = inet_select_addr(dev_out, 0,
2602 RT_SCOPE_LINK);
1da177e4
LT		 2603 goto make_route;
2604 }
813b3b5d
DM		 2605 if (fl4->saddr) {
2606 if (ipv4_is_multicast(fl4->daddr))
2607 fl4->saddr = inet_select_addr(dev_out, 0,
2608 fl4->flowi4_scope);
2609 else if (!fl4->daddr)
2610 fl4->saddr = inet_select_addr(dev_out, 0,
2611 RT_SCOPE_HOST);
1da177e4
LT		 2612 }
2613 }
2614
813b3b5d
DM		 2615 if (!fl4->daddr) {
2616 fl4->daddr = fl4->saddr;
2617 if (!fl4->daddr)
2618 fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
b40afd0e 2619 dev_out = net->loopback_dev;
813b3b5d 2620 fl4->flowi4_oif = net->loopback_dev->ifindex;
1da177e4
LT		 2621 res.type = RTN_LOCAL;
2622 flags |= RTCF_LOCAL;
2623 goto make_route;
2624 }
2625
813b3b5d 2626 if (fib_lookup(net, fl4, &res)) {
1da177e4 2627 res.fi = NULL;
8b96d22d 2628 res.table = NULL;
813b3b5d 2629 if (fl4->flowi4_oif) {
1da177e4
LT		 2630 /* Apparently, routing tables are wrong. Assume,
2631 that the destination is on link.
2632
2633 WHY? DW.
2634 Because we are allowed to send to iface
2635 even if it has NO routes and NO assigned
2636 addresses. When oif is specified, routing
2637 tables are looked up with only one purpose:
2638 to catch if destination is gatewayed, rather than
2639 direct. Moreover, if MSG_DONTROUTE is set,
2640 we send packet, ignoring both routing tables
2641 and ifaddr state. --ANK
2642
2643
2644 We could make it even if oif is unknown,
2645 likely IPv6, but we do not.
2646 */
2647
813b3b5d
DM		 2648 if (fl4->saddr == 0)
2649 fl4->saddr = inet_select_addr(dev_out, 0,
2650 RT_SCOPE_LINK);
1da177e4
LT		 2651 res.type = RTN_UNICAST;
2652 goto make_route;
2653 }
b23dd4fe 2654 rth = ERR_PTR(-ENETUNREACH);
1da177e4
LT		 2655 goto out;
2656 }
1da177e4
LT		 2657
2658 if (res.type == RTN_LOCAL) {
813b3b5d 2659 if (!fl4->saddr) {
9fc3bbb4 2660 if (res.fi->fib_prefsrc)
813b3b5d 2661 fl4->saddr = res.fi->fib_prefsrc;
9fc3bbb4 2662 else
813b3b5d 2663 fl4->saddr = fl4->daddr;
9fc3bbb4 2664 }
b40afd0e 2665 dev_out = net->loopback_dev;
813b3b5d 2666 fl4->flowi4_oif = dev_out->ifindex;
1da177e4
LT		 2667 res.fi = NULL;
2668 flags |= RTCF_LOCAL;
2669 goto make_route;
2670 }
2671
2672#ifdef CONFIG_IP_ROUTE_MULTIPATH
813b3b5d 2673 if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
1b7fe593 2674 fib_select_multipath(&res);
1da177e4
LT		 2675 else
2676#endif
21d8c49e
DM		 2677 if (!res.prefixlen &&
2678 res.table->tb_num_default > 1 &&
813b3b5d 2679 res.type == RTN_UNICAST && !fl4->flowi4_oif)
0c838ff1 2680 fib_select_default(&res);
1da177e4 2681
813b3b5d
DM		 2682 if (!fl4->saddr)
2683 fl4->saddr = FIB_RES_PREFSRC(net, res);
1da177e4 2684
1da177e4 2685 dev_out = FIB_RES_DEV(res);
813b3b5d 2686 fl4->flowi4_oif = dev_out->ifindex;
1da177e4
LT		 2687
2688
2689make_route:
813b3b5d 2690 rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
f61759e6 2691 tos, dev_out, flags);
b23dd4fe 2692 if (!IS_ERR(rth)) {
5ada5527
DM		 2693 unsigned int hash;
2694
813b3b5d 2695 hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
5ada5527 2696 rt_genid(dev_net(dev_out)));
813b3b5d 2697 rth = rt_intern_hash(hash, rth, NULL, orig_oif);
5ada5527 2698 }
1da177e4 2699
010c2708
DM		 2700out:
2701 rcu_read_unlock();
b23dd4fe 2702 return rth;
1da177e4
LT		 2703}
2704
813b3b5d 2705struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
1da177e4 2706{
1da177e4 2707 struct rtable *rth;
010c2708 2708 unsigned int hash;
1da177e4 2709
1080d709
NH		 2710 if (!rt_caching(net))
2711 goto slow_output;
2712
9d6ec938 2713 hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
1da177e4
LT		 2714
2715 rcu_read_lock_bh();
a898def2 2716 for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
d8d1f30b 2717 rth = rcu_dereference_bh(rth->dst.rt_next)) {
9d6ec938
DM		 2718 if (rth->rt_key_dst == flp4->daddr &&
2719 rth->rt_key_src == flp4->saddr &&
c7537967 2720 rt_is_output_route(rth) &&
9d6ec938
DM		 2721 rth->rt_oif == flp4->flowi4_oif &&
2722 rth->rt_mark == flp4->flowi4_mark &&
475949d8 2723 !((rth->rt_key_tos ^ flp4->flowi4_tos) &
b5921910 2724 (IPTOS_RT_MASK | RTO_ONLINK)) &&
d8d1f30b 2725 net_eq(dev_net(rth->dst.dev), net) &&
e84f84f2 2726 !rt_is_expired(rth)) {
de398fb8 2727 ipv4_validate_peer(rth);
d8d1f30b 2728 dst_use(&rth->dst, jiffies);
1da177e4
LT		 2729 RT_CACHE_STAT_INC(out_hit);
2730 rcu_read_unlock_bh();
56157872
DM		 2731 if (!flp4->saddr)
2732 flp4->saddr = rth->rt_src;
2733 if (!flp4->daddr)
2734 flp4->daddr = rth->rt_dst;
b23dd4fe 2735 return rth;
1da177e4
LT		 2736 }
2737 RT_CACHE_STAT_INC(out_hlist_search);
2738 }
2739 rcu_read_unlock_bh();
2740
1080d709 2741slow_output:
9d6ec938 2742 return ip_route_output_slow(net, flp4);
1da177e4 2743}
d8c97a94
ACM		 2744EXPORT_SYMBOL_GPL(__ip_route_output_key);
2745
ae2688d5
JW		 2746static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2747{
2748 return NULL;
2749}
2750
ebb762f2 2751static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
ec831ea7 2752{
618f9bc7
SK		 2753 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2754
2755 return mtu ? : dst->dev->mtu;
ec831ea7
RD		 2756}
2757
14e50e57
DM		 2758static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2759{
2760}
2761
0972ddb2
HB		 2762static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2763 unsigned long old)
2764{
2765 return NULL;
2766}
2767
14e50e57
DM		 2768static struct dst_ops ipv4_dst_blackhole_ops = {
2769 .family = AF_INET,
09640e63 2770 .protocol = cpu_to_be16(ETH_P_IP),
14e50e57 2771 .destroy = ipv4_dst_destroy,
ae2688d5 2772 .check = ipv4_blackhole_dst_check,
ebb762f2 2773 .mtu = ipv4_blackhole_mtu,
214f45c9 2774 .default_advmss = ipv4_default_advmss,
14e50e57 2775 .update_pmtu = ipv4_rt_blackhole_update_pmtu,
0972ddb2 2776 .cow_metrics = ipv4_rt_blackhole_cow_metrics,
d3aaeb38 2777 .neigh_lookup = ipv4_neigh_lookup,
14e50e57
DM		 2778};
2779
2774c131 2780struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
14e50e57 2781{
5c1e6aa3 2782 struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
2774c131 2783 struct rtable *ort = (struct rtable *) dst_orig;
14e50e57
DM		 2784
2785 if (rt) {
d8d1f30b 2786 struct dst_entry *new = &rt->dst;
14e50e57 2787
14e50e57 2788 new->__use = 1;
352e512c
HX		 2789 new->input = dst_discard;
2790 new->output = dst_discard;
defb3519 2791 dst_copy_metrics(new, &ort->dst);
14e50e57 2792
d8d1f30b 2793 new->dev = ort->dst.dev;
14e50e57
DM		 2794 if (new->dev)
2795 dev_hold(new->dev);
2796
5e2b61f7
DM		 2797 rt->rt_key_dst = ort->rt_key_dst;
2798 rt->rt_key_src = ort->rt_key_src;
475949d8 2799 rt->rt_key_tos = ort->rt_key_tos;
1b86a58f 2800 rt->rt_route_iif = ort->rt_route_iif;
5e2b61f7
DM		 2801 rt->rt_iif = ort->rt_iif;
2802 rt->rt_oif = ort->rt_oif;
2803 rt->rt_mark = ort->rt_mark;
14e50e57 2804
e84f84f2 2805 rt->rt_genid = rt_genid(net);
14e50e57
DM		 2806 rt->rt_flags = ort->rt_flags;
2807 rt->rt_type = ort->rt_type;
2808 rt->rt_dst = ort->rt_dst;
2809 rt->rt_src = ort->rt_src;
14e50e57 2810 rt->rt_gateway = ort->rt_gateway;
97bab73f 2811 rt_transfer_peer(rt, ort);
62fa8a84
DM		 2812 rt->fi = ort->fi;
2813 if (rt->fi)
2814 atomic_inc(&rt->fi->fib_clntref);
14e50e57
DM		 2815
2816 dst_free(new);
2817 }
2818
2774c131
DM		 2819 dst_release(dst_orig);
2820
2821 return rt ? &rt->dst : ERR_PTR(-ENOMEM);
14e50e57
DM		 2822}
2823
9d6ec938 2824struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
b23dd4fe 2825 struct sock *sk)
1da177e4 2826{
9d6ec938 2827 struct rtable *rt = __ip_route_output_key(net, flp4);
1da177e4 2828
b23dd4fe
DM		 2829 if (IS_ERR(rt))
2830 return rt;
1da177e4 2831
56157872 2832 if (flp4->flowi4_proto)
9d6ec938
DM		 2833 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2834 flowi4_to_flowi(flp4),
2835 sk, 0);
1da177e4 2836
b23dd4fe 2837 return rt;
1da177e4 2838}
d8c97a94
ACM		 2839EXPORT_SYMBOL_GPL(ip_route_output_flow);
2840
4feb88e5
BT		 2841static int rt_fill_info(struct net *net,
2842 struct sk_buff *skb, u32 pid, u32 seq, int event,
b6544c0b 2843 int nowait, unsigned int flags)
1da177e4 2844{
511c3f92 2845 struct rtable *rt = skb_rtable(skb);
1da177e4 2846 struct rtmsg *r;
be403ea1 2847 struct nlmsghdr *nlh;
2bc8ca40 2848 unsigned long expires = 0;
81166dd6 2849 u32 id = 0, error;
be403ea1
TG		 2850
2851 nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2852 if (nlh == NULL)
26932566 2853 return -EMSGSIZE;
be403ea1
TG		 2854
2855 r = nlmsg_data(nlh);
1da177e4
LT		 2856 r->rtm_family = AF_INET;
2857 r->rtm_dst_len = 32;
2858 r->rtm_src_len = 0;
475949d8 2859 r->rtm_tos = rt->rt_key_tos;
1da177e4 2860 r->rtm_table = RT_TABLE_MAIN;
f3756b79
DM		 2861 if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2862 goto nla_put_failure;
1da177e4
LT		 2863 r->rtm_type = rt->rt_type;
2864 r->rtm_scope = RT_SCOPE_UNIVERSE;
2865 r->rtm_protocol = RTPROT_UNSPEC;
2866 r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2867 if (rt->rt_flags & RTCF_NOTIFY)
2868 r->rtm_flags |= RTM_F_NOTIFY;
be403ea1 2869
f3756b79
DM		 2870 if (nla_put_be32(skb, RTA_DST, rt->rt_dst))
2871 goto nla_put_failure;
5e2b61f7 2872 if (rt->rt_key_src) {
1da177e4 2873 r->rtm_src_len = 32;
f3756b79
DM		 2874 if (nla_put_be32(skb, RTA_SRC, rt->rt_key_src))
2875 goto nla_put_failure;
1da177e4 2876 }
f3756b79
DM		 2877 if (rt->dst.dev &&
2878 nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2879 goto nla_put_failure;
c7066f70 2880#ifdef CONFIG_IP_ROUTE_CLASSID
f3756b79
DM		 2881 if (rt->dst.tclassid &&
2882 nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2883 goto nla_put_failure;
1da177e4 2884#endif
41347dcd
DM		 2885 if (!rt_is_input_route(rt) &&
2886 rt->rt_src != rt->rt_key_src) {
f3756b79
DM		 2887 if (nla_put_be32(skb, RTA_PREFSRC, rt->rt_src))
2888 goto nla_put_failure;
2889 }
2890 if (rt->rt_dst != rt->rt_gateway &&
2891 nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2892 goto nla_put_failure;
be403ea1 2893
defb3519 2894 if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
be403ea1
TG		 2895 goto nla_put_failure;
2896
f3756b79
DM		 2897 if (rt->rt_mark &&
2898 nla_put_be32(skb, RTA_MARK, rt->rt_mark))
2899 goto nla_put_failure;
963bfeee 2900
d8d1f30b 2901 error = rt->dst.error;
97bab73f
DM		 2902 if (rt_has_peer(rt)) {
2903 const struct inet_peer *peer = rt_peer_ptr(rt);
2904 inet_peer_refcheck(peer);
fe6fe792 2905 id = atomic_read(&peer->ip_id_count) & 0xffff;
fe6fe792 2906 expires = ACCESS_ONCE(peer->pmtu_expires);
2bc8ca40
SK		 2907 if (expires) {
2908 if (time_before(jiffies, expires))
2909 expires -= jiffies;
2910 else
2911 expires = 0;
2912 }
1da177e4 2913 }
be403ea1 2914
c7537967 2915 if (rt_is_input_route(rt)) {
1da177e4 2916#ifdef CONFIG_IP_MROUTE
e448515c 2917 __be32 dst = rt->rt_dst;
1da177e4 2918
f97c1e0c 2919 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
4feb88e5 2920 IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
9a1b9496
DM		 2921 int err = ipmr_get_route(net, skb,
2922 rt->rt_src, rt->rt_dst,
2923 r, nowait);
1da177e4
LT		 2924 if (err <= 0) {
2925 if (!nowait) {
2926 if (err == 0)
2927 return 0;
be403ea1 2928 goto nla_put_failure;
1da177e4
LT		 2929 } else {
2930 if (err == -EMSGSIZE)
be403ea1 2931 goto nla_put_failure;
e3703b3d 2932 error = err;
1da177e4
LT		 2933 }
2934 }
2935 } else
2936#endif
f3756b79
DM		 2937 if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2938 goto nla_put_failure;
1da177e4
LT		 2939 }
2940
81166dd6 2941 if (rtnl_put_cacheinfo(skb, &rt->dst, id, 0, 0,
e3703b3d
TG		 2942 expires, error) < 0)
2943 goto nla_put_failure;
be403ea1
TG		 2944
2945 return nlmsg_end(skb, nlh);
1da177e4 2946
be403ea1 2947nla_put_failure:
26932566
PM		 2948 nlmsg_cancel(skb, nlh);
2949 return -EMSGSIZE;
1da177e4
LT		 2950}
2951
5e73ea1a 2952static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, void *arg)
1da177e4 2953{
3b1e0a65 2954 struct net *net = sock_net(in_skb->sk);
d889ce3b
TG		 2955 struct rtmsg *rtm;
2956 struct nlattr *tb[RTA_MAX+1];
1da177e4 2957 struct rtable *rt = NULL;
9e12bb22
AV		 2958 __be32 dst = 0;
2959 __be32 src = 0;
2960 u32 iif;
d889ce3b 2961 int err;
963bfeee 2962 int mark;
1da177e4
LT		 2963 struct sk_buff *skb;
2964
d889ce3b
TG		 2965 err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2966 if (err < 0)
2967 goto errout;
2968
2969 rtm = nlmsg_data(nlh);
2970
1da177e4 2971 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
d889ce3b
TG		 2972 if (skb == NULL) {
2973 err = -ENOBUFS;
2974 goto errout;
2975 }
1da177e4
LT		 2976
2977 /* Reserve room for dummy headers, this skb can pass
2978 through good chunk of routing engine.
2979 */
459a98ed 2980 skb_reset_mac_header(skb);
c1d2bbe1 2981 skb_reset_network_header(skb);
d2c962b8
SH		 2982
2983 /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
eddc9ec5 2984 ip_hdr(skb)->protocol = IPPROTO_ICMP;
1da177e4
LT		 2985 skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2986
17fb2c64
AV		 2987 src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2988 dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
d889ce3b 2989 iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
963bfeee 2990 mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
1da177e4
LT		 2991
2992 if (iif) {
d889ce3b
TG		 2993 struct net_device *dev;
2994
1937504d 2995 dev = __dev_get_by_index(net, iif);
d889ce3b
TG		 2996 if (dev == NULL) {
2997 err = -ENODEV;
2998 goto errout_free;
2999 }
3000
1da177e4
LT		 3001 skb->protocol = htons(ETH_P_IP);
3002 skb->dev = dev;
963bfeee 3003 skb->mark = mark;
1da177e4
LT		 3004 local_bh_disable();
3005 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
3006 local_bh_enable();
d889ce3b 3007
511c3f92 3008 rt = skb_rtable(skb);
d8d1f30b
CG		 3009 if (err == 0 && rt->dst.error)
3010 err = -rt->dst.error;
1da177e4 3011 } else {
68a5e3dd
DM		 3012 struct flowi4 fl4 = {
3013 .daddr = dst,
3014 .saddr = src,
3015 .flowi4_tos = rtm->rtm_tos,
3016 .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
3017 .flowi4_mark = mark,
d889ce3b 3018 };
9d6ec938 3019 rt = ip_route_output_key(net, &fl4);
b23dd4fe
DM		 3020
3021 err = 0;
3022 if (IS_ERR(rt))
3023 err = PTR_ERR(rt);
1da177e4 3024 }
d889ce3b 3025
1da177e4 3026 if (err)
d889ce3b 3027 goto errout_free;
1da177e4 3028
d8d1f30b 3029 skb_dst_set(skb, &rt->dst);
1da177e4
LT		 3030 if (rtm->rtm_flags & RTM_F_NOTIFY)
3031 rt->rt_flags |= RTCF_NOTIFY;
3032
4feb88e5 3033 err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
1937504d 3034 RTM_NEWROUTE, 0, 0);
d889ce3b
TG		 3035 if (err <= 0)
3036 goto errout_free;
1da177e4 3037
1937504d 3038 err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
d889ce3b 3039errout:
2942e900 3040 return err;
1da177e4 3041
d889ce3b 3042errout_free:
1da177e4 3043 kfree_skb(skb);
d889ce3b 3044 goto errout;
1da177e4
LT		 3045}
3046
3047int ip_rt_dump(struct sk_buff *skb, struct netlink_callback *cb)
3048{
3049 struct rtable *rt;
3050 int h, s_h;
3051 int idx, s_idx;
1937504d
DL		 3052 struct net *net;
3053
3b1e0a65 3054 net = sock_net(skb->sk);
1da177e4
LT		 3055
3056 s_h = cb->args[0];
d8c92830
ED		 3057 if (s_h < 0)
3058 s_h = 0;
1da177e4 3059 s_idx = idx = cb->args[1];
a6272665
ED		 3060 for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3061 if (!rt_hash_table[h].chain)
3062 continue;
1da177e4 3063 rcu_read_lock_bh();
a898def2 3064 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
d8d1f30b
CG		 3065 rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3066 if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
1da177e4 3067 continue;
e84f84f2 3068 if (rt_is_expired(rt))
29e75252 3069 continue;
d8d1f30b 3070 skb_dst_set_noref(skb, &rt->dst);
4feb88e5 3071 if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
e905a9ed 3072 cb->nlh->nlmsg_seq, RTM_NEWROUTE,
b6544c0b 3073 1, NLM_F_MULTI) <= 0) {
adf30907 3074 skb_dst_drop(skb);
1da177e4
LT		 3075 rcu_read_unlock_bh();
3076 goto done;
3077 }
adf30907 3078 skb_dst_drop(skb);
1da177e4
LT		 3079 }
3080 rcu_read_unlock_bh();
3081 }
3082
3083done:
3084 cb->args[0] = h;
3085 cb->args[1] = idx;
3086 return skb->len;
3087}
3088
3089void ip_rt_multicast_event(struct in_device *in_dev)
3090{
76e6ebfb 3091 rt_cache_flush(dev_net(in_dev->dev), 0);
1da177e4
LT		 3092}
3093
3094#ifdef CONFIG_SYSCTL
81c684d1 3095static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
8d65af78 3096 void __user *buffer,
1da177e4
LT		 3097 size_t *lenp, loff_t *ppos)
3098{
3099 if (write) {
639e104f 3100 int flush_delay;
81c684d1 3101 ctl_table ctl;
39a23e75 3102 struct net *net;
639e104f 3103
81c684d1
DL		 3104 memcpy(&ctl, __ctl, sizeof(ctl));
3105 ctl.data = &flush_delay;
8d65af78 3106 proc_dointvec(&ctl, write, buffer, lenp, ppos);
639e104f 3107
81c684d1 3108 net = (struct net *)__ctl->extra1;
39a23e75 3109 rt_cache_flush(net, flush_delay);
1da177e4 3110 return 0;
e905a9ed 3111 }
1da177e4
LT		 3112
3113 return -EINVAL;
3114}
3115
eeb61f71 3116static ctl_table ipv4_route_table[] = {
1da177e4 3117 {
1da177e4
LT		 3118 .procname = "gc_thresh",
3119 .data = &ipv4_dst_ops.gc_thresh,
3120 .maxlen = sizeof(int),
3121 .mode = 0644,
6d9f239a 3122 .proc_handler = proc_dointvec,
1da177e4
LT		 3123 },
3124 {
1da177e4
LT		 3125 .procname = "max_size",
3126 .data = &ip_rt_max_size,
3127 .maxlen = sizeof(int),
3128 .mode = 0644,
6d9f239a 3129 .proc_handler = proc_dointvec,
1da177e4
LT		 3130 },
3131 {
3132 /* Deprecated. Use gc_min_interval_ms */
e905a9ed 3133
1da177e4
LT		 3134 .procname = "gc_min_interval",
3135 .data = &ip_rt_gc_min_interval,
3136 .maxlen = sizeof(int),
3137 .mode = 0644,
6d9f239a 3138 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 3139 },
3140 {
1da177e4
LT		 3141 .procname = "gc_min_interval_ms",
3142 .data = &ip_rt_gc_min_interval,
3143 .maxlen = sizeof(int),
3144 .mode = 0644,
6d9f239a 3145 .proc_handler = proc_dointvec_ms_jiffies,
1da177e4
LT		 3146 },
3147 {
1da177e4
LT		 3148 .procname = "gc_timeout",
3149 .data = &ip_rt_gc_timeout,
3150 .maxlen = sizeof(int),
3151 .mode = 0644,
6d9f239a 3152 .proc_handler = proc_dointvec_jiffies,
1da177e4 3153 },
9f28a2fc
ED		 3154 {
3155 .procname = "gc_interval",
3156 .data = &ip_rt_gc_interval,
3157 .maxlen = sizeof(int),
3158 .mode = 0644,
3159 .proc_handler = proc_dointvec_jiffies,
3160 },
1da177e4 3161 {
1da177e4
LT		 3162 .procname = "redirect_load",
3163 .data = &ip_rt_redirect_load,
3164 .maxlen = sizeof(int),
3165 .mode = 0644,
6d9f239a 3166 .proc_handler = proc_dointvec,
1da177e4
LT		 3167 },
3168 {
1da177e4
LT		 3169 .procname = "redirect_number",
3170 .data = &ip_rt_redirect_number,
3171 .maxlen = sizeof(int),
3172 .mode = 0644,
6d9f239a 3173 .proc_handler = proc_dointvec,
1da177e4
LT		 3174 },
3175 {
1da177e4
LT		 3176 .procname = "redirect_silence",
3177 .data = &ip_rt_redirect_silence,
3178 .maxlen = sizeof(int),
3179 .mode = 0644,
6d9f239a 3180 .proc_handler = proc_dointvec,
1da177e4
LT		 3181 },
3182 {
1da177e4
LT		 3183 .procname = "error_cost",
3184 .data = &ip_rt_error_cost,
3185 .maxlen = sizeof(int),
3186 .mode = 0644,
6d9f239a 3187 .proc_handler = proc_dointvec,
1da177e4
LT		 3188 },
3189 {
1da177e4
LT		 3190 .procname = "error_burst",
3191 .data = &ip_rt_error_burst,
3192 .maxlen = sizeof(int),
3193 .mode = 0644,
6d9f239a 3194 .proc_handler = proc_dointvec,
1da177e4
LT		 3195 },
3196 {
1da177e4
LT		 3197 .procname = "gc_elasticity",
3198 .data = &ip_rt_gc_elasticity,
3199 .maxlen = sizeof(int),
3200 .mode = 0644,
6d9f239a 3201 .proc_handler = proc_dointvec,
1da177e4
LT		 3202 },
3203 {
1da177e4
LT		 3204 .procname = "mtu_expires",
3205 .data = &ip_rt_mtu_expires,
3206 .maxlen = sizeof(int),
3207 .mode = 0644,
6d9f239a 3208 .proc_handler = proc_dointvec_jiffies,
1da177e4
LT		 3209 },
3210 {
1da177e4
LT		 3211 .procname = "min_pmtu",
3212 .data = &ip_rt_min_pmtu,
3213 .maxlen = sizeof(int),
3214 .mode = 0644,
6d9f239a 3215 .proc_handler = proc_dointvec,
1da177e4
LT		 3216 },
3217 {
1da177e4
LT		 3218 .procname = "min_adv_mss",
3219 .data = &ip_rt_min_advmss,
3220 .maxlen = sizeof(int),
3221 .mode = 0644,
6d9f239a 3222 .proc_handler = proc_dointvec,
1da177e4 3223 },
f8572d8f 3224 { }
1da177e4 3225};
39a23e75 3226
39a23e75
DL		 3227static struct ctl_table ipv4_route_flush_table[] = {
3228 {
39a23e75
DL		 3229 .procname = "flush",
3230 .maxlen = sizeof(int),
3231 .mode = 0200,
6d9f239a 3232 .proc_handler = ipv4_sysctl_rtcache_flush,
39a23e75 3233 },
f8572d8f 3234 { },
39a23e75
DL		 3235};
3236
3237static __net_init int sysctl_route_net_init(struct net *net)
3238{
3239 struct ctl_table *tbl;
3240
3241 tbl = ipv4_route_flush_table;
09ad9bc7 3242 if (!net_eq(net, &init_net)) {
39a23e75
DL		 3243 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3244 if (tbl == NULL)
3245 goto err_dup;
3246 }
3247 tbl[0].extra1 = net;
3248
ec8f23ce 3249 net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
39a23e75
DL		 3250 if (net->ipv4.route_hdr == NULL)
3251 goto err_reg;
3252 return 0;
3253
3254err_reg:
3255 if (tbl != ipv4_route_flush_table)
3256 kfree(tbl);
3257err_dup:
3258 return -ENOMEM;
3259}
3260
3261static __net_exit void sysctl_route_net_exit(struct net *net)
3262{
3263 struct ctl_table *tbl;
3264
3265 tbl = net->ipv4.route_hdr->ctl_table_arg;
3266 unregister_net_sysctl_table(net->ipv4.route_hdr);
3267 BUG_ON(tbl == ipv4_route_flush_table);
3268 kfree(tbl);
3269}
3270
3271static __net_initdata struct pernet_operations sysctl_route_ops = {
3272 .init = sysctl_route_net_init,
3273 .exit = sysctl_route_net_exit,
3274};
1da177e4
LT		 3275#endif
3276
3ee94372 3277static __net_init int rt_genid_init(struct net *net)
9f5e97e5 3278{
3ee94372
NH		 3279 get_random_bytes(&net->ipv4.rt_genid,
3280 sizeof(net->ipv4.rt_genid));
436c3b66
DM		 3281 get_random_bytes(&net->ipv4.dev_addr_genid,
3282 sizeof(net->ipv4.dev_addr_genid));
9f5e97e5
DL		 3283 return 0;
3284}
3285
3ee94372
NH		 3286static __net_initdata struct pernet_operations rt_genid_ops = {
3287 .init = rt_genid_init,
9f5e97e5
DL		 3288};
3289
c3426b47
DM		 3290static int __net_init ipv4_inetpeer_init(struct net *net)
3291{
3292 struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3293
3294 if (!bp)
3295 return -ENOMEM;
3296 inet_peer_base_init(bp);
3297 net->ipv4.peers = bp;
3298 return 0;
3299}
3300
3301static void __net_exit ipv4_inetpeer_exit(struct net *net)
3302{
3303 struct inet_peer_base *bp = net->ipv4.peers;
3304
3305 net->ipv4.peers = NULL;
56a6b248 3306 inetpeer_invalidate_tree(bp);
c3426b47
DM		 3307 kfree(bp);
3308}
3309
3310static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3311 .init = ipv4_inetpeer_init,
3312 .exit = ipv4_inetpeer_exit,
3313};
9f5e97e5 3314
c7066f70 3315#ifdef CONFIG_IP_ROUTE_CLASSID
7d720c3e 3316struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
c7066f70 3317#endif /* CONFIG_IP_ROUTE_CLASSID */
1da177e4
LT		 3318
3319static __initdata unsigned long rhash_entries;
3320static int __init set_rhash_entries(char *str)
3321{
413c27d8
EZ		 3322 ssize_t ret;
3323
1da177e4
LT		 3324 if (!str)
3325 return 0;
413c27d8
EZ		 3326
3327 ret = kstrtoul(str, 0, &rhash_entries);
3328 if (ret)
3329 return 0;
3330
1da177e4
LT		 3331 return 1;
3332}
3333__setup("rhash_entries=", set_rhash_entries);
3334
3335int __init ip_rt_init(void)
3336{
424c4b70 3337 int rc = 0;
1da177e4 3338
c7066f70 3339#ifdef CONFIG_IP_ROUTE_CLASSID
0dcec8c2 3340 ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
1da177e4
LT		 3341 if (!ip_rt_acct)
3342 panic("IP: failed to allocate ip_rt_acct\n");
1da177e4
LT		 3343#endif
3344
e5d679f3
AD		 3345 ipv4_dst_ops.kmem_cachep =
3346 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
20c2df83 3347 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
1da177e4 3348
14e50e57
DM		 3349 ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3350
fc66f95c
ED		 3351 if (dst_entries_init(&ipv4_dst_ops) < 0)
3352 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3353
3354 if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3355 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3356
424c4b70
ED		 3357 rt_hash_table = (struct rt_hash_bucket *)
3358 alloc_large_system_hash("IP route cache",
3359 sizeof(struct rt_hash_bucket),
3360 rhash_entries,
4481374c 3361 (totalram_pages >= 128 * 1024) ?
18955cfc 3362 15 : 17,
8d1502de 3363 0,
424c4b70
ED		 3364 &rt_hash_log,
3365 &rt_hash_mask,
31fe62b9 3366 0,
c9503e0f 3367 rhash_entries ? 0 : 512 * 1024);
22c047cc
ED		 3368 memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3369 rt_hash_lock_init();
1da177e4
LT		 3370
3371 ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3372 ip_rt_max_size = (rt_hash_mask + 1) * 16;
3373
1da177e4
LT		 3374 devinet_init();
3375 ip_fib_init();
3376
9f28a2fc
ED		 3377 INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3378 expires_ljiffies = jiffies;
3379 schedule_delayed_work(&expires_work,
3380 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3381
73b38711 3382 if (ip_rt_proc_init())
058bd4d2 3383 pr_err("Unable to create route proc files\n");
1da177e4
LT		 3384#ifdef CONFIG_XFRM
3385 xfrm_init();
a33bc5c1 3386 xfrm4_init(ip_rt_max_size);
1da177e4 3387#endif
c7ac8679 3388 rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
63f3444f 3389
39a23e75
DL		 3390#ifdef CONFIG_SYSCTL
3391 register_pernet_subsys(&sysctl_route_ops);
3392#endif
3ee94372 3393 register_pernet_subsys(&rt_genid_ops);
c3426b47 3394 register_pernet_subsys(&ipv4_inetpeer_ops);
1da177e4
LT		 3395 return rc;
3396}
3397
a1bc6eb4 3398#ifdef CONFIG_SYSCTL
eeb61f71
AV		 3399/*
3400 * We really need to sanitize the damn ipv4 init order, then all
3401 * this nonsense will go away.
3402 */
3403void __init ip_static_sysctl_init(void)
3404{
4e5ca785 3405 register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
eeb61f71 3406}
a1bc6eb4 3407#endif
Linux 6.x block layer and io_uring tree(s)