net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <linux/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #endif
 112 #include <net/secure_seq.h>
 113 #include <net/ip_tunnels.h>
 114 #include <net/l3mdev.h>
 115
 116 #include "fib_lookup.h"
 117
 118 #define RT_FL_TOS(oldflp4) \
 119         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_redirect_number __read_mostly  = 9;
 125 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 126 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 127 static int ip_rt_error_cost __read_mostly       = HZ;
 128 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static u32 ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132
 133 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 134
 135 /*
 136  *      Interface to generic destination cache.
 137  */
 138
 139 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 140 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 141 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 142 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 143 static void              ipv4_link_failure(struct sk_buff *skb);
 144 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 145                                            struct sk_buff *skb, u32 mtu);
 146 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 147                                         struct sk_buff *skb);
 148 static void             ipv4_dst_destroy(struct dst_entry *dst);
 149
 150 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 151 {
 152         WARN_ON(1);
 153         return NULL;
 154 }
 155
 156 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 157                                            struct sk_buff *skb,
 158                                            const void *daddr);
 159 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
 160
 161 static struct dst_ops ipv4_dst_ops = {
 162         .family =               AF_INET,
 163         .check =                ipv4_dst_check,
 164         .default_advmss =       ipv4_default_advmss,
 165         .mtu =                  ipv4_mtu,
 166         .cow_metrics =          ipv4_cow_metrics,
 167         .destroy =              ipv4_dst_destroy,
 168         .negative_advice =      ipv4_negative_advice,
 169         .link_failure =         ipv4_link_failure,
 170         .update_pmtu =          ip_rt_update_pmtu,
 171         .redirect =             ip_do_redirect,
 172         .local_out =            __ip_local_out,
 173         .neigh_lookup =         ipv4_neigh_lookup,
 174         .confirm_neigh =        ipv4_confirm_neigh,
 175 };
 176
 177 #define ECN_OR_COST(class)      TC_PRIO_##class
 178
 179 const __u8 ip_tos2prio[16] = {
 180         TC_PRIO_BESTEFFORT,
 181         ECN_OR_COST(BESTEFFORT),
 182         TC_PRIO_BESTEFFORT,
 183         ECN_OR_COST(BESTEFFORT),
 184         TC_PRIO_BULK,
 185         ECN_OR_COST(BULK),
 186         TC_PRIO_BULK,
 187         ECN_OR_COST(BULK),
 188         TC_PRIO_INTERACTIVE,
 189         ECN_OR_COST(INTERACTIVE),
 190         TC_PRIO_INTERACTIVE,
 191         ECN_OR_COST(INTERACTIVE),
 192         TC_PRIO_INTERACTIVE_BULK,
 193         ECN_OR_COST(INTERACTIVE_BULK),
 194         TC_PRIO_INTERACTIVE_BULK,
 195         ECN_OR_COST(INTERACTIVE_BULK)
 196 };
 197 EXPORT_SYMBOL(ip_tos2prio);
 198
 199 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 200 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 201
 202 #ifdef CONFIG_PROC_FS
 203 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 204 {
 205         if (*pos)
 206                 return NULL;
 207         return SEQ_START_TOKEN;
 208 }
 209
 210 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 211 {
 212         ++*pos;
 213         return NULL;
 214 }
 215
 216 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 217 {
 218 }
 219
 220 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 221 {
 222         if (v == SEQ_START_TOKEN)
 223                 seq_printf(seq, "%-127s\n",
 224                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 225                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 226                            "HHUptod\tSpecDst");
 227         return 0;
 228 }
 229
 230 static const struct seq_operations rt_cache_seq_ops = {
 231         .start  = rt_cache_seq_start,
 232         .next   = rt_cache_seq_next,
 233         .stop   = rt_cache_seq_stop,
 234         .show   = rt_cache_seq_show,
 235 };
 236
 237 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 238 {
 239         return seq_open(file, &rt_cache_seq_ops);
 240 }
 241
 242 static const struct file_operations rt_cache_seq_fops = {
 243         .open    = rt_cache_seq_open,
 244         .read    = seq_read,
 245         .llseek  = seq_lseek,
 246         .release = seq_release,
 247 };
 248
 249
 250 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 251 {
 252         int cpu;
 253
 254         if (*pos == 0)
 255                 return SEQ_START_TOKEN;
 256
 257         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 258                 if (!cpu_possible(cpu))
 259                         continue;
 260                 *pos = cpu+1;
 261                 return &per_cpu(rt_cache_stat, cpu);
 262         }
 263         return NULL;
 264 }
 265
 266 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 267 {
 268         int cpu;
 269
 270         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277
 278 }
 279
 280 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 281 {
 282
 283 }
 284
 285 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 286 {
 287         struct rt_cache_stat *st = v;
 288
 289         if (v == SEQ_START_TOKEN) {
 290                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 291                 return 0;
 292         }
 293
 294         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 295                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 296                    dst_entries_get_slow(&ipv4_dst_ops),
 297                    0, /* st->in_hit */
 298                    st->in_slow_tot,
 299                    st->in_slow_mc,
 300                    st->in_no_route,
 301                    st->in_brd,
 302                    st->in_martian_dst,
 303                    st->in_martian_src,
 304
 305                    0, /* st->out_hit */
 306                    st->out_slow_tot,
 307                    st->out_slow_mc,
 308
 309                    0, /* st->gc_total */
 310                    0, /* st->gc_ignored */
 311                    0, /* st->gc_goal_miss */
 312                    0, /* st->gc_dst_overflow */
 313                    0, /* st->in_hlist_search */
 314                    0  /* st->out_hlist_search */
 315                 );
 316         return 0;
 317 }
 318
 319 static const struct seq_operations rt_cpu_seq_ops = {
 320         .start  = rt_cpu_seq_start,
 321         .next   = rt_cpu_seq_next,
 322         .stop   = rt_cpu_seq_stop,
 323         .show   = rt_cpu_seq_show,
 324 };
 325
 326
 327 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 328 {
 329         return seq_open(file, &rt_cpu_seq_ops);
 330 }
 331
 332 static const struct file_operations rt_cpu_seq_fops = {
 333         .open    = rt_cpu_seq_open,
 334         .read    = seq_read,
 335         .llseek  = seq_lseek,
 336         .release = seq_release,
 337 };
 338
 339 #ifdef CONFIG_IP_ROUTE_CLASSID
 340 static int rt_acct_proc_show(struct seq_file *m, void *v)
 341 {
 342         struct ip_rt_acct *dst, *src;
 343         unsigned int i, j;
 344
 345         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 346         if (!dst)
 347                 return -ENOMEM;
 348
 349         for_each_possible_cpu(i) {
 350                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 351                 for (j = 0; j < 256; j++) {
 352                         dst[j].o_bytes   += src[j].o_bytes;
 353                         dst[j].o_packets += src[j].o_packets;
 354                         dst[j].i_bytes   += src[j].i_bytes;
 355                         dst[j].i_packets += src[j].i_packets;
 356                 }
 357         }
 358
 359         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 360         kfree(dst);
 361         return 0;
 362 }
 363 #endif
 364
 365 static int __net_init ip_rt_do_proc_init(struct net *net)
 366 {
 367         struct proc_dir_entry *pde;
 368
 369         pde = proc_create("rt_cache", 0444, net->proc_net,
 370                           &rt_cache_seq_fops);
 371         if (!pde)
 372                 goto err1;
 373
 374         pde = proc_create("rt_cache", 0444,
 375                           net->proc_net_stat, &rt_cpu_seq_fops);
 376         if (!pde)
 377                 goto err2;
 378
 379 #ifdef CONFIG_IP_ROUTE_CLASSID
 380         pde = proc_create_single("rt_acct", 0, net->proc_net,
 381                         rt_acct_proc_show);
 382         if (!pde)
 383                 goto err3;
 384 #endif
 385         return 0;
 386
 387 #ifdef CONFIG_IP_ROUTE_CLASSID
 388 err3:
 389         remove_proc_entry("rt_cache", net->proc_net_stat);
 390 #endif
 391 err2:
 392         remove_proc_entry("rt_cache", net->proc_net);
 393 err1:
 394         return -ENOMEM;
 395 }
 396
 397 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 398 {
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400         remove_proc_entry("rt_cache", net->proc_net);
 401 #ifdef CONFIG_IP_ROUTE_CLASSID
 402         remove_proc_entry("rt_acct", net->proc_net);
 403 #endif
 404 }
 405
 406 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 407         .init = ip_rt_do_proc_init,
 408         .exit = ip_rt_do_proc_exit,
 409 };
 410
 411 static int __init ip_rt_proc_init(void)
 412 {
 413         return register_pernet_subsys(&ip_rt_proc_ops);
 414 }
 415
 416 #else
 417 static inline int ip_rt_proc_init(void)
 418 {
 419         return 0;
 420 }
 421 #endif /* CONFIG_PROC_FS */
 422
 423 static inline bool rt_is_expired(const struct rtable *rth)
 424 {
 425         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 426 }
 427
 428 void rt_cache_flush(struct net *net)
 429 {
 430         rt_genid_bump_ipv4(net);
 431 }
 432
 433 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 434                                            struct sk_buff *skb,
 435                                            const void *daddr)
 436 {
 437         struct net_device *dev = dst->dev;
 438         const __be32 *pkey = daddr;
 439         const struct rtable *rt;
 440         struct neighbour *n;
 441
 442         rt = (const struct rtable *) dst;
 443         if (rt->rt_gateway)
 444                 pkey = (const __be32 *) &rt->rt_gateway;
 445         else if (skb)
 446                 pkey = &ip_hdr(skb)->daddr;
 447
 448         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 449         if (n)
 450                 return n;
 451         return neigh_create(&arp_tbl, pkey, dev);
 452 }
 453
 454 static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
 455 {
 456         struct net_device *dev = dst->dev;
 457         const __be32 *pkey = daddr;
 458         const struct rtable *rt;
 459
 460         rt = (const struct rtable *)dst;
 461         if (rt->rt_gateway)
 462                 pkey = (const __be32 *)&rt->rt_gateway;
 463         else if (!daddr ||
 464                  (rt->rt_flags &
 465                   (RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL)))
 466                 return;
 467
 468         __ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
 469 }
 470
 471 #define IP_IDENTS_SZ 2048u
 472
 473 static atomic_t *ip_idents __read_mostly;
 474 static u32 *ip_tstamps __read_mostly;
 475
 476 /* In order to protect privacy, we add a perturbation to identifiers
 477  * if one generator is seldom used. This makes hard for an attacker
 478  * to infer how many packets were sent between two points in time.
 479  */
 480 u32 ip_idents_reserve(u32 hash, int segs)
 481 {
 482         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 483         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 484         u32 old = READ_ONCE(*p_tstamp);
 485         u32 now = (u32)jiffies;
 486         u32 new, delta = 0;
 487
 488         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 489                 delta = prandom_u32_max(now - old);
 490
 491         /* Do not use atomic_add_return() as it makes UBSAN unhappy */
 492         do {
 493                 old = (u32)atomic_read(p_id);
 494                 new = old + delta + segs;
 495         } while (atomic_cmpxchg(p_id, old, new) != old);
 496
 497         return new - segs;
 498 }
 499 EXPORT_SYMBOL(ip_idents_reserve);
 500
 501 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 502 {
 503         static u32 ip_idents_hashrnd __read_mostly;
 504         u32 hash, id;
 505
 506         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 507
 508         hash = jhash_3words((__force u32)iph->daddr,
 509                             (__force u32)iph->saddr,
 510                             iph->protocol ^ net_hash_mix(net),
 511                             ip_idents_hashrnd);
 512         id = ip_idents_reserve(hash, segs);
 513         iph->id = htons(id);
 514 }
 515 EXPORT_SYMBOL(__ip_select_ident);
 516
 517 static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
 518                              const struct sock *sk,
 519                              const struct iphdr *iph,
 520                              int oif, u8 tos,
 521                              u8 prot, u32 mark, int flow_flags)
 522 {
 523         if (sk) {
 524                 const struct inet_sock *inet = inet_sk(sk);
 525
 526                 oif = sk->sk_bound_dev_if;
 527                 mark = sk->sk_mark;
 528                 tos = RT_CONN_FLAGS(sk);
 529                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 530         }
 531         flowi4_init_output(fl4, oif, mark, tos,
 532                            RT_SCOPE_UNIVERSE, prot,
 533                            flow_flags,
 534                            iph->daddr, iph->saddr, 0, 0,
 535                            sock_net_uid(net, sk));
 536 }
 537
 538 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 539                                const struct sock *sk)
 540 {
 541         const struct net *net = dev_net(skb->dev);
 542         const struct iphdr *iph = ip_hdr(skb);
 543         int oif = skb->dev->ifindex;
 544         u8 tos = RT_TOS(iph->tos);
 545         u8 prot = iph->protocol;
 546         u32 mark = skb->mark;
 547
 548         __build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
 549 }
 550
 551 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 552 {
 553         const struct inet_sock *inet = inet_sk(sk);
 554         const struct ip_options_rcu *inet_opt;
 555         __be32 daddr = inet->inet_daddr;
 556
 557         rcu_read_lock();
 558         inet_opt = rcu_dereference(inet->inet_opt);
 559         if (inet_opt && inet_opt->opt.srr)
 560                 daddr = inet_opt->opt.faddr;
 561         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 562                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 563                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 564                            inet_sk_flowi_flags(sk),
 565                            daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
 566         rcu_read_unlock();
 567 }
 568
 569 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 570                                  const struct sk_buff *skb)
 571 {
 572         if (skb)
 573                 build_skb_flow_key(fl4, skb, sk);
 574         else
 575                 build_sk_flow_key(fl4, sk);
 576 }
 577
 578 static DEFINE_SPINLOCK(fnhe_lock);
 579
 580 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 581 {
 582         struct rtable *rt;
 583
 584         rt = rcu_dereference(fnhe->fnhe_rth_input);
 585         if (rt) {
 586                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 587                 dst_dev_put(&rt->dst);
 588                 dst_release(&rt->dst);
 589         }
 590         rt = rcu_dereference(fnhe->fnhe_rth_output);
 591         if (rt) {
 592                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 593                 dst_dev_put(&rt->dst);
 594                 dst_release(&rt->dst);
 595         }
 596 }
 597
 598 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 599 {
 600         struct fib_nh_exception *fnhe, *oldest;
 601
 602         oldest = rcu_dereference(hash->chain);
 603         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 604              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 605                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 606                         oldest = fnhe;
 607         }
 608         fnhe_flush_routes(oldest);
 609         return oldest;
 610 }
 611
 612 static inline u32 fnhe_hashfun(__be32 daddr)
 613 {
 614         static u32 fnhe_hashrnd __read_mostly;
 615         u32 hval;
 616
 617         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 618         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 619         return hash_32(hval, FNHE_HASH_SHIFT);
 620 }
 621
 622 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 623 {
 624         rt->rt_pmtu = fnhe->fnhe_pmtu;
 625         rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
 626         rt->dst.expires = fnhe->fnhe_expires;
 627
 628         if (fnhe->fnhe_gw) {
 629                 rt->rt_flags |= RTCF_REDIRECTED;
 630                 rt->rt_gateway = fnhe->fnhe_gw;
 631                 rt->rt_uses_gateway = 1;
 632         }
 633 }
 634
 635 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 636                                   u32 pmtu, bool lock, unsigned long expires)
 637 {
 638         struct fnhe_hash_bucket *hash;
 639         struct fib_nh_exception *fnhe;
 640         struct rtable *rt;
 641         u32 genid, hval;
 642         unsigned int i;
 643         int depth;
 644
 645         genid = fnhe_genid(dev_net(nh->nh_dev));
 646         hval = fnhe_hashfun(daddr);
 647
 648         spin_lock_bh(&fnhe_lock);
 649
 650         hash = rcu_dereference(nh->nh_exceptions);
 651         if (!hash) {
 652                 hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
 653                 if (!hash)
 654                         goto out_unlock;
 655                 rcu_assign_pointer(nh->nh_exceptions, hash);
 656         }
 657
 658         hash += hval;
 659
 660         depth = 0;
 661         for (fnhe = rcu_dereference(hash->chain); fnhe;
 662              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 663                 if (fnhe->fnhe_daddr == daddr)
 664                         break;
 665                 depth++;
 666         }
 667
 668         if (fnhe) {
 669                 if (fnhe->fnhe_genid != genid)
 670                         fnhe->fnhe_genid = genid;
 671                 if (gw)
 672                         fnhe->fnhe_gw = gw;
 673                 if (pmtu) {
 674                         fnhe->fnhe_pmtu = pmtu;
 675                         fnhe->fnhe_mtu_locked = lock;
 676                 }
 677                 fnhe->fnhe_expires = max(1UL, expires);
 678                 /* Update all cached dsts too */
 679                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 680                 if (rt)
 681                         fill_route_from_fnhe(rt, fnhe);
 682                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 683                 if (rt)
 684                         fill_route_from_fnhe(rt, fnhe);
 685         } else {
 686                 if (depth > FNHE_RECLAIM_DEPTH)
 687                         fnhe = fnhe_oldest(hash);
 688                 else {
 689                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 690                         if (!fnhe)
 691                                 goto out_unlock;
 692
 693                         fnhe->fnhe_next = hash->chain;
 694                         rcu_assign_pointer(hash->chain, fnhe);
 695                 }
 696                 fnhe->fnhe_genid = genid;
 697                 fnhe->fnhe_daddr = daddr;
 698                 fnhe->fnhe_gw = gw;
 699                 fnhe->fnhe_pmtu = pmtu;
 700                 fnhe->fnhe_mtu_locked = lock;
 701                 fnhe->fnhe_expires = max(1UL, expires);
 702
 703                 /* Exception created; mark the cached routes for the nexthop
 704                  * stale, so anyone caching it rechecks if this exception
 705                  * applies to them.
 706                  */
 707                 rt = rcu_dereference(nh->nh_rth_input);
 708                 if (rt)
 709                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 710
 711                 for_each_possible_cpu(i) {
 712                         struct rtable __rcu **prt;
 713                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 714                         rt = rcu_dereference(*prt);
 715                         if (rt)
 716                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 717                 }
 718         }
 719
 720         fnhe->fnhe_stamp = jiffies;
 721
 722 out_unlock:
 723         spin_unlock_bh(&fnhe_lock);
 724 }
 725
 726 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 727                              bool kill_route)
 728 {
 729         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 730         __be32 old_gw = ip_hdr(skb)->saddr;
 731         struct net_device *dev = skb->dev;
 732         struct in_device *in_dev;
 733         struct fib_result res;
 734         struct neighbour *n;
 735         struct net *net;
 736
 737         switch (icmp_hdr(skb)->code & 7) {
 738         case ICMP_REDIR_NET:
 739         case ICMP_REDIR_NETTOS:
 740         case ICMP_REDIR_HOST:
 741         case ICMP_REDIR_HOSTTOS:
 742                 break;
 743
 744         default:
 745                 return;
 746         }
 747
 748         if (rt->rt_gateway != old_gw)
 749                 return;
 750
 751         in_dev = __in_dev_get_rcu(dev);
 752         if (!in_dev)
 753                 return;
 754
 755         net = dev_net(dev);
 756         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 757             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 758             ipv4_is_zeronet(new_gw))
 759                 goto reject_redirect;
 760
 761         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 762                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 763                         goto reject_redirect;
 764                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 765                         goto reject_redirect;
 766         } else {
 767                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 768                         goto reject_redirect;
 769         }
 770
 771         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 772         if (!n)
 773                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 774         if (!IS_ERR(n)) {
 775                 if (!(n->nud_state & NUD_VALID)) {
 776                         neigh_event_send(n, NULL);
 777                 } else {
 778                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 779                                 struct fib_nh *nh = &FIB_RES_NH(res);
 780
 781                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 782                                                 0, false,
 783                                                 jiffies + ip_rt_gc_timeout);
 784                         }
 785                         if (kill_route)
 786                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 787                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 788                 }
 789                 neigh_release(n);
 790         }
 791         return;
 792
 793 reject_redirect:
 794 #ifdef CONFIG_IP_ROUTE_VERBOSE
 795         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 796                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 797                 __be32 daddr = iph->daddr;
 798                 __be32 saddr = iph->saddr;
 799
 800                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 801                                      "  Advised path = %pI4 -> %pI4\n",
 802                                      &old_gw, dev->name, &new_gw,
 803                                      &saddr, &daddr);
 804         }
 805 #endif
 806         ;
 807 }
 808
 809 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 810 {
 811         struct rtable *rt;
 812         struct flowi4 fl4;
 813         const struct iphdr *iph = (const struct iphdr *) skb->data;
 814         struct net *net = dev_net(skb->dev);
 815         int oif = skb->dev->ifindex;
 816         u8 tos = RT_TOS(iph->tos);
 817         u8 prot = iph->protocol;
 818         u32 mark = skb->mark;
 819
 820         rt = (struct rtable *) dst;
 821
 822         __build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
 823         __ip_do_redirect(rt, skb, &fl4, true);
 824 }
 825
 826 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 827 {
 828         struct rtable *rt = (struct rtable *)dst;
 829         struct dst_entry *ret = dst;
 830
 831         if (rt) {
 832                 if (dst->obsolete > 0) {
 833                         ip_rt_put(rt);
 834                         ret = NULL;
 835                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 836                            rt->dst.expires) {
 837                         ip_rt_put(rt);
 838                         ret = NULL;
 839                 }
 840         }
 841         return ret;
 842 }
 843
 844 /*
 845  * Algorithm:
 846  *      1. The first ip_rt_redirect_number redirects are sent
 847  *         with exponential backoff, then we stop sending them at all,
 848  *         assuming that the host ignores our redirects.
 849  *      2. If we did not see packets requiring redirects
 850  *         during ip_rt_redirect_silence, we assume that the host
 851  *         forgot redirected route and start to send redirects again.
 852  *
 853  * This algorithm is much cheaper and more intelligent than dumb load limiting
 854  * in icmp.c.
 855  *
 856  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 857  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 858  */
 859
 860 void ip_rt_send_redirect(struct sk_buff *skb)
 861 {
 862         struct rtable *rt = skb_rtable(skb);
 863         struct in_device *in_dev;
 864         struct inet_peer *peer;
 865         struct net *net;
 866         int log_martians;
 867         int vif;
 868
 869         rcu_read_lock();
 870         in_dev = __in_dev_get_rcu(rt->dst.dev);
 871         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 872                 rcu_read_unlock();
 873                 return;
 874         }
 875         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 876         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 877         rcu_read_unlock();
 878
 879         net = dev_net(rt->dst.dev);
 880         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 881         if (!peer) {
 882                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 883                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 884                 return;
 885         }
 886
 887         /* No redirected packets during ip_rt_redirect_silence;
 888          * reset the algorithm.
 889          */
 890         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 891                 peer->rate_tokens = 0;
 892
 893         /* Too many ignored redirects; do not send anything
 894          * set dst.rate_last to the last seen redirected packet.
 895          */
 896         if (peer->rate_tokens >= ip_rt_redirect_number) {
 897                 peer->rate_last = jiffies;
 898                 goto out_put_peer;
 899         }
 900
 901         /* Check for load limit; set rate_last to the latest sent
 902          * redirect.
 903          */
 904         if (peer->rate_tokens == 0 ||
 905             time_after(jiffies,
 906                        (peer->rate_last +
 907                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 908                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 909
 910                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 911                 peer->rate_last = jiffies;
 912                 ++peer->rate_tokens;
 913 #ifdef CONFIG_IP_ROUTE_VERBOSE
 914                 if (log_martians &&
 915                     peer->rate_tokens == ip_rt_redirect_number)
 916                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 917                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 918                                              &ip_hdr(skb)->daddr, &gw);
 919 #endif
 920         }
 921 out_put_peer:
 922         inet_putpeer(peer);
 923 }
 924
 925 static int ip_error(struct sk_buff *skb)
 926 {
 927         struct rtable *rt = skb_rtable(skb);
 928         struct net_device *dev = skb->dev;
 929         struct in_device *in_dev;
 930         struct inet_peer *peer;
 931         unsigned long now;
 932         struct net *net;
 933         bool send;
 934         int code;
 935
 936         if (netif_is_l3_master(skb->dev)) {
 937                 dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
 938                 if (!dev)
 939                         goto out;
 940         }
 941
 942         in_dev = __in_dev_get_rcu(dev);
 943
 944         /* IP on this device is disabled. */
 945         if (!in_dev)
 946                 goto out;
 947
 948         net = dev_net(rt->dst.dev);
 949         if (!IN_DEV_FORWARD(in_dev)) {
 950                 switch (rt->dst.error) {
 951                 case EHOSTUNREACH:
 952                         __IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
 953                         break;
 954
 955                 case ENETUNREACH:
 956                         __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 957                         break;
 958                 }
 959                 goto out;
 960         }
 961
 962         switch (rt->dst.error) {
 963         case EINVAL:
 964         default:
 965                 goto out;
 966         case EHOSTUNREACH:
 967                 code = ICMP_HOST_UNREACH;
 968                 break;
 969         case ENETUNREACH:
 970                 code = ICMP_NET_UNREACH;
 971                 __IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
 972                 break;
 973         case EACCES:
 974                 code = ICMP_PKT_FILTERED;
 975                 break;
 976         }
 977
 978         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 979                                l3mdev_master_ifindex(skb->dev), 1);
 980
 981         send = true;
 982         if (peer) {
 983                 now = jiffies;
 984                 peer->rate_tokens += now - peer->rate_last;
 985                 if (peer->rate_tokens > ip_rt_error_burst)
 986                         peer->rate_tokens = ip_rt_error_burst;
 987                 peer->rate_last = now;
 988                 if (peer->rate_tokens >= ip_rt_error_cost)
 989                         peer->rate_tokens -= ip_rt_error_cost;
 990                 else
 991                         send = false;
 992                 inet_putpeer(peer);
 993         }
 994         if (send)
 995                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 996
 997 out:    kfree_skb(skb);
 998         return 0;
 999 }
1000
1001 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
1002 {
1003         struct dst_entry *dst = &rt->dst;
1004         struct fib_result res;
1005         bool lock = false;
1006
1007         if (ip_mtu_locked(dst))
1008                 return;
1009
1010         if (ipv4_mtu(dst) < mtu)
1011                 return;
1012
1013         if (mtu < ip_rt_min_pmtu) {
1014                 lock = true;
1015                 mtu = ip_rt_min_pmtu;
1016         }
1017
1018         if (rt->rt_pmtu == mtu &&
1019             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
1020                 return;
1021
1022         rcu_read_lock();
1023         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
1024                 struct fib_nh *nh = &FIB_RES_NH(res);
1025
1026                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu, lock,
1027                                       jiffies + ip_rt_mtu_expires);
1028         }
1029         rcu_read_unlock();
1030 }
1031
1032 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
1033                               struct sk_buff *skb, u32 mtu)
1034 {
1035         struct rtable *rt = (struct rtable *) dst;
1036         struct flowi4 fl4;
1037
1038         ip_rt_build_flow_key(&fl4, sk, skb);
1039         __ip_rt_update_pmtu(rt, &fl4, mtu);
1040 }
1041
1042 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1043                       int oif, u8 protocol)
1044 {
1045         const struct iphdr *iph = (const struct iphdr *) skb->data;
1046         struct flowi4 fl4;
1047         struct rtable *rt;
1048         u32 mark = IP4_REPLY_MARK(net, skb->mark);
1049
1050         __build_flow_key(net, &fl4, NULL, iph, oif,
1051                          RT_TOS(iph->tos), protocol, mark, 0);
1052         rt = __ip_route_output_key(net, &fl4);
1053         if (!IS_ERR(rt)) {
1054                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1055                 ip_rt_put(rt);
1056         }
1057 }
1058 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1059
1060 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1061 {
1062         const struct iphdr *iph = (const struct iphdr *) skb->data;
1063         struct flowi4 fl4;
1064         struct rtable *rt;
1065
1066         __build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
1067
1068         if (!fl4.flowi4_mark)
1069                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1070
1071         rt = __ip_route_output_key(sock_net(sk), &fl4);
1072         if (!IS_ERR(rt)) {
1073                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1074                 ip_rt_put(rt);
1075         }
1076 }
1077
1078 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1079 {
1080         const struct iphdr *iph = (const struct iphdr *) skb->data;
1081         struct flowi4 fl4;
1082         struct rtable *rt;
1083         struct dst_entry *odst = NULL;
1084         bool new = false;
1085         struct net *net = sock_net(sk);
1086
1087         bh_lock_sock(sk);
1088
1089         if (!ip_sk_accept_pmtu(sk))
1090                 goto out;
1091
1092         odst = sk_dst_get(sk);
1093
1094         if (sock_owned_by_user(sk) || !odst) {
1095                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1096                 goto out;
1097         }
1098
1099         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1100
1101         rt = (struct rtable *)odst;
1102         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1103                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1104                 if (IS_ERR(rt))
1105                         goto out;
1106
1107                 new = true;
1108         }
1109
1110         __ip_rt_update_pmtu((struct rtable *) xfrm_dst_path(&rt->dst), &fl4, mtu);
1111
1112         if (!dst_check(&rt->dst, 0)) {
1113                 if (new)
1114                         dst_release(&rt->dst);
1115
1116                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1117                 if (IS_ERR(rt))
1118                         goto out;
1119
1120                 new = true;
1121         }
1122
1123         if (new)
1124                 sk_dst_set(sk, &rt->dst);
1125
1126 out:
1127         bh_unlock_sock(sk);
1128         dst_release(odst);
1129 }
1130 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1131
1132 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1133                    int oif, u8 protocol)
1134 {
1135         const struct iphdr *iph = (const struct iphdr *) skb->data;
1136         struct flowi4 fl4;
1137         struct rtable *rt;
1138
1139         __build_flow_key(net, &fl4, NULL, iph, oif,
1140                          RT_TOS(iph->tos), protocol, 0, 0);
1141         rt = __ip_route_output_key(net, &fl4);
1142         if (!IS_ERR(rt)) {
1143                 __ip_do_redirect(rt, skb, &fl4, false);
1144                 ip_rt_put(rt);
1145         }
1146 }
1147 EXPORT_SYMBOL_GPL(ipv4_redirect);
1148
1149 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1150 {
1151         const struct iphdr *iph = (const struct iphdr *) skb->data;
1152         struct flowi4 fl4;
1153         struct rtable *rt;
1154         struct net *net = sock_net(sk);
1155
1156         __build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
1157         rt = __ip_route_output_key(net, &fl4);
1158         if (!IS_ERR(rt)) {
1159                 __ip_do_redirect(rt, skb, &fl4, false);
1160                 ip_rt_put(rt);
1161         }
1162 }
1163 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1164
1165 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1166 {
1167         struct rtable *rt = (struct rtable *) dst;
1168
1169         /* All IPV4 dsts are created with ->obsolete set to the value
1170          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1171          * into this function always.
1172          *
1173          * When a PMTU/redirect information update invalidates a route,
1174          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1175          * DST_OBSOLETE_DEAD by dst_free().
1176          */
1177         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1178                 return NULL;
1179         return dst;
1180 }
1181
1182 static void ipv4_link_failure(struct sk_buff *skb)
1183 {
1184         struct rtable *rt;
1185
1186         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1187
1188         rt = skb_rtable(skb);
1189         if (rt)
1190                 dst_set_expires(&rt->dst, 0);
1191 }
1192
1193 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1194 {
1195         pr_debug("%s: %pI4 -> %pI4, %s\n",
1196                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1197                  skb->dev ? skb->dev->name : "?");
1198         kfree_skb(skb);
1199         WARN_ON(1);
1200         return 0;
1201 }
1202
1203 /*
1204    We do not cache source address of outgoing interface,
1205    because it is used only by IP RR, TS and SRR options,
1206    so that it out of fast path.
1207
1208    BTW remember: "addr" is allowed to be not aligned
1209    in IP options!
1210  */
1211
1212 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1213 {
1214         __be32 src;
1215
1216         if (rt_is_output_route(rt))
1217                 src = ip_hdr(skb)->saddr;
1218         else {
1219                 struct fib_result res;
1220                 struct flowi4 fl4;
1221                 struct iphdr *iph;
1222
1223                 iph = ip_hdr(skb);
1224
1225                 memset(&fl4, 0, sizeof(fl4));
1226                 fl4.daddr = iph->daddr;
1227                 fl4.saddr = iph->saddr;
1228                 fl4.flowi4_tos = RT_TOS(iph->tos);
1229                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1230                 fl4.flowi4_iif = skb->dev->ifindex;
1231                 fl4.flowi4_mark = skb->mark;
1232
1233                 rcu_read_lock();
1234                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1235                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1236                 else
1237                         src = inet_select_addr(rt->dst.dev,
1238                                                rt_nexthop(rt, iph->daddr),
1239                                                RT_SCOPE_UNIVERSE);
1240                 rcu_read_unlock();
1241         }
1242         memcpy(addr, &src, 4);
1243 }
1244
1245 #ifdef CONFIG_IP_ROUTE_CLASSID
1246 static void set_class_tag(struct rtable *rt, u32 tag)
1247 {
1248         if (!(rt->dst.tclassid & 0xFFFF))
1249                 rt->dst.tclassid |= tag & 0xFFFF;
1250         if (!(rt->dst.tclassid & 0xFFFF0000))
1251                 rt->dst.tclassid |= tag & 0xFFFF0000;
1252 }
1253 #endif
1254
1255 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1256 {
1257         unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
1258         unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
1259                                     ip_rt_min_advmss);
1260
1261         return min(advmss, IPV4_MAX_PMTU - header_size);
1262 }
1263
1264 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1265 {
1266         const struct rtable *rt = (const struct rtable *) dst;
1267         unsigned int mtu = rt->rt_pmtu;
1268
1269         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1270                 mtu = dst_metric_raw(dst, RTAX_MTU);
1271
1272         if (mtu)
1273                 return mtu;
1274
1275         mtu = READ_ONCE(dst->dev->mtu);
1276
1277         if (unlikely(ip_mtu_locked(dst))) {
1278                 if (rt->rt_uses_gateway && mtu > 576)
1279                         mtu = 576;
1280         }
1281
1282         mtu = min_t(unsigned int, mtu, IP_MAX_MTU);
1283
1284         return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
1285 }
1286
1287 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1288 {
1289         struct fnhe_hash_bucket *hash;
1290         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1291         u32 hval = fnhe_hashfun(daddr);
1292
1293         spin_lock_bh(&fnhe_lock);
1294
1295         hash = rcu_dereference_protected(nh->nh_exceptions,
1296                                          lockdep_is_held(&fnhe_lock));
1297         hash += hval;
1298
1299         fnhe_p = &hash->chain;
1300         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1301         while (fnhe) {
1302                 if (fnhe->fnhe_daddr == daddr) {
1303                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1304                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1305                         fnhe_flush_routes(fnhe);
1306                         kfree_rcu(fnhe, rcu);
1307                         break;
1308                 }
1309                 fnhe_p = &fnhe->fnhe_next;
1310                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1311                                                  lockdep_is_held(&fnhe_lock));
1312         }
1313
1314         spin_unlock_bh(&fnhe_lock);
1315 }
1316
1317 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1318 {
1319         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1320         struct fib_nh_exception *fnhe;
1321         u32 hval;
1322
1323         if (!hash)
1324                 return NULL;
1325
1326         hval = fnhe_hashfun(daddr);
1327
1328         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1329              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1330                 if (fnhe->fnhe_daddr == daddr) {
1331                         if (fnhe->fnhe_expires &&
1332                             time_after(jiffies, fnhe->fnhe_expires)) {
1333                                 ip_del_fnhe(nh, daddr);
1334                                 break;
1335                         }
1336                         return fnhe;
1337                 }
1338         }
1339         return NULL;
1340 }
1341
1342 /* MTU selection:
1343  * 1. mtu on route is locked - use it
1344  * 2. mtu from nexthop exception
1345  * 3. mtu from egress device
1346  */
1347
1348 u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
1349 {
1350         struct fib_info *fi = res->fi;
1351         struct fib_nh *nh = &fi->fib_nh[res->nh_sel];
1352         struct net_device *dev = nh->nh_dev;
1353         u32 mtu = 0;
1354
1355         if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
1356             fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
1357                 mtu = fi->fib_mtu;
1358
1359         if (likely(!mtu)) {
1360                 struct fib_nh_exception *fnhe;
1361
1362                 fnhe = find_exception(nh, daddr);
1363                 if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
1364                         mtu = fnhe->fnhe_pmtu;
1365         }
1366
1367         if (likely(!mtu))
1368                 mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
1369
1370         return mtu - lwtunnel_headroom(nh->nh_lwtstate, mtu);
1371 }
1372
1373 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1374                               __be32 daddr, const bool do_cache)
1375 {
1376         bool ret = false;
1377
1378         spin_lock_bh(&fnhe_lock);
1379
1380         if (daddr == fnhe->fnhe_daddr) {
1381                 struct rtable __rcu **porig;
1382                 struct rtable *orig;
1383                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1384
1385                 if (rt_is_input_route(rt))
1386                         porig = &fnhe->fnhe_rth_input;
1387                 else
1388                         porig = &fnhe->fnhe_rth_output;
1389                 orig = rcu_dereference(*porig);
1390
1391                 if (fnhe->fnhe_genid != genid) {
1392                         fnhe->fnhe_genid = genid;
1393                         fnhe->fnhe_gw = 0;
1394                         fnhe->fnhe_pmtu = 0;
1395                         fnhe->fnhe_expires = 0;
1396                         fnhe->fnhe_mtu_locked = false;
1397                         fnhe_flush_routes(fnhe);
1398                         orig = NULL;
1399                 }
1400                 fill_route_from_fnhe(rt, fnhe);
1401                 if (!rt->rt_gateway)
1402                         rt->rt_gateway = daddr;
1403
1404                 if (do_cache) {
1405                         dst_hold(&rt->dst);
1406                         rcu_assign_pointer(*porig, rt);
1407                         if (orig) {
1408                                 dst_dev_put(&orig->dst);
1409                                 dst_release(&orig->dst);
1410                         }
1411                         ret = true;
1412                 }
1413
1414                 fnhe->fnhe_stamp = jiffies;
1415         }
1416         spin_unlock_bh(&fnhe_lock);
1417
1418         return ret;
1419 }
1420
1421 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1422 {
1423         struct rtable *orig, *prev, **p;
1424         bool ret = true;
1425
1426         if (rt_is_input_route(rt)) {
1427                 p = (struct rtable **)&nh->nh_rth_input;
1428         } else {
1429                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1430         }
1431         orig = *p;
1432
1433         /* hold dst before doing cmpxchg() to avoid race condition
1434          * on this dst
1435          */
1436         dst_hold(&rt->dst);
1437         prev = cmpxchg(p, orig, rt);
1438         if (prev == orig) {
1439                 if (orig) {
1440                         dst_dev_put(&orig->dst);
1441                         dst_release(&orig->dst);
1442                 }
1443         } else {
1444                 dst_release(&rt->dst);
1445                 ret = false;
1446         }
1447
1448         return ret;
1449 }
1450
1451 struct uncached_list {
1452         spinlock_t              lock;
1453         struct list_head        head;
1454 };
1455
1456 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1457
1458 void rt_add_uncached_list(struct rtable *rt)
1459 {
1460         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1461
1462         rt->rt_uncached_list = ul;
1463
1464         spin_lock_bh(&ul->lock);
1465         list_add_tail(&rt->rt_uncached, &ul->head);
1466         spin_unlock_bh(&ul->lock);
1467 }
1468
1469 void rt_del_uncached_list(struct rtable *rt)
1470 {
1471         if (!list_empty(&rt->rt_uncached)) {
1472                 struct uncached_list *ul = rt->rt_uncached_list;
1473
1474                 spin_lock_bh(&ul->lock);
1475                 list_del(&rt->rt_uncached);
1476                 spin_unlock_bh(&ul->lock);
1477         }
1478 }
1479
1480 static void ipv4_dst_destroy(struct dst_entry *dst)
1481 {
1482         struct dst_metrics *p = (struct dst_metrics *)DST_METRICS_PTR(dst);
1483         struct rtable *rt = (struct rtable *)dst;
1484
1485         if (p != &dst_default_metrics && refcount_dec_and_test(&p->refcnt))
1486                 kfree(p);
1487
1488         rt_del_uncached_list(rt);
1489 }
1490
1491 void rt_flush_dev(struct net_device *dev)
1492 {
1493         struct net *net = dev_net(dev);
1494         struct rtable *rt;
1495         int cpu;
1496
1497         for_each_possible_cpu(cpu) {
1498                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1499
1500                 spin_lock_bh(&ul->lock);
1501                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1502                         if (rt->dst.dev != dev)
1503                                 continue;
1504                         rt->dst.dev = net->loopback_dev;
1505                         dev_hold(rt->dst.dev);
1506                         dev_put(dev);
1507                 }
1508                 spin_unlock_bh(&ul->lock);
1509         }
1510 }
1511
1512 static bool rt_cache_valid(const struct rtable *rt)
1513 {
1514         return  rt &&
1515                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1516                 !rt_is_expired(rt);
1517 }
1518
1519 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1520                            const struct fib_result *res,
1521                            struct fib_nh_exception *fnhe,
1522                            struct fib_info *fi, u16 type, u32 itag,
1523                            const bool do_cache)
1524 {
1525         bool cached = false;
1526
1527         if (fi) {
1528                 struct fib_nh *nh = &FIB_RES_NH(*res);
1529
1530                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1531                         rt->rt_gateway = nh->nh_gw;
1532                         rt->rt_uses_gateway = 1;
1533                 }
1534                 dst_init_metrics(&rt->dst, fi->fib_metrics->metrics, true);
1535                 if (fi->fib_metrics != &dst_default_metrics) {
1536                         rt->dst._metrics |= DST_METRICS_REFCOUNTED;
1537                         refcount_inc(&fi->fib_metrics->refcnt);
1538                 }
1539 #ifdef CONFIG_IP_ROUTE_CLASSID
1540                 rt->dst.tclassid = nh->nh_tclassid;
1541 #endif
1542                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1543                 if (unlikely(fnhe))
1544                         cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
1545                 else if (do_cache)
1546                         cached = rt_cache_route(nh, rt);
1547                 if (unlikely(!cached)) {
1548                         /* Routes we intend to cache in nexthop exception or
1549                          * FIB nexthop have the DST_NOCACHE bit clear.
1550                          * However, if we are unsuccessful at storing this
1551                          * route into the cache we really need to set it.
1552                          */
1553                         if (!rt->rt_gateway)
1554                                 rt->rt_gateway = daddr;
1555                         rt_add_uncached_list(rt);
1556                 }
1557         } else
1558                 rt_add_uncached_list(rt);
1559
1560 #ifdef CONFIG_IP_ROUTE_CLASSID
1561 #ifdef CONFIG_IP_MULTIPLE_TABLES
1562         set_class_tag(rt, res->tclassid);
1563 #endif
1564         set_class_tag(rt, itag);
1565 #endif
1566 }
1567
1568 struct rtable *rt_dst_alloc(struct net_device *dev,
1569                             unsigned int flags, u16 type,
1570                             bool nopolicy, bool noxfrm, bool will_cache)
1571 {
1572         struct rtable *rt;
1573
1574         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1575                        (will_cache ? 0 : DST_HOST) |
1576                        (nopolicy ? DST_NOPOLICY : 0) |
1577                        (noxfrm ? DST_NOXFRM : 0));
1578
1579         if (rt) {
1580                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1581                 rt->rt_flags = flags;
1582                 rt->rt_type = type;
1583                 rt->rt_is_input = 0;
1584                 rt->rt_iif = 0;
1585                 rt->rt_pmtu = 0;
1586                 rt->rt_mtu_locked = 0;
1587                 rt->rt_gateway = 0;
1588                 rt->rt_uses_gateway = 0;
1589                 INIT_LIST_HEAD(&rt->rt_uncached);
1590
1591                 rt->dst.output = ip_output;
1592                 if (flags & RTCF_LOCAL)
1593                         rt->dst.input = ip_local_deliver;
1594         }
1595
1596         return rt;
1597 }
1598 EXPORT_SYMBOL(rt_dst_alloc);
1599
1600 /* called in rcu_read_lock() section */
1601 int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1602                           u8 tos, struct net_device *dev,
1603                           struct in_device *in_dev, u32 *itag)
1604 {
1605         int err;
1606
1607         /* Primary sanity checks. */
1608         if (!in_dev)
1609                 return -EINVAL;
1610
1611         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1612             skb->protocol != htons(ETH_P_IP))
1613                 return -EINVAL;
1614
1615         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1616                 return -EINVAL;
1617
1618         if (ipv4_is_zeronet(saddr)) {
1619                 if (!ipv4_is_local_multicast(daddr))
1620                         return -EINVAL;
1621         } else {
1622                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1623                                           in_dev, itag);
1624                 if (err < 0)
1625                         return err;
1626         }
1627         return 0;
1628 }
1629
1630 /* called in rcu_read_lock() section */
1631 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1632                              u8 tos, struct net_device *dev, int our)
1633 {
1634         struct in_device *in_dev = __in_dev_get_rcu(dev);
1635         unsigned int flags = RTCF_MULTICAST;
1636         struct rtable *rth;
1637         u32 itag = 0;
1638         int err;
1639
1640         err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
1641         if (err)
1642                 return err;
1643
1644         if (our)
1645                 flags |= RTCF_LOCAL;
1646
1647         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1648                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1649         if (!rth)
1650                 return -ENOBUFS;
1651
1652 #ifdef CONFIG_IP_ROUTE_CLASSID
1653         rth->dst.tclassid = itag;
1654 #endif
1655         rth->dst.output = ip_rt_bug;
1656         rth->rt_is_input= 1;
1657
1658 #ifdef CONFIG_IP_MROUTE
1659         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1660                 rth->dst.input = ip_mr_input;
1661 #endif
1662         RT_CACHE_STAT_INC(in_slow_mc);
1663
1664         skb_dst_set(skb, &rth->dst);
1665         return 0;
1666 }
1667
1668
1669 static void ip_handle_martian_source(struct net_device *dev,
1670                                      struct in_device *in_dev,
1671                                      struct sk_buff *skb,
1672                                      __be32 daddr,
1673                                      __be32 saddr)
1674 {
1675         RT_CACHE_STAT_INC(in_martian_src);
1676 #ifdef CONFIG_IP_ROUTE_VERBOSE
1677         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1678                 /*
1679                  *      RFC1812 recommendation, if source is martian,
1680                  *      the only hint is MAC header.
1681                  */
1682                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1683                         &daddr, &saddr, dev->name);
1684                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1685                         print_hex_dump(KERN_WARNING, "ll header: ",
1686                                        DUMP_PREFIX_OFFSET, 16, 1,
1687                                        skb_mac_header(skb),
1688                                        dev->hard_header_len, true);
1689                 }
1690         }
1691 #endif
1692 }
1693
1694 /* called in rcu_read_lock() section */
1695 static int __mkroute_input(struct sk_buff *skb,
1696                            const struct fib_result *res,
1697                            struct in_device *in_dev,
1698                            __be32 daddr, __be32 saddr, u32 tos)
1699 {
1700         struct fib_nh_exception *fnhe;
1701         struct rtable *rth;
1702         int err;
1703         struct in_device *out_dev;
1704         bool do_cache;
1705         u32 itag = 0;
1706
1707         /* get a working reference to the output device */
1708         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1709         if (!out_dev) {
1710                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1711                 return -EINVAL;
1712         }
1713
1714         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1715                                   in_dev->dev, in_dev, &itag);
1716         if (err < 0) {
1717                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1718                                          saddr);
1719
1720                 goto cleanup;
1721         }
1722
1723         do_cache = res->fi && !itag;
1724         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1725             skb->protocol == htons(ETH_P_IP) &&
1726             (IN_DEV_SHARED_MEDIA(out_dev) ||
1727              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1728                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1729
1730         if (skb->protocol != htons(ETH_P_IP)) {
1731                 /* Not IP (i.e. ARP). Do not create route, if it is
1732                  * invalid for proxy arp. DNAT routes are always valid.
1733                  *
1734                  * Proxy arp feature have been extended to allow, ARP
1735                  * replies back to the same interface, to support
1736                  * Private VLAN switch technologies. See arp.c.
1737                  */
1738                 if (out_dev == in_dev &&
1739                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1740                         err = -EINVAL;
1741                         goto cleanup;
1742                 }
1743         }
1744
1745         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1746         if (do_cache) {
1747                 if (fnhe)
1748                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1749                 else
1750                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1751                 if (rt_cache_valid(rth)) {
1752                         skb_dst_set_noref(skb, &rth->dst);
1753                         goto out;
1754                 }
1755         }
1756
1757         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1758                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1759                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1760         if (!rth) {
1761                 err = -ENOBUFS;
1762                 goto cleanup;
1763         }
1764
1765         rth->rt_is_input = 1;
1766         RT_CACHE_STAT_INC(in_slow_tot);
1767
1768         rth->dst.input = ip_forward;
1769
1770         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
1771                        do_cache);
1772         lwtunnel_set_redirect(&rth->dst);
1773         skb_dst_set(skb, &rth->dst);
1774 out:
1775         err = 0;
1776  cleanup:
1777         return err;
1778 }
1779
1780 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1781 /* To make ICMP packets follow the right flow, the multipath hash is
1782  * calculated from the inner IP addresses.
1783  */
1784 static void ip_multipath_l3_keys(const struct sk_buff *skb,
1785                                  struct flow_keys *hash_keys)
1786 {
1787         const struct iphdr *outer_iph = ip_hdr(skb);
1788         const struct iphdr *key_iph = outer_iph;
1789         const struct iphdr *inner_iph;
1790         const struct icmphdr *icmph;
1791         struct iphdr _inner_iph;
1792         struct icmphdr _icmph;
1793
1794         if (likely(outer_iph->protocol != IPPROTO_ICMP))
1795                 goto out;
1796
1797         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1798                 goto out;
1799
1800         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1801                                    &_icmph);
1802         if (!icmph)
1803                 goto out;
1804
1805         if (icmph->type != ICMP_DEST_UNREACH &&
1806             icmph->type != ICMP_REDIRECT &&
1807             icmph->type != ICMP_TIME_EXCEEDED &&
1808             icmph->type != ICMP_PARAMETERPROB)
1809                 goto out;
1810
1811         inner_iph = skb_header_pointer(skb,
1812                                        outer_iph->ihl * 4 + sizeof(_icmph),
1813                                        sizeof(_inner_iph), &_inner_iph);
1814         if (!inner_iph)
1815                 goto out;
1816
1817         key_iph = inner_iph;
1818 out:
1819         hash_keys->addrs.v4addrs.src = key_iph->saddr;
1820         hash_keys->addrs.v4addrs.dst = key_iph->daddr;
1821 }
1822
1823 /* if skb is set it will be used and fl4 can be NULL */
1824 int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
1825                        const struct sk_buff *skb, struct flow_keys *flkeys)
1826 {
1827         struct flow_keys hash_keys;
1828         u32 mhash;
1829
1830         switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
1831         case 0:
1832                 memset(&hash_keys, 0, sizeof(hash_keys));
1833                 hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1834                 if (skb) {
1835                         ip_multipath_l3_keys(skb, &hash_keys);
1836                 } else {
1837                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1838                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1839                 }
1840                 break;
1841         case 1:
1842                 /* skb is currently provided only when forwarding */
1843                 if (skb) {
1844                         unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
1845                         struct flow_keys keys;
1846
1847                         /* short-circuit if we already have L4 hash present */
1848                         if (skb->l4_hash)
1849                                 return skb_get_hash_raw(skb) >> 1;
1850
1851                         memset(&hash_keys, 0, sizeof(hash_keys));
1852
1853                         if (!flkeys) {
1854                                 skb_flow_dissect_flow_keys(skb, &keys, flag);
1855                                 flkeys = &keys;
1856                         }
1857
1858                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1859                         hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
1860                         hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
1861                         hash_keys.ports.src = flkeys->ports.src;
1862                         hash_keys.ports.dst = flkeys->ports.dst;
1863                         hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
1864                 } else {
1865                         memset(&hash_keys, 0, sizeof(hash_keys));
1866                         hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1867                         hash_keys.addrs.v4addrs.src = fl4->saddr;
1868                         hash_keys.addrs.v4addrs.dst = fl4->daddr;
1869                         hash_keys.ports.src = fl4->fl4_sport;
1870                         hash_keys.ports.dst = fl4->fl4_dport;
1871                         hash_keys.basic.ip_proto = fl4->flowi4_proto;
1872                 }
1873                 break;
1874         }
1875         mhash = flow_hash_from_keys(&hash_keys);
1876
1877         return mhash >> 1;
1878 }
1879 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1880
1881 static int ip_mkroute_input(struct sk_buff *skb,
1882                             struct fib_result *res,
1883                             struct in_device *in_dev,
1884                             __be32 daddr, __be32 saddr, u32 tos,
1885                             struct flow_keys *hkeys)
1886 {
1887 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1888         if (res->fi && res->fi->fib_nhs > 1) {
1889                 int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
1890
1891                 fib_select_multipath(res, h);
1892         }
1893 #endif
1894
1895         /* create a routing cache entry */
1896         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1897 }
1898
1899 /*
1900  *      NOTE. We drop all the packets that has local source
1901  *      addresses, because every properly looped back packet
1902  *      must have correct destination already attached by output routine.
1903  *
1904  *      Such approach solves two big problems:
1905  *      1. Not simplex devices are handled properly.
1906  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1907  *      called with rcu_read_lock()
1908  */
1909
1910 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1911                                u8 tos, struct net_device *dev,
1912                                struct fib_result *res)
1913 {
1914         struct in_device *in_dev = __in_dev_get_rcu(dev);
1915         struct flow_keys *flkeys = NULL, _flkeys;
1916         struct net    *net = dev_net(dev);
1917         struct ip_tunnel_info *tun_info;
1918         int             err = -EINVAL;
1919         unsigned int    flags = 0;
1920         u32             itag = 0;
1921         struct rtable   *rth;
1922         struct flowi4   fl4;
1923         bool do_cache;
1924
1925         /* IP on this device is disabled. */
1926
1927         if (!in_dev)
1928                 goto out;
1929
1930         /* Check for the most weird martians, which can be not detected
1931            by fib_lookup.
1932          */
1933
1934         tun_info = skb_tunnel_info(skb);
1935         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1936                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1937         else
1938                 fl4.flowi4_tun_key.tun_id = 0;
1939         skb_dst_drop(skb);
1940
1941         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1942                 goto martian_source;
1943
1944         res->fi = NULL;
1945         res->table = NULL;
1946         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1947                 goto brd_input;
1948
1949         /* Accept zero addresses only to limited broadcast;
1950          * I even do not know to fix it or not. Waiting for complains :-)
1951          */
1952         if (ipv4_is_zeronet(saddr))
1953                 goto martian_source;
1954
1955         if (ipv4_is_zeronet(daddr))
1956                 goto martian_destination;
1957
1958         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1959          * and call it once if daddr or/and saddr are loopback addresses
1960          */
1961         if (ipv4_is_loopback(daddr)) {
1962                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1963                         goto martian_destination;
1964         } else if (ipv4_is_loopback(saddr)) {
1965                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1966                         goto martian_source;
1967         }
1968
1969         /*
1970          *      Now we are ready to route packet.
1971          */
1972         fl4.flowi4_oif = 0;
1973         fl4.flowi4_iif = dev->ifindex;
1974         fl4.flowi4_mark = skb->mark;
1975         fl4.flowi4_tos = tos;
1976         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1977         fl4.flowi4_flags = 0;
1978         fl4.daddr = daddr;
1979         fl4.saddr = saddr;
1980         fl4.flowi4_uid = sock_net_uid(net, NULL);
1981
1982         if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
1983                 flkeys = &_flkeys;
1984         } else {
1985                 fl4.flowi4_proto = 0;
1986                 fl4.fl4_sport = 0;
1987                 fl4.fl4_dport = 0;
1988         }
1989
1990         err = fib_lookup(net, &fl4, res, 0);
1991         if (err != 0) {
1992                 if (!IN_DEV_FORWARD(in_dev))
1993                         err = -EHOSTUNREACH;
1994                 goto no_route;
1995         }
1996
1997         if (res->type == RTN_BROADCAST) {
1998                 if (IN_DEV_BFORWARD(in_dev))
1999                         goto make_route;
2000                 goto brd_input;
2001         }
2002
2003         if (res->type == RTN_LOCAL) {
2004                 err = fib_validate_source(skb, saddr, daddr, tos,
2005                                           0, dev, in_dev, &itag);
2006                 if (err < 0)
2007                         goto martian_source;
2008                 goto local_input;
2009         }
2010
2011         if (!IN_DEV_FORWARD(in_dev)) {
2012                 err = -EHOSTUNREACH;
2013                 goto no_route;
2014         }
2015         if (res->type != RTN_UNICAST)
2016                 goto martian_destination;
2017
2018 make_route:
2019         err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
2020 out:    return err;
2021
2022 brd_input:
2023         if (skb->protocol != htons(ETH_P_IP))
2024                 goto e_inval;
2025
2026         if (!ipv4_is_zeronet(saddr)) {
2027                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2028                                           in_dev, &itag);
2029                 if (err < 0)
2030                         goto martian_source;
2031         }
2032         flags |= RTCF_BROADCAST;
2033         res->type = RTN_BROADCAST;
2034         RT_CACHE_STAT_INC(in_brd);
2035
2036 local_input:
2037         do_cache = false;
2038         if (res->fi) {
2039                 if (!itag) {
2040                         rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
2041                         if (rt_cache_valid(rth)) {
2042                                 skb_dst_set_noref(skb, &rth->dst);
2043                                 err = 0;
2044                                 goto out;
2045                         }
2046                         do_cache = true;
2047                 }
2048         }
2049
2050         rth = rt_dst_alloc(l3mdev_master_dev_rcu(dev) ? : net->loopback_dev,
2051                            flags | RTCF_LOCAL, res->type,
2052                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
2053         if (!rth)
2054                 goto e_nobufs;
2055
2056         rth->dst.output= ip_rt_bug;
2057 #ifdef CONFIG_IP_ROUTE_CLASSID
2058         rth->dst.tclassid = itag;
2059 #endif
2060         rth->rt_is_input = 1;
2061
2062         RT_CACHE_STAT_INC(in_slow_tot);
2063         if (res->type == RTN_UNREACHABLE) {
2064                 rth->dst.input= ip_error;
2065                 rth->dst.error= -err;
2066                 rth->rt_flags   &= ~RTCF_LOCAL;
2067         }
2068
2069         if (do_cache) {
2070                 struct fib_nh *nh = &FIB_RES_NH(*res);
2071
2072                 rth->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
2073                 if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
2074                         WARN_ON(rth->dst.input == lwtunnel_input);
2075                         rth->dst.lwtstate->orig_input = rth->dst.input;
2076                         rth->dst.input = lwtunnel_input;
2077                 }
2078
2079                 if (unlikely(!rt_cache_route(nh, rth)))
2080                         rt_add_uncached_list(rth);
2081         }
2082         skb_dst_set(skb, &rth->dst);
2083         err = 0;
2084         goto out;
2085
2086 no_route:
2087         RT_CACHE_STAT_INC(in_no_route);
2088         res->type = RTN_UNREACHABLE;
2089         res->fi = NULL;
2090         res->table = NULL;
2091         goto local_input;
2092
2093         /*
2094          *      Do not cache martian addresses: they should be logged (RFC1812)
2095          */
2096 martian_destination:
2097         RT_CACHE_STAT_INC(in_martian_dst);
2098 #ifdef CONFIG_IP_ROUTE_VERBOSE
2099         if (IN_DEV_LOG_MARTIANS(in_dev))
2100                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2101                                      &daddr, &saddr, dev->name);
2102 #endif
2103
2104 e_inval:
2105         err = -EINVAL;
2106         goto out;
2107
2108 e_nobufs:
2109         err = -ENOBUFS;
2110         goto out;
2111
2112 martian_source:
2113         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2114         goto out;
2115 }
2116
2117 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2118                          u8 tos, struct net_device *dev)
2119 {
2120         struct fib_result res;
2121         int err;
2122
2123         tos &= IPTOS_RT_MASK;
2124         rcu_read_lock();
2125         err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
2126         rcu_read_unlock();
2127
2128         return err;
2129 }
2130 EXPORT_SYMBOL(ip_route_input_noref);
2131
2132 /* called with rcu_read_lock held */
2133 int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2134                        u8 tos, struct net_device *dev, struct fib_result *res)
2135 {
2136         /* Multicast recognition logic is moved from route cache to here.
2137            The problem was that too many Ethernet cards have broken/missing
2138            hardware multicast filters :-( As result the host on multicasting
2139            network acquires a lot of useless route cache entries, sort of
2140            SDR messages from all the world. Now we try to get rid of them.
2141            Really, provided software IP multicast filter is organized
2142            reasonably (at least, hashed), it does not result in a slowdown
2143            comparing with route cache reject entries.
2144            Note, that multicast routers are not affected, because
2145            route cache entry is created eventually.
2146          */
2147         if (ipv4_is_multicast(daddr)) {
2148                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2149                 int our = 0;
2150                 int err = -EINVAL;
2151
2152                 if (in_dev)
2153                         our = ip_check_mc_rcu(in_dev, daddr, saddr,
2154                                               ip_hdr(skb)->protocol);
2155
2156                 /* check l3 master if no match yet */
2157                 if ((!in_dev || !our) && netif_is_l3_slave(dev)) {
2158                         struct in_device *l3_in_dev;
2159
2160                         l3_in_dev = __in_dev_get_rcu(skb->dev);
2161                         if (l3_in_dev)
2162                                 our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
2163                                                       ip_hdr(skb)->protocol);
2164                 }
2165
2166                 if (our
2167 #ifdef CONFIG_IP_MROUTE
2168                         ||
2169                     (!ipv4_is_local_multicast(daddr) &&
2170                      IN_DEV_MFORWARD(in_dev))
2171 #endif
2172                    ) {
2173                         err = ip_route_input_mc(skb, daddr, saddr,
2174                                                 tos, dev, our);
2175                 }
2176                 return err;
2177         }
2178
2179         return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
2180 }
2181
2182 /* called with rcu_read_lock() */
2183 static struct rtable *__mkroute_output(const struct fib_result *res,
2184                                        const struct flowi4 *fl4, int orig_oif,
2185                                        struct net_device *dev_out,
2186                                        unsigned int flags)
2187 {
2188         struct fib_info *fi = res->fi;
2189         struct fib_nh_exception *fnhe;
2190         struct in_device *in_dev;
2191         u16 type = res->type;
2192         struct rtable *rth;
2193         bool do_cache;
2194
2195         in_dev = __in_dev_get_rcu(dev_out);
2196         if (!in_dev)
2197                 return ERR_PTR(-EINVAL);
2198
2199         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2200                 if (ipv4_is_loopback(fl4->saddr) &&
2201                     !(dev_out->flags & IFF_LOOPBACK) &&
2202                     !netif_is_l3_master(dev_out))
2203                         return ERR_PTR(-EINVAL);
2204
2205         if (ipv4_is_lbcast(fl4->daddr))
2206                 type = RTN_BROADCAST;
2207         else if (ipv4_is_multicast(fl4->daddr))
2208                 type = RTN_MULTICAST;
2209         else if (ipv4_is_zeronet(fl4->daddr))
2210                 return ERR_PTR(-EINVAL);
2211
2212         if (dev_out->flags & IFF_LOOPBACK)
2213                 flags |= RTCF_LOCAL;
2214
2215         do_cache = true;
2216         if (type == RTN_BROADCAST) {
2217                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2218                 fi = NULL;
2219         } else if (type == RTN_MULTICAST) {
2220                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2221                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2222                                      fl4->flowi4_proto))
2223                         flags &= ~RTCF_LOCAL;
2224                 else
2225                         do_cache = false;
2226                 /* If multicast route do not exist use
2227                  * default one, but do not gateway in this case.
2228                  * Yes, it is hack.
2229                  */
2230                 if (fi && res->prefixlen < 4)
2231                         fi = NULL;
2232         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2233                    (orig_oif != dev_out->ifindex)) {
2234                 /* For local routes that require a particular output interface
2235                  * we do not want to cache the result.  Caching the result
2236                  * causes incorrect behaviour when there are multiple source
2237                  * addresses on the interface, the end result being that if the
2238                  * intended recipient is waiting on that interface for the
2239                  * packet he won't receive it because it will be delivered on
2240                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2241                  * be set to the loopback interface as well.
2242                  */
2243                 do_cache = false;
2244         }
2245
2246         fnhe = NULL;
2247         do_cache &= fi != NULL;
2248         if (fi) {
2249                 struct rtable __rcu **prth;
2250                 struct fib_nh *nh = &FIB_RES_NH(*res);
2251
2252                 fnhe = find_exception(nh, fl4->daddr);
2253                 if (!do_cache)
2254                         goto add;
2255                 if (fnhe) {
2256                         prth = &fnhe->fnhe_rth_output;
2257                 } else {
2258                         if (unlikely(fl4->flowi4_flags &
2259                                      FLOWI_FLAG_KNOWN_NH &&
2260                                      !(nh->nh_gw &&
2261                                        nh->nh_scope == RT_SCOPE_LINK))) {
2262                                 do_cache = false;
2263                                 goto add;
2264                         }
2265                         prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2266                 }
2267                 rth = rcu_dereference(*prth);
2268                 if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
2269                         return rth;
2270         }
2271
2272 add:
2273         rth = rt_dst_alloc(dev_out, flags, type,
2274                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2275                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2276                            do_cache);
2277         if (!rth)
2278                 return ERR_PTR(-ENOBUFS);
2279
2280         rth->rt_iif = orig_oif;
2281
2282         RT_CACHE_STAT_INC(out_slow_tot);
2283
2284         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2285                 if (flags & RTCF_LOCAL &&
2286                     !(dev_out->flags & IFF_LOOPBACK)) {
2287                         rth->dst.output = ip_mc_output;
2288                         RT_CACHE_STAT_INC(out_slow_mc);
2289                 }
2290 #ifdef CONFIG_IP_MROUTE
2291                 if (type == RTN_MULTICAST) {
2292                         if (IN_DEV_MFORWARD(in_dev) &&
2293                             !ipv4_is_local_multicast(fl4->daddr)) {
2294                                 rth->dst.input = ip_mr_input;
2295                                 rth->dst.output = ip_mc_output;
2296                         }
2297                 }
2298 #endif
2299         }
2300
2301         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
2302         lwtunnel_set_redirect(&rth->dst);
2303
2304         return rth;
2305 }
2306
2307 /*
2308  * Major route resolver routine.
2309  */
2310
2311 struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2312                                         const struct sk_buff *skb)
2313 {
2314         __u8 tos = RT_FL_TOS(fl4);
2315         struct fib_result res = {
2316                 .type           = RTN_UNSPEC,
2317                 .fi             = NULL,
2318                 .table          = NULL,
2319                 .tclassid       = 0,
2320         };
2321         struct rtable *rth;
2322
2323         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2324         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2325         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2326                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2327
2328         rcu_read_lock();
2329         rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
2330         rcu_read_unlock();
2331
2332         return rth;
2333 }
2334 EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
2335
2336 struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
2337                                             struct fib_result *res,
2338                                             const struct sk_buff *skb)
2339 {
2340         struct net_device *dev_out = NULL;
2341         int orig_oif = fl4->flowi4_oif;
2342         unsigned int flags = 0;
2343         struct rtable *rth;
2344         int err = -ENETUNREACH;
2345
2346         if (fl4->saddr) {
2347                 rth = ERR_PTR(-EINVAL);
2348                 if (ipv4_is_multicast(fl4->saddr) ||
2349                     ipv4_is_lbcast(fl4->saddr) ||
2350                     ipv4_is_zeronet(fl4->saddr))
2351                         goto out;
2352
2353                 /* I removed check for oif == dev_out->oif here.
2354                    It was wrong for two reasons:
2355                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2356                       is assigned to multiple interfaces.
2357                    2. Moreover, we are allowed to send packets with saddr
2358                       of another iface. --ANK
2359                  */
2360
2361                 if (fl4->flowi4_oif == 0 &&
2362                     (ipv4_is_multicast(fl4->daddr) ||
2363                      ipv4_is_lbcast(fl4->daddr))) {
2364                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2365                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2366                         if (!dev_out)
2367                                 goto out;
2368
2369                         /* Special hack: user can direct multicasts
2370                            and limited broadcast via necessary interface
2371                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2372                            This hack is not just for fun, it allows
2373                            vic,vat and friends to work.
2374                            They bind socket to loopback, set ttl to zero
2375                            and expect that it will work.
2376                            From the viewpoint of routing cache they are broken,
2377                            because we are not allowed to build multicast path
2378                            with loopback source addr (look, routing cache
2379                            cannot know, that ttl is zero, so that packet
2380                            will not leave this host and route is valid).
2381                            Luckily, this hack is good workaround.
2382                          */
2383
2384                         fl4->flowi4_oif = dev_out->ifindex;
2385                         goto make_route;
2386                 }
2387
2388                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2389                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2390                         if (!__ip_dev_find(net, fl4->saddr, false))
2391                                 goto out;
2392                 }
2393         }
2394
2395
2396         if (fl4->flowi4_oif) {
2397                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2398                 rth = ERR_PTR(-ENODEV);
2399                 if (!dev_out)
2400                         goto out;
2401
2402                 /* RACE: Check return value of inet_select_addr instead. */
2403                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2404                         rth = ERR_PTR(-ENETUNREACH);
2405                         goto out;
2406                 }
2407                 if (ipv4_is_local_multicast(fl4->daddr) ||
2408                     ipv4_is_lbcast(fl4->daddr) ||
2409                     fl4->flowi4_proto == IPPROTO_IGMP) {
2410                         if (!fl4->saddr)
2411                                 fl4->saddr = inet_select_addr(dev_out, 0,
2412                                                               RT_SCOPE_LINK);
2413                         goto make_route;
2414                 }
2415                 if (!fl4->saddr) {
2416                         if (ipv4_is_multicast(fl4->daddr))
2417                                 fl4->saddr = inet_select_addr(dev_out, 0,
2418                                                               fl4->flowi4_scope);
2419                         else if (!fl4->daddr)
2420                                 fl4->saddr = inet_select_addr(dev_out, 0,
2421                                                               RT_SCOPE_HOST);
2422                 }
2423         }
2424
2425         if (!fl4->daddr) {
2426                 fl4->daddr = fl4->saddr;
2427                 if (!fl4->daddr)
2428                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2429                 dev_out = net->loopback_dev;
2430                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2431                 res->type = RTN_LOCAL;
2432                 flags |= RTCF_LOCAL;
2433                 goto make_route;
2434         }
2435
2436         err = fib_lookup(net, fl4, res, 0);
2437         if (err) {
2438                 res->fi = NULL;
2439                 res->table = NULL;
2440                 if (fl4->flowi4_oif &&
2441                     (ipv4_is_multicast(fl4->daddr) ||
2442                     !netif_index_is_l3_master(net, fl4->flowi4_oif))) {
2443                         /* Apparently, routing tables are wrong. Assume,
2444                            that the destination is on link.
2445
2446                            WHY? DW.
2447                            Because we are allowed to send to iface
2448                            even if it has NO routes and NO assigned
2449                            addresses. When oif is specified, routing
2450                            tables are looked up with only one purpose:
2451                            to catch if destination is gatewayed, rather than
2452                            direct. Moreover, if MSG_DONTROUTE is set,
2453                            we send packet, ignoring both routing tables
2454                            and ifaddr state. --ANK
2455
2456
2457                            We could make it even if oif is unknown,
2458                            likely IPv6, but we do not.
2459                          */
2460
2461                         if (fl4->saddr == 0)
2462                                 fl4->saddr = inet_select_addr(dev_out, 0,
2463                                                               RT_SCOPE_LINK);
2464                         res->type = RTN_UNICAST;
2465                         goto make_route;
2466                 }
2467                 rth = ERR_PTR(err);
2468                 goto out;
2469         }
2470
2471         if (res->type == RTN_LOCAL) {
2472                 if (!fl4->saddr) {
2473                         if (res->fi->fib_prefsrc)
2474                                 fl4->saddr = res->fi->fib_prefsrc;
2475                         else
2476                                 fl4->saddr = fl4->daddr;
2477                 }
2478
2479                 /* L3 master device is the loopback for that domain */
2480                 dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
2481                         net->loopback_dev;
2482
2483                 /* make sure orig_oif points to fib result device even
2484                  * though packet rx/tx happens over loopback or l3mdev
2485                  */
2486                 orig_oif = FIB_RES_OIF(*res);
2487
2488                 fl4->flowi4_oif = dev_out->ifindex;
2489                 flags |= RTCF_LOCAL;
2490                 goto make_route;
2491         }
2492
2493         fib_select_path(net, res, fl4, skb);
2494
2495         dev_out = FIB_RES_DEV(*res);
2496         fl4->flowi4_oif = dev_out->ifindex;
2497
2498
2499 make_route:
2500         rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
2501
2502 out:
2503         return rth;
2504 }
2505
2506 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2507 {
2508         return NULL;
2509 }
2510
2511 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2512 {
2513         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2514
2515         return mtu ? : dst->dev->mtu;
2516 }
2517
2518 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2519                                           struct sk_buff *skb, u32 mtu)
2520 {
2521 }
2522
2523 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2524                                        struct sk_buff *skb)
2525 {
2526 }
2527
2528 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2529                                           unsigned long old)
2530 {
2531         return NULL;
2532 }
2533
2534 static struct dst_ops ipv4_dst_blackhole_ops = {
2535         .family                 =       AF_INET,
2536         .check                  =       ipv4_blackhole_dst_check,
2537         .mtu                    =       ipv4_blackhole_mtu,
2538         .default_advmss         =       ipv4_default_advmss,
2539         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2540         .redirect               =       ipv4_rt_blackhole_redirect,
2541         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2542         .neigh_lookup           =       ipv4_neigh_lookup,
2543 };
2544
2545 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2546 {
2547         struct rtable *ort = (struct rtable *) dst_orig;
2548         struct rtable *rt;
2549
2550         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
2551         if (rt) {
2552                 struct dst_entry *new = &rt->dst;
2553
2554                 new->__use = 1;
2555                 new->input = dst_discard;
2556                 new->output = dst_discard_out;
2557
2558                 new->dev = net->loopback_dev;
2559                 if (new->dev)
2560                         dev_hold(new->dev);
2561
2562                 rt->rt_is_input = ort->rt_is_input;
2563                 rt->rt_iif = ort->rt_iif;
2564                 rt->rt_pmtu = ort->rt_pmtu;
2565                 rt->rt_mtu_locked = ort->rt_mtu_locked;
2566
2567                 rt->rt_genid = rt_genid_ipv4(net);
2568                 rt->rt_flags = ort->rt_flags;
2569                 rt->rt_type = ort->rt_type;
2570                 rt->rt_gateway = ort->rt_gateway;
2571                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2572
2573                 INIT_LIST_HEAD(&rt->rt_uncached);
2574         }
2575
2576         dst_release(dst_orig);
2577
2578         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2579 }
2580
2581 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2582                                     const struct sock *sk)
2583 {
2584         struct rtable *rt = __ip_route_output_key(net, flp4);
2585
2586         if (IS_ERR(rt))
2587                 return rt;
2588
2589         if (flp4->flowi4_proto)
2590                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2591                                                         flowi4_to_flowi(flp4),
2592                                                         sk, 0);
2593
2594         return rt;
2595 }
2596 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2597
2598 /* called with rcu_read_lock held */
2599 static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
2600                         struct rtable *rt, u32 table_id, struct flowi4 *fl4,
2601                         struct sk_buff *skb, u32 portid, u32 seq)
2602 {
2603         struct rtmsg *r;
2604         struct nlmsghdr *nlh;
2605         unsigned long expires = 0;
2606         u32 error;
2607         u32 metrics[RTAX_MAX];
2608
2609         nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), 0);
2610         if (!nlh)
2611                 return -EMSGSIZE;
2612
2613         r = nlmsg_data(nlh);
2614         r->rtm_family    = AF_INET;
2615         r->rtm_dst_len  = 32;
2616         r->rtm_src_len  = 0;
2617         r->rtm_tos      = fl4->flowi4_tos;
2618         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2619         if (nla_put_u32(skb, RTA_TABLE, table_id))
2620                 goto nla_put_failure;
2621         r->rtm_type     = rt->rt_type;
2622         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2623         r->rtm_protocol = RTPROT_UNSPEC;
2624         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2625         if (rt->rt_flags & RTCF_NOTIFY)
2626                 r->rtm_flags |= RTM_F_NOTIFY;
2627         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2628                 r->rtm_flags |= RTCF_DOREDIRECT;
2629
2630         if (nla_put_in_addr(skb, RTA_DST, dst))
2631                 goto nla_put_failure;
2632         if (src) {
2633                 r->rtm_src_len = 32;
2634                 if (nla_put_in_addr(skb, RTA_SRC, src))
2635                         goto nla_put_failure;
2636         }
2637         if (rt->dst.dev &&
2638             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2639                 goto nla_put_failure;
2640 #ifdef CONFIG_IP_ROUTE_CLASSID
2641         if (rt->dst.tclassid &&
2642             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2643                 goto nla_put_failure;
2644 #endif
2645         if (!rt_is_input_route(rt) &&
2646             fl4->saddr != src) {
2647                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2648                         goto nla_put_failure;
2649         }
2650         if (rt->rt_uses_gateway &&
2651             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2652                 goto nla_put_failure;
2653
2654         expires = rt->dst.expires;
2655         if (expires) {
2656                 unsigned long now = jiffies;
2657
2658                 if (time_before(now, expires))
2659                         expires -= now;
2660                 else
2661                         expires = 0;
2662         }
2663
2664         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2665         if (rt->rt_pmtu && expires)
2666                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2667         if (rt->rt_mtu_locked && expires)
2668                 metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
2669         if (rtnetlink_put_metrics(skb, metrics) < 0)
2670                 goto nla_put_failure;
2671
2672         if (fl4->flowi4_mark &&
2673             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2674                 goto nla_put_failure;
2675
2676         if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
2677             nla_put_u32(skb, RTA_UID,
2678                         from_kuid_munged(current_user_ns(), fl4->flowi4_uid)))
2679                 goto nla_put_failure;
2680
2681         error = rt->dst.error;
2682
2683         if (rt_is_input_route(rt)) {
2684 #ifdef CONFIG_IP_MROUTE
2685                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2686                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2687                         int err = ipmr_get_route(net, skb,
2688                                                  fl4->saddr, fl4->daddr,
2689                                                  r, portid);
2690
2691                         if (err <= 0) {
2692                                 if (err == 0)
2693                                         return 0;
2694                                 goto nla_put_failure;
2695                         }
2696                 } else
2697 #endif
2698                         if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
2699                                 goto nla_put_failure;
2700         }
2701
2702         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2703                 goto nla_put_failure;
2704
2705         nlmsg_end(skb, nlh);
2706         return 0;
2707
2708 nla_put_failure:
2709         nlmsg_cancel(skb, nlh);
2710         return -EMSGSIZE;
2711 }
2712
2713 static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
2714                                                    u8 ip_proto, __be16 sport,
2715                                                    __be16 dport)
2716 {
2717         struct sk_buff *skb;
2718         struct iphdr *iph;
2719
2720         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2721         if (!skb)
2722                 return NULL;
2723
2724         /* Reserve room for dummy headers, this skb can pass
2725          * through good chunk of routing engine.
2726          */
2727         skb_reset_mac_header(skb);
2728         skb_reset_network_header(skb);
2729         skb->protocol = htons(ETH_P_IP);
2730         iph = skb_put(skb, sizeof(struct iphdr));
2731         iph->protocol = ip_proto;
2732         iph->saddr = src;
2733         iph->daddr = dst;
2734         iph->version = 0x4;
2735         iph->frag_off = 0;
2736         iph->ihl = 0x5;
2737         skb_set_transport_header(skb, skb->len);
2738
2739         switch (iph->protocol) {
2740         case IPPROTO_UDP: {
2741                 struct udphdr *udph;
2742
2743                 udph = skb_put_zero(skb, sizeof(struct udphdr));
2744                 udph->source = sport;
2745                 udph->dest = dport;
2746                 udph->len = sizeof(struct udphdr);
2747                 udph->check = 0;
2748                 break;
2749         }
2750         case IPPROTO_TCP: {
2751                 struct tcphdr *tcph;
2752
2753                 tcph = skb_put_zero(skb, sizeof(struct tcphdr));
2754                 tcph->source    = sport;
2755                 tcph->dest      = dport;
2756                 tcph->doff      = sizeof(struct tcphdr) / 4;
2757                 tcph->rst = 1;
2758                 tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
2759                                             src, dst, 0);
2760                 break;
2761         }
2762         case IPPROTO_ICMP: {
2763                 struct icmphdr *icmph;
2764
2765                 icmph = skb_put_zero(skb, sizeof(struct icmphdr));
2766                 icmph->type = ICMP_ECHO;
2767                 icmph->code = 0;
2768         }
2769         }
2770
2771         return skb;
2772 }
2773
2774 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
2775                              struct netlink_ext_ack *extack)
2776 {
2777         struct net *net = sock_net(in_skb->sk);
2778         struct nlattr *tb[RTA_MAX+1];
2779         u32 table_id = RT_TABLE_MAIN;
2780         __be16 sport = 0, dport = 0;
2781         struct fib_result res = {};
2782         u8 ip_proto = IPPROTO_UDP;
2783         struct rtable *rt = NULL;
2784         struct sk_buff *skb;
2785         struct rtmsg *rtm;
2786         struct flowi4 fl4;
2787         __be32 dst = 0;
2788         __be32 src = 0;
2789         kuid_t uid;
2790         u32 iif;
2791         int err;
2792         int mark;
2793
2794         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy,
2795                           extack);
2796         if (err < 0)
2797                 return err;
2798
2799         rtm = nlmsg_data(nlh);
2800         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2801         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2802         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2803         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2804         if (tb[RTA_UID])
2805                 uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
2806         else
2807                 uid = (iif ? INVALID_UID : current_uid());
2808
2809         if (tb[RTA_IP_PROTO]) {
2810                 err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
2811                                                   &ip_proto, extack);
2812                 if (err)
2813                         return err;
2814         }
2815
2816         if (tb[RTA_SPORT])
2817                 sport = nla_get_be16(tb[RTA_SPORT]);
2818
2819         if (tb[RTA_DPORT])
2820                 dport = nla_get_be16(tb[RTA_DPORT]);
2821
2822         skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
2823         if (!skb)
2824                 return -ENOBUFS;
2825
2826         memset(&fl4, 0, sizeof(fl4));
2827         fl4.daddr = dst;
2828         fl4.saddr = src;
2829         fl4.flowi4_tos = rtm->rtm_tos;
2830         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2831         fl4.flowi4_mark = mark;
2832         fl4.flowi4_uid = uid;
2833         if (sport)
2834                 fl4.fl4_sport = sport;
2835         if (dport)
2836                 fl4.fl4_dport = dport;
2837         fl4.flowi4_proto = ip_proto;
2838
2839         rcu_read_lock();
2840
2841         if (iif) {
2842                 struct net_device *dev;
2843
2844                 dev = dev_get_by_index_rcu(net, iif);
2845                 if (!dev) {
2846                         err = -ENODEV;
2847                         goto errout_rcu;
2848                 }
2849
2850                 fl4.flowi4_iif = iif; /* for rt_fill_info */
2851                 skb->dev        = dev;
2852                 skb->mark       = mark;
2853                 err = ip_route_input_rcu(skb, dst, src, rtm->rtm_tos,
2854                                          dev, &res);
2855
2856                 rt = skb_rtable(skb);
2857                 if (err == 0 && rt->dst.error)
2858                         err = -rt->dst.error;
2859         } else {
2860                 fl4.flowi4_iif = LOOPBACK_IFINDEX;
2861                 rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
2862                 err = 0;
2863                 if (IS_ERR(rt))
2864                         err = PTR_ERR(rt);
2865                 else
2866                         skb_dst_set(skb, &rt->dst);
2867         }
2868
2869         if (err)
2870                 goto errout_rcu;
2871
2872         if (rtm->rtm_flags & RTM_F_NOTIFY)
2873                 rt->rt_flags |= RTCF_NOTIFY;
2874
2875         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2876                 table_id = res.table ? res.table->tb_id : 0;
2877
2878         /* reset skb for netlink reply msg */
2879         skb_trim(skb, 0);
2880         skb_reset_network_header(skb);
2881         skb_reset_transport_header(skb);
2882         skb_reset_mac_header(skb);
2883
2884         if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
2885                 if (!res.fi) {
2886                         err = fib_props[res.type].error;
2887                         if (!err)
2888                                 err = -EHOSTUNREACH;
2889                         goto errout_rcu;
2890                 }
2891                 err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
2892                                     nlh->nlmsg_seq, RTM_NEWROUTE, table_id,
2893                                     rt->rt_type, res.prefix, res.prefixlen,
2894                                     fl4.flowi4_tos, res.fi, 0);
2895         } else {
2896                 err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
2897                                    NETLINK_CB(in_skb).portid, nlh->nlmsg_seq);
2898         }
2899         if (err < 0)
2900                 goto errout_rcu;
2901
2902         rcu_read_unlock();
2903
2904         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2905
2906 errout_free:
2907         return err;
2908 errout_rcu:
2909         rcu_read_unlock();
2910         kfree_skb(skb);
2911         goto errout_free;
2912 }
2913
2914 void ip_rt_multicast_event(struct in_device *in_dev)
2915 {
2916         rt_cache_flush(dev_net(in_dev->dev));
2917 }
2918
2919 #ifdef CONFIG_SYSCTL
2920 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2921 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2922 static int ip_rt_gc_elasticity __read_mostly    = 8;
2923 static int ip_min_valid_pmtu __read_mostly      = IPV4_MIN_MTU;
2924
2925 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2926                                         void __user *buffer,
2927                                         size_t *lenp, loff_t *ppos)
2928 {
2929         struct net *net = (struct net *)__ctl->extra1;
2930
2931         if (write) {
2932                 rt_cache_flush(net);
2933                 fnhe_genid_bump(net);
2934                 return 0;
2935         }
2936
2937         return -EINVAL;
2938 }
2939
2940 static struct ctl_table ipv4_route_table[] = {
2941         {
2942                 .procname       = "gc_thresh",
2943                 .data           = &ipv4_dst_ops.gc_thresh,
2944                 .maxlen         = sizeof(int),
2945                 .mode           = 0644,
2946                 .proc_handler   = proc_dointvec,
2947         },
2948         {
2949                 .procname       = "max_size",
2950                 .data           = &ip_rt_max_size,
2951                 .maxlen         = sizeof(int),
2952                 .mode           = 0644,
2953                 .proc_handler   = proc_dointvec,
2954         },
2955         {
2956                 /*  Deprecated. Use gc_min_interval_ms */
2957
2958                 .procname       = "gc_min_interval",
2959                 .data           = &ip_rt_gc_min_interval,
2960                 .maxlen         = sizeof(int),
2961                 .mode           = 0644,
2962                 .proc_handler   = proc_dointvec_jiffies,
2963         },
2964         {
2965                 .procname       = "gc_min_interval_ms",
2966                 .data           = &ip_rt_gc_min_interval,
2967                 .maxlen         = sizeof(int),
2968                 .mode           = 0644,
2969                 .proc_handler   = proc_dointvec_ms_jiffies,
2970         },
2971         {
2972                 .procname       = "gc_timeout",
2973                 .data           = &ip_rt_gc_timeout,
2974                 .maxlen         = sizeof(int),
2975                 .mode           = 0644,
2976                 .proc_handler   = proc_dointvec_jiffies,
2977         },
2978         {
2979                 .procname       = "gc_interval",
2980                 .data           = &ip_rt_gc_interval,
2981                 .maxlen         = sizeof(int),
2982                 .mode           = 0644,
2983                 .proc_handler   = proc_dointvec_jiffies,
2984         },
2985         {
2986                 .procname       = "redirect_load",
2987                 .data           = &ip_rt_redirect_load,
2988                 .maxlen         = sizeof(int),
2989                 .mode           = 0644,
2990                 .proc_handler   = proc_dointvec,
2991         },
2992         {
2993                 .procname       = "redirect_number",
2994                 .data           = &ip_rt_redirect_number,
2995                 .maxlen         = sizeof(int),
2996                 .mode           = 0644,
2997                 .proc_handler   = proc_dointvec,
2998         },
2999         {
3000                 .procname       = "redirect_silence",
3001                 .data           = &ip_rt_redirect_silence,
3002                 .maxlen         = sizeof(int),
3003                 .mode           = 0644,
3004                 .proc_handler   = proc_dointvec,
3005         },
3006         {
3007                 .procname       = "error_cost",
3008                 .data           = &ip_rt_error_cost,
3009                 .maxlen         = sizeof(int),
3010                 .mode           = 0644,
3011                 .proc_handler   = proc_dointvec,
3012         },
3013         {
3014                 .procname       = "error_burst",
3015                 .data           = &ip_rt_error_burst,
3016                 .maxlen         = sizeof(int),
3017                 .mode           = 0644,
3018                 .proc_handler   = proc_dointvec,
3019         },
3020         {
3021                 .procname       = "gc_elasticity",
3022                 .data           = &ip_rt_gc_elasticity,
3023                 .maxlen         = sizeof(int),
3024                 .mode           = 0644,
3025                 .proc_handler   = proc_dointvec,
3026         },
3027         {
3028                 .procname       = "mtu_expires",
3029                 .data           = &ip_rt_mtu_expires,
3030                 .maxlen         = sizeof(int),
3031                 .mode           = 0644,
3032                 .proc_handler   = proc_dointvec_jiffies,
3033         },
3034         {
3035                 .procname       = "min_pmtu",
3036                 .data           = &ip_rt_min_pmtu,
3037                 .maxlen         = sizeof(int),
3038                 .mode           = 0644,
3039                 .proc_handler   = proc_dointvec_minmax,
3040                 .extra1         = &ip_min_valid_pmtu,
3041         },
3042         {
3043                 .procname       = "min_adv_mss",
3044                 .data           = &ip_rt_min_advmss,
3045                 .maxlen         = sizeof(int),
3046                 .mode           = 0644,
3047                 .proc_handler   = proc_dointvec,
3048         },
3049         { }
3050 };
3051
3052 static struct ctl_table ipv4_route_flush_table[] = {
3053         {
3054                 .procname       = "flush",
3055                 .maxlen         = sizeof(int),
3056                 .mode           = 0200,
3057                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3058         },
3059         { },
3060 };
3061
3062 static __net_init int sysctl_route_net_init(struct net *net)
3063 {
3064         struct ctl_table *tbl;
3065
3066         tbl = ipv4_route_flush_table;
3067         if (!net_eq(net, &init_net)) {
3068                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3069                 if (!tbl)
3070                         goto err_dup;
3071
3072                 /* Don't export sysctls to unprivileged users */
3073                 if (net->user_ns != &init_user_ns)
3074                         tbl[0].procname = NULL;
3075         }
3076         tbl[0].extra1 = net;
3077
3078         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3079         if (!net->ipv4.route_hdr)
3080                 goto err_reg;
3081         return 0;
3082
3083 err_reg:
3084         if (tbl != ipv4_route_flush_table)
3085                 kfree(tbl);
3086 err_dup:
3087         return -ENOMEM;
3088 }
3089
3090 static __net_exit void sysctl_route_net_exit(struct net *net)
3091 {
3092         struct ctl_table *tbl;
3093
3094         tbl = net->ipv4.route_hdr->ctl_table_arg;
3095         unregister_net_sysctl_table(net->ipv4.route_hdr);
3096         BUG_ON(tbl == ipv4_route_flush_table);
3097         kfree(tbl);
3098 }
3099
3100 static __net_initdata struct pernet_operations sysctl_route_ops = {
3101         .init = sysctl_route_net_init,
3102         .exit = sysctl_route_net_exit,
3103 };
3104 #endif
3105
3106 static __net_init int rt_genid_init(struct net *net)
3107 {
3108         atomic_set(&net->ipv4.rt_genid, 0);
3109         atomic_set(&net->fnhe_genid, 0);
3110         atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
3111         return 0;
3112 }
3113
3114 static __net_initdata struct pernet_operations rt_genid_ops = {
3115         .init = rt_genid_init,
3116 };
3117
3118 static int __net_init ipv4_inetpeer_init(struct net *net)
3119 {
3120         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3121
3122         if (!bp)
3123                 return -ENOMEM;
3124         inet_peer_base_init(bp);
3125         net->ipv4.peers = bp;
3126         return 0;
3127 }
3128
3129 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3130 {
3131         struct inet_peer_base *bp = net->ipv4.peers;
3132
3133         net->ipv4.peers = NULL;
3134         inetpeer_invalidate_tree(bp);
3135         kfree(bp);
3136 }
3137
3138 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3139         .init   =       ipv4_inetpeer_init,
3140         .exit   =       ipv4_inetpeer_exit,
3141 };
3142
3143 #ifdef CONFIG_IP_ROUTE_CLASSID
3144 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3145 #endif /* CONFIG_IP_ROUTE_CLASSID */
3146
3147 int __init ip_rt_init(void)
3148 {
3149         int cpu;
3150
3151         ip_idents = kmalloc_array(IP_IDENTS_SZ, sizeof(*ip_idents),
3152                                   GFP_KERNEL);
3153         if (!ip_idents)
3154                 panic("IP: failed to allocate ip_idents\n");
3155
3156         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
3157
3158         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
3159         if (!ip_tstamps)
3160                 panic("IP: failed to allocate ip_tstamps\n");
3161
3162         for_each_possible_cpu(cpu) {
3163                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
3164
3165                 INIT_LIST_HEAD(&ul->head);
3166                 spin_lock_init(&ul->lock);
3167         }
3168 #ifdef CONFIG_IP_ROUTE_CLASSID
3169         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3170         if (!ip_rt_acct)
3171                 panic("IP: failed to allocate ip_rt_acct\n");
3172 #endif
3173
3174         ipv4_dst_ops.kmem_cachep =
3175                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3176                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3177
3178         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3179
3180         if (dst_entries_init(&ipv4_dst_ops) < 0)
3181                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3182
3183         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3184                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3185
3186         ipv4_dst_ops.gc_thresh = ~0;
3187         ip_rt_max_size = INT_MAX;
3188
3189         devinet_init();
3190         ip_fib_init();
3191
3192         if (ip_rt_proc_init())
3193                 pr_err("Unable to create route proc files\n");
3194 #ifdef CONFIG_XFRM
3195         xfrm_init();
3196         xfrm4_init();
3197 #endif
3198         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
3199                       RTNL_FLAG_DOIT_UNLOCKED);
3200
3201 #ifdef CONFIG_SYSCTL
3202         register_pernet_subsys(&sysctl_route_ops);
3203 #endif
3204         register_pernet_subsys(&rt_genid_ops);
3205         register_pernet_subsys(&ipv4_inetpeer_ops);
3206         return 0;
3207 }
3208
3209 #ifdef CONFIG_SYSCTL
3210 /*
3211  * We really need to sanitize the damn ipv4 init order, then all
3212  * this nonsense will go away.
3213  */
3214 void __init ip_static_sysctl_init(void)
3215 {
3216         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3217 }
3218 #endif