net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/bootmem.h>
  74 #include <linux/string.h>
  75 #include <linux/socket.h>
  76 #include <linux/sockios.h>
  77 #include <linux/errno.h>
  78 #include <linux/in.h>
  79 #include <linux/inet.h>
  80 #include <linux/netdevice.h>
  81 #include <linux/proc_fs.h>
  82 #include <linux/init.h>
  83 #include <linux/workqueue.h>
  84 #include <linux/skbuff.h>
  85 #include <linux/inetdevice.h>
  86 #include <linux/igmp.h>
  87 #include <linux/pkt_sched.h>
  88 #include <linux/mroute.h>
  89 #include <linux/netfilter_ipv4.h>
  90 #include <linux/random.h>
  91 #include <linux/jhash.h>
  92 #include <linux/rcupdate.h>
  93 #include <linux/times.h>
  94 #include <linux/slab.h>
  95 #include <linux/prefetch.h>
  96 #include <net/dst.h>
  97 #include <net/net_namespace.h>
  98 #include <net/protocol.h>
  99 #include <net/ip.h>
 100 #include <net/route.h>
 101 #include <net/inetpeer.h>
 102 #include <net/sock.h>
 103 #include <net/ip_fib.h>
 104 #include <net/arp.h>
 105 #include <net/tcp.h>
 106 #include <net/icmp.h>
 107 #include <net/xfrm.h>
 108 #include <net/netevent.h>
 109 #include <net/rtnetlink.h>
 110 #ifdef CONFIG_SYSCTL
 111 #include <linux/sysctl.h>
 112 #include <linux/kmemleak.h>
 113 #endif
 114 #include <net/secure_seq.h>
 115
 116 #define RT_FL_TOS(oldflp4) \
 117         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 118
 119 #define IP_MAX_MTU      0xFFF0
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 125 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
 126 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
 127 static int ip_rt_redirect_number __read_mostly  = 9;
 128 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 129 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 130 static int ip_rt_error_cost __read_mostly       = HZ;
 131 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 132 static int ip_rt_gc_elasticity __read_mostly    = 8;
 133 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 134 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 135 static int ip_rt_min_advmss __read_mostly       = 256;
 136
 137 /*
 138  *      Interface to generic destination cache.
 139  */
 140
 141 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 142 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 143 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 144 static void              ipv4_dst_destroy(struct dst_entry *dst);
 145 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 146 static void              ipv4_link_failure(struct sk_buff *skb);
 147 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 148                                            struct sk_buff *skb, u32 mtu);
 149 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 150                                         struct sk_buff *skb);
 151
 152 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 153                             int how)
 154 {
 155 }
 156
 157 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 158 {
 159         WARN_ON(1);
 160         return NULL;
 161 }
 162
 163 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 164                                            struct sk_buff *skb,
 165                                            const void *daddr);
 166
 167 static struct dst_ops ipv4_dst_ops = {
 168         .family =               AF_INET,
 169         .protocol =             cpu_to_be16(ETH_P_IP),
 170         .check =                ipv4_dst_check,
 171         .default_advmss =       ipv4_default_advmss,
 172         .mtu =                  ipv4_mtu,
 173         .cow_metrics =          ipv4_cow_metrics,
 174         .destroy =              ipv4_dst_destroy,
 175         .ifdown =               ipv4_dst_ifdown,
 176         .negative_advice =      ipv4_negative_advice,
 177         .link_failure =         ipv4_link_failure,
 178         .update_pmtu =          ip_rt_update_pmtu,
 179         .redirect =             ip_do_redirect,
 180         .local_out =            __ip_local_out,
 181         .neigh_lookup =         ipv4_neigh_lookup,
 182 };
 183
 184 #define ECN_OR_COST(class)      TC_PRIO_##class
 185
 186 const __u8 ip_tos2prio[16] = {
 187         TC_PRIO_BESTEFFORT,
 188         ECN_OR_COST(BESTEFFORT),
 189         TC_PRIO_BESTEFFORT,
 190         ECN_OR_COST(BESTEFFORT),
 191         TC_PRIO_BULK,
 192         ECN_OR_COST(BULK),
 193         TC_PRIO_BULK,
 194         ECN_OR_COST(BULK),
 195         TC_PRIO_INTERACTIVE,
 196         ECN_OR_COST(INTERACTIVE),
 197         TC_PRIO_INTERACTIVE,
 198         ECN_OR_COST(INTERACTIVE),
 199         TC_PRIO_INTERACTIVE_BULK,
 200         ECN_OR_COST(INTERACTIVE_BULK),
 201         TC_PRIO_INTERACTIVE_BULK,
 202         ECN_OR_COST(INTERACTIVE_BULK)
 203 };
 204 EXPORT_SYMBOL(ip_tos2prio);
 205
 206 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 207 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 208
 209 static inline int rt_genid(struct net *net)
 210 {
 211         return atomic_read(&net->ipv4.rt_genid);
 212 }
 213
 214 #ifdef CONFIG_PROC_FS
 215 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 216 {
 217         if (*pos)
 218                 return NULL;
 219         return SEQ_START_TOKEN;
 220 }
 221
 222 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 223 {
 224         ++*pos;
 225         return NULL;
 226 }
 227
 228 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 229 {
 230 }
 231
 232 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 233 {
 234         if (v == SEQ_START_TOKEN)
 235                 seq_printf(seq, "%-127s\n",
 236                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 237                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 238                            "HHUptod\tSpecDst");
 239         return 0;
 240 }
 241
 242 static const struct seq_operations rt_cache_seq_ops = {
 243         .start  = rt_cache_seq_start,
 244         .next   = rt_cache_seq_next,
 245         .stop   = rt_cache_seq_stop,
 246         .show   = rt_cache_seq_show,
 247 };
 248
 249 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 250 {
 251         return seq_open(file, &rt_cache_seq_ops);
 252 }
 253
 254 static const struct file_operations rt_cache_seq_fops = {
 255         .owner   = THIS_MODULE,
 256         .open    = rt_cache_seq_open,
 257         .read    = seq_read,
 258         .llseek  = seq_lseek,
 259         .release = seq_release,
 260 };
 261
 262
 263 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 264 {
 265         int cpu;
 266
 267         if (*pos == 0)
 268                 return SEQ_START_TOKEN;
 269
 270         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 271                 if (!cpu_possible(cpu))
 272                         continue;
 273                 *pos = cpu+1;
 274                 return &per_cpu(rt_cache_stat, cpu);
 275         }
 276         return NULL;
 277 }
 278
 279 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 280 {
 281         int cpu;
 282
 283         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 284                 if (!cpu_possible(cpu))
 285                         continue;
 286                 *pos = cpu+1;
 287                 return &per_cpu(rt_cache_stat, cpu);
 288         }
 289         return NULL;
 290
 291 }
 292
 293 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 294 {
 295
 296 }
 297
 298 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 299 {
 300         struct rt_cache_stat *st = v;
 301
 302         if (v == SEQ_START_TOKEN) {
 303                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 304                 return 0;
 305         }
 306
 307         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 308                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 309                    dst_entries_get_slow(&ipv4_dst_ops),
 310                    st->in_hit,
 311                    st->in_slow_tot,
 312                    st->in_slow_mc,
 313                    st->in_no_route,
 314                    st->in_brd,
 315                    st->in_martian_dst,
 316                    st->in_martian_src,
 317
 318                    st->out_hit,
 319                    st->out_slow_tot,
 320                    st->out_slow_mc,
 321
 322                    st->gc_total,
 323                    st->gc_ignored,
 324                    st->gc_goal_miss,
 325                    st->gc_dst_overflow,
 326                    st->in_hlist_search,
 327                    st->out_hlist_search
 328                 );
 329         return 0;
 330 }
 331
 332 static const struct seq_operations rt_cpu_seq_ops = {
 333         .start  = rt_cpu_seq_start,
 334         .next   = rt_cpu_seq_next,
 335         .stop   = rt_cpu_seq_stop,
 336         .show   = rt_cpu_seq_show,
 337 };
 338
 339
 340 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 341 {
 342         return seq_open(file, &rt_cpu_seq_ops);
 343 }
 344
 345 static const struct file_operations rt_cpu_seq_fops = {
 346         .owner   = THIS_MODULE,
 347         .open    = rt_cpu_seq_open,
 348         .read    = seq_read,
 349         .llseek  = seq_lseek,
 350         .release = seq_release,
 351 };
 352
 353 #ifdef CONFIG_IP_ROUTE_CLASSID
 354 static int rt_acct_proc_show(struct seq_file *m, void *v)
 355 {
 356         struct ip_rt_acct *dst, *src;
 357         unsigned int i, j;
 358
 359         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 360         if (!dst)
 361                 return -ENOMEM;
 362
 363         for_each_possible_cpu(i) {
 364                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 365                 for (j = 0; j < 256; j++) {
 366                         dst[j].o_bytes   += src[j].o_bytes;
 367                         dst[j].o_packets += src[j].o_packets;
 368                         dst[j].i_bytes   += src[j].i_bytes;
 369                         dst[j].i_packets += src[j].i_packets;
 370                 }
 371         }
 372
 373         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 374         kfree(dst);
 375         return 0;
 376 }
 377
 378 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 379 {
 380         return single_open(file, rt_acct_proc_show, NULL);
 381 }
 382
 383 static const struct file_operations rt_acct_proc_fops = {
 384         .owner          = THIS_MODULE,
 385         .open           = rt_acct_proc_open,
 386         .read           = seq_read,
 387         .llseek         = seq_lseek,
 388         .release        = single_release,
 389 };
 390 #endif
 391
 392 static int __net_init ip_rt_do_proc_init(struct net *net)
 393 {
 394         struct proc_dir_entry *pde;
 395
 396         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
 397                         &rt_cache_seq_fops);
 398         if (!pde)
 399                 goto err1;
 400
 401         pde = proc_create("rt_cache", S_IRUGO,
 402                           net->proc_net_stat, &rt_cpu_seq_fops);
 403         if (!pde)
 404                 goto err2;
 405
 406 #ifdef CONFIG_IP_ROUTE_CLASSID
 407         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 408         if (!pde)
 409                 goto err3;
 410 #endif
 411         return 0;
 412
 413 #ifdef CONFIG_IP_ROUTE_CLASSID
 414 err3:
 415         remove_proc_entry("rt_cache", net->proc_net_stat);
 416 #endif
 417 err2:
 418         remove_proc_entry("rt_cache", net->proc_net);
 419 err1:
 420         return -ENOMEM;
 421 }
 422
 423 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 424 {
 425         remove_proc_entry("rt_cache", net->proc_net_stat);
 426         remove_proc_entry("rt_cache", net->proc_net);
 427 #ifdef CONFIG_IP_ROUTE_CLASSID
 428         remove_proc_entry("rt_acct", net->proc_net);
 429 #endif
 430 }
 431
 432 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 433         .init = ip_rt_do_proc_init,
 434         .exit = ip_rt_do_proc_exit,
 435 };
 436
 437 static int __init ip_rt_proc_init(void)
 438 {
 439         return register_pernet_subsys(&ip_rt_proc_ops);
 440 }
 441
 442 #else
 443 static inline int ip_rt_proc_init(void)
 444 {
 445         return 0;
 446 }
 447 #endif /* CONFIG_PROC_FS */
 448
 449 static inline int rt_is_expired(struct rtable *rth)
 450 {
 451         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 452 }
 453
 454 /*
 455  * Perturbation of rt_genid by a small quantity [1..256]
 456  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
 457  * many times (2^24) without giving recent rt_genid.
 458  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
 459  */
 460 static void rt_cache_invalidate(struct net *net)
 461 {
 462         unsigned char shuffle;
 463
 464         get_random_bytes(&shuffle, sizeof(shuffle));
 465         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
 466 }
 467
 468 /*
 469  * delay < 0  : invalidate cache (fast : entries will be deleted later)
 470  * delay >= 0 : invalidate & flush cache (can be long)
 471  */
 472 void rt_cache_flush(struct net *net, int delay)
 473 {
 474         rt_cache_invalidate(net);
 475 }
 476
 477 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 478                                            struct sk_buff *skb,
 479                                            const void *daddr)
 480 {
 481         struct net_device *dev = dst->dev;
 482         const __be32 *pkey = daddr;
 483         const struct rtable *rt;
 484         struct neighbour *n;
 485
 486         rt = (const struct rtable *) dst;
 487         if (rt->rt_gateway)
 488                 pkey = (const __be32 *) &rt->rt_gateway;
 489         else if (skb)
 490                 pkey = &ip_hdr(skb)->daddr;
 491
 492         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 493         if (n)
 494                 return n;
 495         return neigh_create(&arp_tbl, pkey, dev);
 496 }
 497
 498 /*
 499  * Peer allocation may fail only in serious out-of-memory conditions.  However
 500  * we still can generate some output.
 501  * Random ID selection looks a bit dangerous because we have no chances to
 502  * select ID being unique in a reasonable period of time.
 503  * But broken packet identifier may be better than no packet at all.
 504  */
 505 static void ip_select_fb_ident(struct iphdr *iph)
 506 {
 507         static DEFINE_SPINLOCK(ip_fb_id_lock);
 508         static u32 ip_fallback_id;
 509         u32 salt;
 510
 511         spin_lock_bh(&ip_fb_id_lock);
 512         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
 513         iph->id = htons(salt & 0xFFFF);
 514         ip_fallback_id = salt;
 515         spin_unlock_bh(&ip_fb_id_lock);
 516 }
 517
 518 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
 519 {
 520         struct net *net = dev_net(dst->dev);
 521         struct inet_peer *peer;
 522
 523         peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
 524         if (peer) {
 525                 iph->id = htons(inet_getid(peer, more));
 526                 inet_putpeer(peer);
 527                 return;
 528         }
 529
 530         ip_select_fb_ident(iph);
 531 }
 532 EXPORT_SYMBOL(__ip_select_ident);
 533
 534 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 535                              const struct iphdr *iph,
 536                              int oif, u8 tos,
 537                              u8 prot, u32 mark, int flow_flags)
 538 {
 539         if (sk) {
 540                 const struct inet_sock *inet = inet_sk(sk);
 541
 542                 oif = sk->sk_bound_dev_if;
 543                 mark = sk->sk_mark;
 544                 tos = RT_CONN_FLAGS(sk);
 545                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 546         }
 547         flowi4_init_output(fl4, oif, mark, tos,
 548                            RT_SCOPE_UNIVERSE, prot,
 549                            flow_flags,
 550                            iph->daddr, iph->saddr, 0, 0);
 551 }
 552
 553 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 554                                const struct sock *sk)
 555 {
 556         const struct iphdr *iph = ip_hdr(skb);
 557         int oif = skb->dev->ifindex;
 558         u8 tos = RT_TOS(iph->tos);
 559         u8 prot = iph->protocol;
 560         u32 mark = skb->mark;
 561
 562         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 563 }
 564
 565 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 566 {
 567         const struct inet_sock *inet = inet_sk(sk);
 568         const struct ip_options_rcu *inet_opt;
 569         __be32 daddr = inet->inet_daddr;
 570
 571         rcu_read_lock();
 572         inet_opt = rcu_dereference(inet->inet_opt);
 573         if (inet_opt && inet_opt->opt.srr)
 574                 daddr = inet_opt->opt.faddr;
 575         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 576                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 577                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 578                            inet_sk_flowi_flags(sk),
 579                            daddr, inet->inet_saddr, 0, 0);
 580         rcu_read_unlock();
 581 }
 582
 583 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 584                                  const struct sk_buff *skb)
 585 {
 586         if (skb)
 587                 build_skb_flow_key(fl4, skb, sk);
 588         else
 589                 build_sk_flow_key(fl4, sk);
 590 }
 591
 592 static DEFINE_SEQLOCK(fnhe_seqlock);
 593
 594 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 595 {
 596         struct fib_nh_exception *fnhe, *oldest;
 597
 598         oldest = rcu_dereference(hash->chain);
 599         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 600              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 601                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 602                         oldest = fnhe;
 603         }
 604         return oldest;
 605 }
 606
 607 static inline u32 fnhe_hashfun(__be32 daddr)
 608 {
 609         u32 hval;
 610
 611         hval = (__force u32) daddr;
 612         hval ^= (hval >> 11) ^ (hval >> 22);
 613
 614         return hval & (FNHE_HASH_SIZE - 1);
 615 }
 616
 617 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 618                                   u32 pmtu, unsigned long expires)
 619 {
 620         struct fnhe_hash_bucket *hash;
 621         struct fib_nh_exception *fnhe;
 622         int depth;
 623         u32 hval = fnhe_hashfun(daddr);
 624
 625         write_seqlock_bh(&fnhe_seqlock);
 626
 627         hash = nh->nh_exceptions;
 628         if (!hash) {
 629                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 630                 if (!hash)
 631                         goto out_unlock;
 632                 nh->nh_exceptions = hash;
 633         }
 634
 635         hash += hval;
 636
 637         depth = 0;
 638         for (fnhe = rcu_dereference(hash->chain); fnhe;
 639              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 640                 if (fnhe->fnhe_daddr == daddr)
 641                         break;
 642                 depth++;
 643         }
 644
 645         if (fnhe) {
 646                 if (gw)
 647                         fnhe->fnhe_gw = gw;
 648                 if (pmtu) {
 649                         fnhe->fnhe_pmtu = pmtu;
 650                         fnhe->fnhe_expires = expires;
 651                 }
 652         } else {
 653                 if (depth > FNHE_RECLAIM_DEPTH)
 654                         fnhe = fnhe_oldest(hash);
 655                 else {
 656                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 657                         if (!fnhe)
 658                                 goto out_unlock;
 659
 660                         fnhe->fnhe_next = hash->chain;
 661                         rcu_assign_pointer(hash->chain, fnhe);
 662                 }
 663                 fnhe->fnhe_daddr = daddr;
 664                 fnhe->fnhe_gw = gw;
 665                 fnhe->fnhe_pmtu = pmtu;
 666                 fnhe->fnhe_expires = expires;
 667         }
 668
 669         fnhe->fnhe_stamp = jiffies;
 670
 671 out_unlock:
 672         write_sequnlock_bh(&fnhe_seqlock);
 673         return;
 674 }
 675
 676 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 677                              bool kill_route)
 678 {
 679         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 680         __be32 old_gw = ip_hdr(skb)->saddr;
 681         struct net_device *dev = skb->dev;
 682         struct in_device *in_dev;
 683         struct fib_result res;
 684         struct neighbour *n;
 685         struct net *net;
 686
 687         switch (icmp_hdr(skb)->code & 7) {
 688         case ICMP_REDIR_NET:
 689         case ICMP_REDIR_NETTOS:
 690         case ICMP_REDIR_HOST:
 691         case ICMP_REDIR_HOSTTOS:
 692                 break;
 693
 694         default:
 695                 return;
 696         }
 697
 698         if (rt->rt_gateway != old_gw)
 699                 return;
 700
 701         in_dev = __in_dev_get_rcu(dev);
 702         if (!in_dev)
 703                 return;
 704
 705         net = dev_net(dev);
 706         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 707             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 708             ipv4_is_zeronet(new_gw))
 709                 goto reject_redirect;
 710
 711         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 712                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 713                         goto reject_redirect;
 714                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 715                         goto reject_redirect;
 716         } else {
 717                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 718                         goto reject_redirect;
 719         }
 720
 721         n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
 722         if (n) {
 723                 if (!(n->nud_state & NUD_VALID)) {
 724                         neigh_event_send(n, NULL);
 725                 } else {
 726                         if (fib_lookup(net, fl4, &res) == 0) {
 727                                 struct fib_nh *nh = &FIB_RES_NH(res);
 728
 729                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 730                                                       0, 0);
 731                         }
 732                         if (kill_route)
 733                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 734                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 735                 }
 736                 neigh_release(n);
 737         }
 738         return;
 739
 740 reject_redirect:
 741 #ifdef CONFIG_IP_ROUTE_VERBOSE
 742         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 743                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 744                 __be32 daddr = iph->daddr;
 745                 __be32 saddr = iph->saddr;
 746
 747                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 748                                      "  Advised path = %pI4 -> %pI4\n",
 749                                      &old_gw, dev->name, &new_gw,
 750                                      &saddr, &daddr);
 751         }
 752 #endif
 753         ;
 754 }
 755
 756 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 757 {
 758         struct rtable *rt;
 759         struct flowi4 fl4;
 760
 761         rt = (struct rtable *) dst;
 762
 763         ip_rt_build_flow_key(&fl4, sk, skb);
 764         __ip_do_redirect(rt, skb, &fl4, true);
 765 }
 766
 767 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 768 {
 769         struct rtable *rt = (struct rtable *)dst;
 770         struct dst_entry *ret = dst;
 771
 772         if (rt) {
 773                 if (dst->obsolete > 0) {
 774                         ip_rt_put(rt);
 775                         ret = NULL;
 776                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 777                            rt->dst.expires) {
 778                         ip_rt_put(rt);
 779                         ret = NULL;
 780                 }
 781         }
 782         return ret;
 783 }
 784
 785 /*
 786  * Algorithm:
 787  *      1. The first ip_rt_redirect_number redirects are sent
 788  *         with exponential backoff, then we stop sending them at all,
 789  *         assuming that the host ignores our redirects.
 790  *      2. If we did not see packets requiring redirects
 791  *         during ip_rt_redirect_silence, we assume that the host
 792  *         forgot redirected route and start to send redirects again.
 793  *
 794  * This algorithm is much cheaper and more intelligent than dumb load limiting
 795  * in icmp.c.
 796  *
 797  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 798  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 799  */
 800
 801 void ip_rt_send_redirect(struct sk_buff *skb)
 802 {
 803         struct rtable *rt = skb_rtable(skb);
 804         struct in_device *in_dev;
 805         struct inet_peer *peer;
 806         struct net *net;
 807         int log_martians;
 808
 809         rcu_read_lock();
 810         in_dev = __in_dev_get_rcu(rt->dst.dev);
 811         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 812                 rcu_read_unlock();
 813                 return;
 814         }
 815         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 816         rcu_read_unlock();
 817
 818         net = dev_net(rt->dst.dev);
 819         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 820         if (!peer) {
 821                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
 822                 return;
 823         }
 824
 825         /* No redirected packets during ip_rt_redirect_silence;
 826          * reset the algorithm.
 827          */
 828         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 829                 peer->rate_tokens = 0;
 830
 831         /* Too many ignored redirects; do not send anything
 832          * set dst.rate_last to the last seen redirected packet.
 833          */
 834         if (peer->rate_tokens >= ip_rt_redirect_number) {
 835                 peer->rate_last = jiffies;
 836                 goto out_put_peer;
 837         }
 838
 839         /* Check for load limit; set rate_last to the latest sent
 840          * redirect.
 841          */
 842         if (peer->rate_tokens == 0 ||
 843             time_after(jiffies,
 844                        (peer->rate_last +
 845                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 846                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
 847                 peer->rate_last = jiffies;
 848                 ++peer->rate_tokens;
 849 #ifdef CONFIG_IP_ROUTE_VERBOSE
 850                 if (log_martians &&
 851                     peer->rate_tokens == ip_rt_redirect_number)
 852                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 853                                              &ip_hdr(skb)->saddr, rt->rt_iif,
 854                                              &ip_hdr(skb)->daddr, &rt->rt_gateway);
 855 #endif
 856         }
 857 out_put_peer:
 858         inet_putpeer(peer);
 859 }
 860
 861 static int ip_error(struct sk_buff *skb)
 862 {
 863         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 864         struct rtable *rt = skb_rtable(skb);
 865         struct inet_peer *peer;
 866         unsigned long now;
 867         struct net *net;
 868         bool send;
 869         int code;
 870
 871         net = dev_net(rt->dst.dev);
 872         if (!IN_DEV_FORWARD(in_dev)) {
 873                 switch (rt->dst.error) {
 874                 case EHOSTUNREACH:
 875                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 876                         break;
 877
 878                 case ENETUNREACH:
 879                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 880                         break;
 881                 }
 882                 goto out;
 883         }
 884
 885         switch (rt->dst.error) {
 886         case EINVAL:
 887         default:
 888                 goto out;
 889         case EHOSTUNREACH:
 890                 code = ICMP_HOST_UNREACH;
 891                 break;
 892         case ENETUNREACH:
 893                 code = ICMP_NET_UNREACH;
 894                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 895                 break;
 896         case EACCES:
 897                 code = ICMP_PKT_FILTERED;
 898                 break;
 899         }
 900
 901         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
 902
 903         send = true;
 904         if (peer) {
 905                 now = jiffies;
 906                 peer->rate_tokens += now - peer->rate_last;
 907                 if (peer->rate_tokens > ip_rt_error_burst)
 908                         peer->rate_tokens = ip_rt_error_burst;
 909                 peer->rate_last = now;
 910                 if (peer->rate_tokens >= ip_rt_error_cost)
 911                         peer->rate_tokens -= ip_rt_error_cost;
 912                 else
 913                         send = false;
 914                 inet_putpeer(peer);
 915         }
 916         if (send)
 917                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 918
 919 out:    kfree_skb(skb);
 920         return 0;
 921 }
 922
 923 static u32 __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 924 {
 925         struct fib_result res;
 926
 927         if (mtu < ip_rt_min_pmtu)
 928                 mtu = ip_rt_min_pmtu;
 929
 930         if (fib_lookup(dev_net(rt->dst.dev), fl4, &res) == 0) {
 931                 struct fib_nh *nh = &FIB_RES_NH(res);
 932
 933                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 934                                       jiffies + ip_rt_mtu_expires);
 935         }
 936         return mtu;
 937 }
 938
 939 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 940                               struct sk_buff *skb, u32 mtu)
 941 {
 942         struct rtable *rt = (struct rtable *) dst;
 943         struct flowi4 fl4;
 944
 945         ip_rt_build_flow_key(&fl4, sk, skb);
 946         mtu = __ip_rt_update_pmtu(rt, &fl4, mtu);
 947
 948         if (!rt->rt_pmtu) {
 949                 dst->obsolete = DST_OBSOLETE_KILL;
 950         } else {
 951                 rt->rt_pmtu = mtu;
 952                 dst_set_expires(&rt->dst, ip_rt_mtu_expires);
 953         }
 954 }
 955
 956 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
 957                       int oif, u32 mark, u8 protocol, int flow_flags)
 958 {
 959         const struct iphdr *iph = (const struct iphdr *) skb->data;
 960         struct flowi4 fl4;
 961         struct rtable *rt;
 962
 963         __build_flow_key(&fl4, NULL, iph, oif,
 964                          RT_TOS(iph->tos), protocol, mark, flow_flags);
 965         rt = __ip_route_output_key(net, &fl4);
 966         if (!IS_ERR(rt)) {
 967                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 968                 ip_rt_put(rt);
 969         }
 970 }
 971 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
 972
 973 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
 974 {
 975         const struct iphdr *iph = (const struct iphdr *) skb->data;
 976         struct flowi4 fl4;
 977         struct rtable *rt;
 978
 979         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
 980         rt = __ip_route_output_key(sock_net(sk), &fl4);
 981         if (!IS_ERR(rt)) {
 982                 __ip_rt_update_pmtu(rt, &fl4, mtu);
 983                 ip_rt_put(rt);
 984         }
 985 }
 986 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
 987
 988 void ipv4_redirect(struct sk_buff *skb, struct net *net,
 989                    int oif, u32 mark, u8 protocol, int flow_flags)
 990 {
 991         const struct iphdr *iph = (const struct iphdr *) skb->data;
 992         struct flowi4 fl4;
 993         struct rtable *rt;
 994
 995         __build_flow_key(&fl4, NULL, iph, oif,
 996                          RT_TOS(iph->tos), protocol, mark, flow_flags);
 997         rt = __ip_route_output_key(net, &fl4);
 998         if (!IS_ERR(rt)) {
 999                 __ip_do_redirect(rt, skb, &fl4, false);
1000                 ip_rt_put(rt);
1001         }
1002 }
1003 EXPORT_SYMBOL_GPL(ipv4_redirect);
1004
1005 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1006 {
1007         const struct iphdr *iph = (const struct iphdr *) skb->data;
1008         struct flowi4 fl4;
1009         struct rtable *rt;
1010
1011         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1012         rt = __ip_route_output_key(sock_net(sk), &fl4);
1013         if (!IS_ERR(rt)) {
1014                 __ip_do_redirect(rt, skb, &fl4, false);
1015                 ip_rt_put(rt);
1016         }
1017 }
1018 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1019
1020 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1021 {
1022         struct rtable *rt = (struct rtable *) dst;
1023
1024         /* All IPV4 dsts are created with ->obsolete set to the value
1025          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1026          * into this function always.
1027          *
1028          * When a PMTU/redirect information update invalidates a
1029          * route, this is indicated by setting obsolete to
1030          * DST_OBSOLETE_KILL.
1031          */
1032         if (dst->obsolete == DST_OBSOLETE_KILL || rt_is_expired(rt))
1033                 return NULL;
1034         return dst;
1035 }
1036
1037 static void ipv4_dst_destroy(struct dst_entry *dst)
1038 {
1039         struct rtable *rt = (struct rtable *) dst;
1040
1041         if (rt->fi) {
1042                 fib_info_put(rt->fi);
1043                 rt->fi = NULL;
1044         }
1045 }
1046
1047
1048 static void ipv4_link_failure(struct sk_buff *skb)
1049 {
1050         struct rtable *rt;
1051
1052         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1053
1054         rt = skb_rtable(skb);
1055         if (rt)
1056                 dst_set_expires(&rt->dst, 0);
1057 }
1058
1059 static int ip_rt_bug(struct sk_buff *skb)
1060 {
1061         pr_debug("%s: %pI4 -> %pI4, %s\n",
1062                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1063                  skb->dev ? skb->dev->name : "?");
1064         kfree_skb(skb);
1065         WARN_ON(1);
1066         return 0;
1067 }
1068
1069 /*
1070    We do not cache source address of outgoing interface,
1071    because it is used only by IP RR, TS and SRR options,
1072    so that it out of fast path.
1073
1074    BTW remember: "addr" is allowed to be not aligned
1075    in IP options!
1076  */
1077
1078 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1079 {
1080         __be32 src;
1081
1082         if (rt_is_output_route(rt))
1083                 src = ip_hdr(skb)->saddr;
1084         else {
1085                 struct fib_result res;
1086                 struct flowi4 fl4;
1087                 struct iphdr *iph;
1088
1089                 iph = ip_hdr(skb);
1090
1091                 memset(&fl4, 0, sizeof(fl4));
1092                 fl4.daddr = iph->daddr;
1093                 fl4.saddr = iph->saddr;
1094                 fl4.flowi4_tos = RT_TOS(iph->tos);
1095                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1096                 fl4.flowi4_iif = skb->dev->ifindex;
1097                 fl4.flowi4_mark = skb->mark;
1098
1099                 rcu_read_lock();
1100                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1101                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1102                 else
1103                         src = inet_select_addr(rt->dst.dev,
1104                                                rt_nexthop(rt, iph->daddr),
1105                                                RT_SCOPE_UNIVERSE);
1106                 rcu_read_unlock();
1107         }
1108         memcpy(addr, &src, 4);
1109 }
1110
1111 #ifdef CONFIG_IP_ROUTE_CLASSID
1112 static void set_class_tag(struct rtable *rt, u32 tag)
1113 {
1114         if (!(rt->dst.tclassid & 0xFFFF))
1115                 rt->dst.tclassid |= tag & 0xFFFF;
1116         if (!(rt->dst.tclassid & 0xFFFF0000))
1117                 rt->dst.tclassid |= tag & 0xFFFF0000;
1118 }
1119 #endif
1120
1121 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1122 {
1123         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1124
1125         if (advmss == 0) {
1126                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1127                                ip_rt_min_advmss);
1128                 if (advmss > 65535 - 40)
1129                         advmss = 65535 - 40;
1130         }
1131         return advmss;
1132 }
1133
1134 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1135 {
1136         const struct rtable *rt = (const struct rtable *) dst;
1137         unsigned int mtu = rt->rt_pmtu;
1138
1139         if (mtu && time_after_eq(jiffies, rt->dst.expires))
1140                 mtu = 0;
1141
1142         if (!mtu)
1143                 mtu = dst_metric_raw(dst, RTAX_MTU);
1144
1145         if (mtu && rt_is_output_route(rt))
1146                 return mtu;
1147
1148         mtu = dst->dev->mtu;
1149
1150         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1151                 if (rt->rt_gateway && mtu > 576)
1152                         mtu = 576;
1153         }
1154
1155         if (mtu > IP_MAX_MTU)
1156                 mtu = IP_MAX_MTU;
1157
1158         return mtu;
1159 }
1160
1161 static void rt_init_metrics(struct rtable *rt, struct fib_info *fi)
1162 {
1163         if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1164                 rt->fi = fi;
1165                 atomic_inc(&fi->fib_clntref);
1166         }
1167         dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1168 }
1169
1170 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1171 {
1172         struct fnhe_hash_bucket *hash = nh->nh_exceptions;
1173         struct fib_nh_exception *fnhe;
1174         u32 hval;
1175
1176         if (!hash)
1177                 return NULL;
1178
1179         hval = fnhe_hashfun(daddr);
1180
1181         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1182              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1183                 if (fnhe->fnhe_daddr == daddr)
1184                         return fnhe;
1185         }
1186         return NULL;
1187 }
1188
1189 static void rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1190                               __be32 daddr)
1191 {
1192         __be32 fnhe_daddr, gw;
1193         unsigned long expires;
1194         unsigned int seq;
1195         u32 pmtu;
1196
1197 restart:
1198         seq = read_seqbegin(&fnhe_seqlock);
1199         fnhe_daddr = fnhe->fnhe_daddr;
1200         gw = fnhe->fnhe_gw;
1201         pmtu = fnhe->fnhe_pmtu;
1202         expires = fnhe->fnhe_expires;
1203         if (read_seqretry(&fnhe_seqlock, seq))
1204                 goto restart;
1205
1206         if (daddr != fnhe_daddr)
1207                 return;
1208
1209         if (pmtu) {
1210                 unsigned long diff = expires - jiffies;
1211
1212                 if (time_before(jiffies, expires)) {
1213                         rt->rt_pmtu = pmtu;
1214                         dst_set_expires(&rt->dst, diff);
1215                 }
1216         }
1217         if (gw) {
1218                 rt->rt_flags |= RTCF_REDIRECTED;
1219                 rt->rt_gateway = gw;
1220         }
1221         fnhe->fnhe_stamp = jiffies;
1222 }
1223
1224 static inline void rt_release_rcu(struct rcu_head *head)
1225 {
1226         struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head);
1227         dst_release(dst);
1228 }
1229
1230 static void rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1231 {
1232         struct rtable *orig, *prev, **p = &nh->nh_rth_output;
1233
1234         orig = *p;
1235
1236         prev = cmpxchg(p, orig, rt);
1237         if (prev == orig) {
1238                 dst_clone(&rt->dst);
1239                 if (orig)
1240                         call_rcu_bh(&orig->dst.rcu_head, rt_release_rcu);
1241         }
1242 }
1243
1244 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1245                            const struct fib_result *res,
1246                            struct fib_nh_exception *fnhe,
1247                            struct fib_info *fi, u16 type, u32 itag)
1248 {
1249         if (fi) {
1250                 struct fib_nh *nh = &FIB_RES_NH(*res);
1251
1252                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK)
1253                         rt->rt_gateway = nh->nh_gw;
1254                 if (unlikely(fnhe))
1255                         rt_bind_exception(rt, fnhe, daddr);
1256                 rt_init_metrics(rt, fi);
1257 #ifdef CONFIG_IP_ROUTE_CLASSID
1258                 rt->dst.tclassid = nh->nh_tclassid;
1259 #endif
1260                 if (!(rt->dst.flags & DST_HOST) &&
1261                     rt_is_output_route(rt))
1262                         rt_cache_route(nh, rt);
1263         }
1264
1265 #ifdef CONFIG_IP_ROUTE_CLASSID
1266 #ifdef CONFIG_IP_MULTIPLE_TABLES
1267         set_class_tag(rt, res->tclassid);
1268 #endif
1269         set_class_tag(rt, itag);
1270 #endif
1271 }
1272
1273 static struct rtable *rt_dst_alloc(struct net_device *dev,
1274                                    bool nopolicy, bool noxfrm, bool will_cache)
1275 {
1276         return dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1277                          (will_cache ? 0 : DST_HOST) | DST_NOCACHE |
1278                          (nopolicy ? DST_NOPOLICY : 0) |
1279                          (noxfrm ? DST_NOXFRM : 0));
1280 }
1281
1282 /* called in rcu_read_lock() section */
1283 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1284                                 u8 tos, struct net_device *dev, int our)
1285 {
1286         struct rtable *rth;
1287         struct in_device *in_dev = __in_dev_get_rcu(dev);
1288         u32 itag = 0;
1289         int err;
1290
1291         /* Primary sanity checks. */
1292
1293         if (in_dev == NULL)
1294                 return -EINVAL;
1295
1296         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1297             skb->protocol != htons(ETH_P_IP))
1298                 goto e_inval;
1299
1300         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1301                 if (ipv4_is_loopback(saddr))
1302                         goto e_inval;
1303
1304         if (ipv4_is_zeronet(saddr)) {
1305                 if (!ipv4_is_local_multicast(daddr))
1306                         goto e_inval;
1307         } else {
1308                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1309                                           in_dev, &itag);
1310                 if (err < 0)
1311                         goto e_err;
1312         }
1313         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1314                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1315         if (!rth)
1316                 goto e_nobufs;
1317
1318 #ifdef CONFIG_IP_ROUTE_CLASSID
1319         rth->dst.tclassid = itag;
1320 #endif
1321         rth->dst.output = ip_rt_bug;
1322
1323         rth->rt_genid   = rt_genid(dev_net(dev));
1324         rth->rt_flags   = RTCF_MULTICAST;
1325         rth->rt_type    = RTN_MULTICAST;
1326         rth->rt_route_iif = dev->ifindex;
1327         rth->rt_iif     = dev->ifindex;
1328         rth->rt_oif     = 0;
1329         rth->rt_pmtu    = 0;
1330         rth->rt_gateway = 0;
1331         rth->fi = NULL;
1332         if (our) {
1333                 rth->dst.input= ip_local_deliver;
1334                 rth->rt_flags |= RTCF_LOCAL;
1335         }
1336
1337 #ifdef CONFIG_IP_MROUTE
1338         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1339                 rth->dst.input = ip_mr_input;
1340 #endif
1341         RT_CACHE_STAT_INC(in_slow_mc);
1342
1343         skb_dst_set(skb, &rth->dst);
1344         return 0;
1345
1346 e_nobufs:
1347         return -ENOBUFS;
1348 e_inval:
1349         return -EINVAL;
1350 e_err:
1351         return err;
1352 }
1353
1354
1355 static void ip_handle_martian_source(struct net_device *dev,
1356                                      struct in_device *in_dev,
1357                                      struct sk_buff *skb,
1358                                      __be32 daddr,
1359                                      __be32 saddr)
1360 {
1361         RT_CACHE_STAT_INC(in_martian_src);
1362 #ifdef CONFIG_IP_ROUTE_VERBOSE
1363         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1364                 /*
1365                  *      RFC1812 recommendation, if source is martian,
1366                  *      the only hint is MAC header.
1367                  */
1368                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1369                         &daddr, &saddr, dev->name);
1370                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1371                         print_hex_dump(KERN_WARNING, "ll header: ",
1372                                        DUMP_PREFIX_OFFSET, 16, 1,
1373                                        skb_mac_header(skb),
1374                                        dev->hard_header_len, true);
1375                 }
1376         }
1377 #endif
1378 }
1379
1380 /* called in rcu_read_lock() section */
1381 static int __mkroute_input(struct sk_buff *skb,
1382                            const struct fib_result *res,
1383                            struct in_device *in_dev,
1384                            __be32 daddr, __be32 saddr, u32 tos,
1385                            struct rtable **result)
1386 {
1387         struct fib_nh_exception *fnhe;
1388         struct rtable *rth;
1389         int err;
1390         struct in_device *out_dev;
1391         unsigned int flags = 0;
1392         u32 itag;
1393
1394         /* get a working reference to the output device */
1395         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1396         if (out_dev == NULL) {
1397                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1398                 return -EINVAL;
1399         }
1400
1401
1402         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1403                                   in_dev->dev, in_dev, &itag);
1404         if (err < 0) {
1405                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1406                                          saddr);
1407
1408                 goto cleanup;
1409         }
1410
1411         if (err)
1412                 flags |= RTCF_DIRECTSRC;
1413
1414         if (out_dev == in_dev && err &&
1415             (IN_DEV_SHARED_MEDIA(out_dev) ||
1416              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1417                 flags |= RTCF_DOREDIRECT;
1418
1419         if (skb->protocol != htons(ETH_P_IP)) {
1420                 /* Not IP (i.e. ARP). Do not create route, if it is
1421                  * invalid for proxy arp. DNAT routes are always valid.
1422                  *
1423                  * Proxy arp feature have been extended to allow, ARP
1424                  * replies back to the same interface, to support
1425                  * Private VLAN switch technologies. See arp.c.
1426                  */
1427                 if (out_dev == in_dev &&
1428                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1429                         err = -EINVAL;
1430                         goto cleanup;
1431                 }
1432         }
1433
1434         fnhe = NULL;
1435         if (res->fi)
1436                 fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1437
1438         rth = rt_dst_alloc(out_dev->dev,
1439                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1440                            IN_DEV_CONF_GET(out_dev, NOXFRM), false);
1441         if (!rth) {
1442                 err = -ENOBUFS;
1443                 goto cleanup;
1444         }
1445
1446         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
1447         rth->rt_flags = flags;
1448         rth->rt_type = res->type;
1449         rth->rt_route_iif = in_dev->dev->ifindex;
1450         rth->rt_iif     = in_dev->dev->ifindex;
1451         rth->rt_oif     = 0;
1452         rth->rt_pmtu    = 0;
1453         rth->rt_gateway = 0;
1454         rth->fi = NULL;
1455
1456         rth->dst.input = ip_forward;
1457         rth->dst.output = ip_output;
1458
1459         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1460
1461         *result = rth;
1462         err = 0;
1463  cleanup:
1464         return err;
1465 }
1466
1467 static int ip_mkroute_input(struct sk_buff *skb,
1468                             struct fib_result *res,
1469                             const struct flowi4 *fl4,
1470                             struct in_device *in_dev,
1471                             __be32 daddr, __be32 saddr, u32 tos)
1472 {
1473         struct rtable *rth = NULL;
1474         int err;
1475
1476 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1477         if (res->fi && res->fi->fib_nhs > 1)
1478                 fib_select_multipath(res);
1479 #endif
1480
1481         /* create a routing cache entry */
1482         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
1483         if (err)
1484                 return err;
1485
1486         skb_dst_set(skb, &rth->dst);
1487         return 0;
1488 }
1489
1490 /*
1491  *      NOTE. We drop all the packets that has local source
1492  *      addresses, because every properly looped back packet
1493  *      must have correct destination already attached by output routine.
1494  *
1495  *      Such approach solves two big problems:
1496  *      1. Not simplex devices are handled properly.
1497  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1498  *      called with rcu_read_lock()
1499  */
1500
1501 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1502                                u8 tos, struct net_device *dev)
1503 {
1504         struct fib_result res;
1505         struct in_device *in_dev = __in_dev_get_rcu(dev);
1506         struct flowi4   fl4;
1507         unsigned int    flags = 0;
1508         u32             itag = 0;
1509         struct rtable   *rth;
1510         int             err = -EINVAL;
1511         struct net    *net = dev_net(dev);
1512
1513         /* IP on this device is disabled. */
1514
1515         if (!in_dev)
1516                 goto out;
1517
1518         /* Check for the most weird martians, which can be not detected
1519            by fib_lookup.
1520          */
1521
1522         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1523                 goto martian_source;
1524
1525         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1526                 goto brd_input;
1527
1528         /* Accept zero addresses only to limited broadcast;
1529          * I even do not know to fix it or not. Waiting for complains :-)
1530          */
1531         if (ipv4_is_zeronet(saddr))
1532                 goto martian_source;
1533
1534         if (ipv4_is_zeronet(daddr))
1535                 goto martian_destination;
1536
1537         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) {
1538                 if (ipv4_is_loopback(daddr))
1539                         goto martian_destination;
1540
1541                 if (ipv4_is_loopback(saddr))
1542                         goto martian_source;
1543         }
1544
1545         /*
1546          *      Now we are ready to route packet.
1547          */
1548         fl4.flowi4_oif = 0;
1549         fl4.flowi4_iif = dev->ifindex;
1550         fl4.flowi4_mark = skb->mark;
1551         fl4.flowi4_tos = tos;
1552         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1553         fl4.daddr = daddr;
1554         fl4.saddr = saddr;
1555         err = fib_lookup(net, &fl4, &res);
1556         if (err != 0)
1557                 goto no_route;
1558
1559         RT_CACHE_STAT_INC(in_slow_tot);
1560
1561         if (res.type == RTN_BROADCAST)
1562                 goto brd_input;
1563
1564         if (res.type == RTN_LOCAL) {
1565                 err = fib_validate_source(skb, saddr, daddr, tos,
1566                                           net->loopback_dev->ifindex,
1567                                           dev, in_dev, &itag);
1568                 if (err < 0)
1569                         goto martian_source_keep_err;
1570                 if (err)
1571                         flags |= RTCF_DIRECTSRC;
1572                 goto local_input;
1573         }
1574
1575         if (!IN_DEV_FORWARD(in_dev))
1576                 goto no_route;
1577         if (res.type != RTN_UNICAST)
1578                 goto martian_destination;
1579
1580         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1581 out:    return err;
1582
1583 brd_input:
1584         if (skb->protocol != htons(ETH_P_IP))
1585                 goto e_inval;
1586
1587         if (!ipv4_is_zeronet(saddr)) {
1588                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1589                                           in_dev, &itag);
1590                 if (err < 0)
1591                         goto martian_source_keep_err;
1592                 if (err)
1593                         flags |= RTCF_DIRECTSRC;
1594         }
1595         flags |= RTCF_BROADCAST;
1596         res.type = RTN_BROADCAST;
1597         RT_CACHE_STAT_INC(in_brd);
1598
1599 local_input:
1600         rth = rt_dst_alloc(net->loopback_dev,
1601                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1602         if (!rth)
1603                 goto e_nobufs;
1604
1605         rth->dst.input= ip_local_deliver;
1606         rth->dst.output= ip_rt_bug;
1607 #ifdef CONFIG_IP_ROUTE_CLASSID
1608         rth->dst.tclassid = itag;
1609 #endif
1610
1611         rth->rt_genid = rt_genid(net);
1612         rth->rt_flags   = flags|RTCF_LOCAL;
1613         rth->rt_type    = res.type;
1614         rth->rt_route_iif = dev->ifindex;
1615         rth->rt_iif     = dev->ifindex;
1616         rth->rt_oif     = 0;
1617         rth->rt_pmtu    = 0;
1618         rth->rt_gateway = 0;
1619         rth->fi = NULL;
1620         if (res.type == RTN_UNREACHABLE) {
1621                 rth->dst.input= ip_error;
1622                 rth->dst.error= -err;
1623                 rth->rt_flags   &= ~RTCF_LOCAL;
1624         }
1625         skb_dst_set(skb, &rth->dst);
1626         err = 0;
1627         goto out;
1628
1629 no_route:
1630         RT_CACHE_STAT_INC(in_no_route);
1631         res.type = RTN_UNREACHABLE;
1632         if (err == -ESRCH)
1633                 err = -ENETUNREACH;
1634         goto local_input;
1635
1636         /*
1637          *      Do not cache martian addresses: they should be logged (RFC1812)
1638          */
1639 martian_destination:
1640         RT_CACHE_STAT_INC(in_martian_dst);
1641 #ifdef CONFIG_IP_ROUTE_VERBOSE
1642         if (IN_DEV_LOG_MARTIANS(in_dev))
1643                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1644                                      &daddr, &saddr, dev->name);
1645 #endif
1646
1647 e_inval:
1648         err = -EINVAL;
1649         goto out;
1650
1651 e_nobufs:
1652         err = -ENOBUFS;
1653         goto out;
1654
1655 martian_source:
1656         err = -EINVAL;
1657 martian_source_keep_err:
1658         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1659         goto out;
1660 }
1661
1662 int ip_route_input(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1663                    u8 tos, struct net_device *dev)
1664 {
1665         int res;
1666
1667         rcu_read_lock();
1668
1669         /* Multicast recognition logic is moved from route cache to here.
1670            The problem was that too many Ethernet cards have broken/missing
1671            hardware multicast filters :-( As result the host on multicasting
1672            network acquires a lot of useless route cache entries, sort of
1673            SDR messages from all the world. Now we try to get rid of them.
1674            Really, provided software IP multicast filter is organized
1675            reasonably (at least, hashed), it does not result in a slowdown
1676            comparing with route cache reject entries.
1677            Note, that multicast routers are not affected, because
1678            route cache entry is created eventually.
1679          */
1680         if (ipv4_is_multicast(daddr)) {
1681                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1682
1683                 if (in_dev) {
1684                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1685                                                   ip_hdr(skb)->protocol);
1686                         if (our
1687 #ifdef CONFIG_IP_MROUTE
1688                                 ||
1689                             (!ipv4_is_local_multicast(daddr) &&
1690                              IN_DEV_MFORWARD(in_dev))
1691 #endif
1692                            ) {
1693                                 int res = ip_route_input_mc(skb, daddr, saddr,
1694                                                             tos, dev, our);
1695                                 rcu_read_unlock();
1696                                 return res;
1697                         }
1698                 }
1699                 rcu_read_unlock();
1700                 return -EINVAL;
1701         }
1702         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1703         rcu_read_unlock();
1704         return res;
1705 }
1706 EXPORT_SYMBOL(ip_route_input);
1707
1708 /* called with rcu_read_lock() */
1709 static struct rtable *__mkroute_output(const struct fib_result *res,
1710                                        const struct flowi4 *fl4, int orig_oif,
1711                                        struct net_device *dev_out,
1712                                        unsigned int flags)
1713 {
1714         struct fib_info *fi = res->fi;
1715         struct fib_nh_exception *fnhe;
1716         struct in_device *in_dev;
1717         u16 type = res->type;
1718         struct rtable *rth;
1719
1720         in_dev = __in_dev_get_rcu(dev_out);
1721         if (!in_dev)
1722                 return ERR_PTR(-EINVAL);
1723
1724         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1725                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
1726                         return ERR_PTR(-EINVAL);
1727
1728         if (ipv4_is_lbcast(fl4->daddr))
1729                 type = RTN_BROADCAST;
1730         else if (ipv4_is_multicast(fl4->daddr))
1731                 type = RTN_MULTICAST;
1732         else if (ipv4_is_zeronet(fl4->daddr))
1733                 return ERR_PTR(-EINVAL);
1734
1735         if (dev_out->flags & IFF_LOOPBACK)
1736                 flags |= RTCF_LOCAL;
1737
1738         if (type == RTN_BROADCAST) {
1739                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
1740                 fi = NULL;
1741         } else if (type == RTN_MULTICAST) {
1742                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
1743                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
1744                                      fl4->flowi4_proto))
1745                         flags &= ~RTCF_LOCAL;
1746                 /* If multicast route do not exist use
1747                  * default one, but do not gateway in this case.
1748                  * Yes, it is hack.
1749                  */
1750                 if (fi && res->prefixlen < 4)
1751                         fi = NULL;
1752         }
1753
1754         fnhe = NULL;
1755         if (fi) {
1756                 fnhe = find_exception(&FIB_RES_NH(*res), fl4->daddr);
1757                 if (!fnhe) {
1758                         rth = FIB_RES_NH(*res).nh_rth_output;
1759                         if (rth &&
1760                             rth->dst.obsolete == DST_OBSOLETE_FORCE_CHK) {
1761                                 dst_use(&rth->dst, jiffies);
1762                                 return rth;
1763                         }
1764                 }
1765         }
1766         rth = rt_dst_alloc(dev_out,
1767                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1768                            IN_DEV_CONF_GET(in_dev, NOXFRM),
1769                            fi && !fnhe);
1770         if (!rth)
1771                 return ERR_PTR(-ENOBUFS);
1772
1773         rth->dst.output = ip_output;
1774
1775         rth->rt_genid = rt_genid(dev_net(dev_out));
1776         rth->rt_flags   = flags;
1777         rth->rt_type    = type;
1778         rth->rt_route_iif = 0;
1779         rth->rt_iif     = orig_oif ? : dev_out->ifindex;
1780         rth->rt_oif     = orig_oif;
1781         rth->rt_pmtu    = 0;
1782         rth->rt_gateway = 0;
1783         rth->fi = NULL;
1784
1785         RT_CACHE_STAT_INC(out_slow_tot);
1786
1787         if (flags & RTCF_LOCAL)
1788                 rth->dst.input = ip_local_deliver;
1789         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
1790                 if (flags & RTCF_LOCAL &&
1791                     !(dev_out->flags & IFF_LOOPBACK)) {
1792                         rth->dst.output = ip_mc_output;
1793                         RT_CACHE_STAT_INC(out_slow_mc);
1794                 }
1795 #ifdef CONFIG_IP_MROUTE
1796                 if (type == RTN_MULTICAST) {
1797                         if (IN_DEV_MFORWARD(in_dev) &&
1798                             !ipv4_is_local_multicast(fl4->daddr)) {
1799                                 rth->dst.input = ip_mr_input;
1800                                 rth->dst.output = ip_mc_output;
1801                         }
1802                 }
1803 #endif
1804         }
1805
1806         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
1807
1808         if (fl4->flowi4_flags & FLOWI_FLAG_RT_NOCACHE)
1809                 rth->dst.flags |= DST_NOCACHE;
1810
1811         return rth;
1812 }
1813
1814 /*
1815  * Major route resolver routine.
1816  */
1817
1818 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *fl4)
1819 {
1820         struct net_device *dev_out = NULL;
1821         __u8 tos = RT_FL_TOS(fl4);
1822         unsigned int flags = 0;
1823         struct fib_result res;
1824         struct rtable *rth;
1825         int orig_oif;
1826
1827         res.tclassid    = 0;
1828         res.fi          = NULL;
1829         res.table       = NULL;
1830
1831         orig_oif = fl4->flowi4_oif;
1832
1833         fl4->flowi4_iif = net->loopback_dev->ifindex;
1834         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
1835         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
1836                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
1837
1838         rcu_read_lock();
1839         if (fl4->saddr) {
1840                 rth = ERR_PTR(-EINVAL);
1841                 if (ipv4_is_multicast(fl4->saddr) ||
1842                     ipv4_is_lbcast(fl4->saddr) ||
1843                     ipv4_is_zeronet(fl4->saddr))
1844                         goto out;
1845
1846                 /* I removed check for oif == dev_out->oif here.
1847                    It was wrong for two reasons:
1848                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
1849                       is assigned to multiple interfaces.
1850                    2. Moreover, we are allowed to send packets with saddr
1851                       of another iface. --ANK
1852                  */
1853
1854                 if (fl4->flowi4_oif == 0 &&
1855                     (ipv4_is_multicast(fl4->daddr) ||
1856                      ipv4_is_lbcast(fl4->daddr))) {
1857                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1858                         dev_out = __ip_dev_find(net, fl4->saddr, false);
1859                         if (dev_out == NULL)
1860                                 goto out;
1861
1862                         /* Special hack: user can direct multicasts
1863                            and limited broadcast via necessary interface
1864                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
1865                            This hack is not just for fun, it allows
1866                            vic,vat and friends to work.
1867                            They bind socket to loopback, set ttl to zero
1868                            and expect that it will work.
1869                            From the viewpoint of routing cache they are broken,
1870                            because we are not allowed to build multicast path
1871                            with loopback source addr (look, routing cache
1872                            cannot know, that ttl is zero, so that packet
1873                            will not leave this host and route is valid).
1874                            Luckily, this hack is good workaround.
1875                          */
1876
1877                         fl4->flowi4_oif = dev_out->ifindex;
1878                         goto make_route;
1879                 }
1880
1881                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
1882                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
1883                         if (!__ip_dev_find(net, fl4->saddr, false))
1884                                 goto out;
1885                 }
1886         }
1887
1888
1889         if (fl4->flowi4_oif) {
1890                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
1891                 rth = ERR_PTR(-ENODEV);
1892                 if (dev_out == NULL)
1893                         goto out;
1894
1895                 /* RACE: Check return value of inet_select_addr instead. */
1896                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
1897                         rth = ERR_PTR(-ENETUNREACH);
1898                         goto out;
1899                 }
1900                 if (ipv4_is_local_multicast(fl4->daddr) ||
1901                     ipv4_is_lbcast(fl4->daddr)) {
1902                         if (!fl4->saddr)
1903                                 fl4->saddr = inet_select_addr(dev_out, 0,
1904                                                               RT_SCOPE_LINK);
1905                         goto make_route;
1906                 }
1907                 if (fl4->saddr) {
1908                         if (ipv4_is_multicast(fl4->daddr))
1909                                 fl4->saddr = inet_select_addr(dev_out, 0,
1910                                                               fl4->flowi4_scope);
1911                         else if (!fl4->daddr)
1912                                 fl4->saddr = inet_select_addr(dev_out, 0,
1913                                                               RT_SCOPE_HOST);
1914                 }
1915         }
1916
1917         if (!fl4->daddr) {
1918                 fl4->daddr = fl4->saddr;
1919                 if (!fl4->daddr)
1920                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
1921                 dev_out = net->loopback_dev;
1922                 fl4->flowi4_oif = net->loopback_dev->ifindex;
1923                 res.type = RTN_LOCAL;
1924                 flags |= RTCF_LOCAL;
1925                 goto make_route;
1926         }
1927
1928         if (fib_lookup(net, fl4, &res)) {
1929                 res.fi = NULL;
1930                 res.table = NULL;
1931                 if (fl4->flowi4_oif) {
1932                         /* Apparently, routing tables are wrong. Assume,
1933                            that the destination is on link.
1934
1935                            WHY? DW.
1936                            Because we are allowed to send to iface
1937                            even if it has NO routes and NO assigned
1938                            addresses. When oif is specified, routing
1939                            tables are looked up with only one purpose:
1940                            to catch if destination is gatewayed, rather than
1941                            direct. Moreover, if MSG_DONTROUTE is set,
1942                            we send packet, ignoring both routing tables
1943                            and ifaddr state. --ANK
1944
1945
1946                            We could make it even if oif is unknown,
1947                            likely IPv6, but we do not.
1948                          */
1949
1950                         if (fl4->saddr == 0)
1951                                 fl4->saddr = inet_select_addr(dev_out, 0,
1952                                                               RT_SCOPE_LINK);
1953                         res.type = RTN_UNICAST;
1954                         goto make_route;
1955                 }
1956                 rth = ERR_PTR(-ENETUNREACH);
1957                 goto out;
1958         }
1959
1960         if (res.type == RTN_LOCAL) {
1961                 if (!fl4->saddr) {
1962                         if (res.fi->fib_prefsrc)
1963                                 fl4->saddr = res.fi->fib_prefsrc;
1964                         else
1965                                 fl4->saddr = fl4->daddr;
1966                 }
1967                 dev_out = net->loopback_dev;
1968                 fl4->flowi4_oif = dev_out->ifindex;
1969                 res.fi = NULL;
1970                 flags |= RTCF_LOCAL;
1971                 goto make_route;
1972         }
1973
1974 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1975         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
1976                 fib_select_multipath(&res);
1977         else
1978 #endif
1979         if (!res.prefixlen &&
1980             res.table->tb_num_default > 1 &&
1981             res.type == RTN_UNICAST && !fl4->flowi4_oif)
1982                 fib_select_default(&res);
1983
1984         if (!fl4->saddr)
1985                 fl4->saddr = FIB_RES_PREFSRC(net, res);
1986
1987         dev_out = FIB_RES_DEV(res);
1988         fl4->flowi4_oif = dev_out->ifindex;
1989
1990
1991 make_route:
1992         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
1993
1994 out:
1995         rcu_read_unlock();
1996         return rth;
1997 }
1998 EXPORT_SYMBOL_GPL(__ip_route_output_key);
1999
2000 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2001 {
2002         return NULL;
2003 }
2004
2005 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2006 {
2007         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2008
2009         return mtu ? : dst->dev->mtu;
2010 }
2011
2012 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2013                                           struct sk_buff *skb, u32 mtu)
2014 {
2015 }
2016
2017 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2018                                        struct sk_buff *skb)
2019 {
2020 }
2021
2022 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2023                                           unsigned long old)
2024 {
2025         return NULL;
2026 }
2027
2028 static struct dst_ops ipv4_dst_blackhole_ops = {
2029         .family                 =       AF_INET,
2030         .protocol               =       cpu_to_be16(ETH_P_IP),
2031         .destroy                =       ipv4_dst_destroy,
2032         .check                  =       ipv4_blackhole_dst_check,
2033         .mtu                    =       ipv4_blackhole_mtu,
2034         .default_advmss         =       ipv4_default_advmss,
2035         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2036         .redirect               =       ipv4_rt_blackhole_redirect,
2037         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2038         .neigh_lookup           =       ipv4_neigh_lookup,
2039 };
2040
2041 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2042 {
2043         struct rtable *ort = (struct rtable *) dst_orig;
2044         struct rtable *rt;
2045
2046         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2047         if (rt) {
2048                 struct dst_entry *new = &rt->dst;
2049
2050                 new->__use = 1;
2051                 new->input = dst_discard;
2052                 new->output = dst_discard;
2053
2054                 new->dev = ort->dst.dev;
2055                 if (new->dev)
2056                         dev_hold(new->dev);
2057
2058                 rt->rt_route_iif = ort->rt_route_iif;
2059                 rt->rt_iif = ort->rt_iif;
2060                 rt->rt_oif = ort->rt_oif;
2061                 rt->rt_pmtu = ort->rt_pmtu;
2062
2063                 rt->rt_genid = rt_genid(net);
2064                 rt->rt_flags = ort->rt_flags;
2065                 rt->rt_type = ort->rt_type;
2066                 rt->rt_gateway = ort->rt_gateway;
2067                 rt->fi = ort->fi;
2068                 if (rt->fi)
2069                         atomic_inc(&rt->fi->fib_clntref);
2070
2071                 dst_free(new);
2072         }
2073
2074         dst_release(dst_orig);
2075
2076         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2077 }
2078
2079 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2080                                     struct sock *sk)
2081 {
2082         struct rtable *rt = __ip_route_output_key(net, flp4);
2083
2084         if (IS_ERR(rt))
2085                 return rt;
2086
2087         if (flp4->flowi4_proto)
2088                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2089                                                    flowi4_to_flowi(flp4),
2090                                                    sk, 0);
2091
2092         return rt;
2093 }
2094 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2095
2096 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src,
2097                         struct flowi4 *fl4, struct sk_buff *skb, u32 pid,
2098                         u32 seq, int event, int nowait, unsigned int flags)
2099 {
2100         struct rtable *rt = skb_rtable(skb);
2101         struct rtmsg *r;
2102         struct nlmsghdr *nlh;
2103         unsigned long expires = 0;
2104         u32 error;
2105         u32 metrics[RTAX_MAX];
2106
2107         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2108         if (nlh == NULL)
2109                 return -EMSGSIZE;
2110
2111         r = nlmsg_data(nlh);
2112         r->rtm_family    = AF_INET;
2113         r->rtm_dst_len  = 32;
2114         r->rtm_src_len  = 0;
2115         r->rtm_tos      = fl4->flowi4_tos;
2116         r->rtm_table    = RT_TABLE_MAIN;
2117         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2118                 goto nla_put_failure;
2119         r->rtm_type     = rt->rt_type;
2120         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2121         r->rtm_protocol = RTPROT_UNSPEC;
2122         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2123         if (rt->rt_flags & RTCF_NOTIFY)
2124                 r->rtm_flags |= RTM_F_NOTIFY;
2125
2126         if (nla_put_be32(skb, RTA_DST, dst))
2127                 goto nla_put_failure;
2128         if (src) {
2129                 r->rtm_src_len = 32;
2130                 if (nla_put_be32(skb, RTA_SRC, src))
2131                         goto nla_put_failure;
2132         }
2133         if (rt->dst.dev &&
2134             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2135                 goto nla_put_failure;
2136 #ifdef CONFIG_IP_ROUTE_CLASSID
2137         if (rt->dst.tclassid &&
2138             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2139                 goto nla_put_failure;
2140 #endif
2141         if (!rt_is_input_route(rt) &&
2142             fl4->saddr != src) {
2143                 if (nla_put_be32(skb, RTA_PREFSRC, fl4->saddr))
2144                         goto nla_put_failure;
2145         }
2146         if (rt->rt_gateway &&
2147             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2148                 goto nla_put_failure;
2149
2150         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2151         if (rt->rt_pmtu)
2152                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2153         if (rtnetlink_put_metrics(skb, metrics) < 0)
2154                 goto nla_put_failure;
2155
2156         if (fl4->flowi4_mark &&
2157             nla_put_be32(skb, RTA_MARK, fl4->flowi4_mark))
2158                 goto nla_put_failure;
2159
2160         error = rt->dst.error;
2161         expires = rt->dst.expires;
2162         if (expires) {
2163                 if (time_before(jiffies, expires))
2164                         expires -= jiffies;
2165                 else
2166                         expires = 0;
2167         }
2168
2169         if (rt_is_input_route(rt)) {
2170                 if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2171                         goto nla_put_failure;
2172         }
2173
2174         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2175                 goto nla_put_failure;
2176
2177         return nlmsg_end(skb, nlh);
2178
2179 nla_put_failure:
2180         nlmsg_cancel(skb, nlh);
2181         return -EMSGSIZE;
2182 }
2183
2184 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, void *arg)
2185 {
2186         struct net *net = sock_net(in_skb->sk);
2187         struct rtmsg *rtm;
2188         struct nlattr *tb[RTA_MAX+1];
2189         struct rtable *rt = NULL;
2190         struct flowi4 fl4;
2191         __be32 dst = 0;
2192         __be32 src = 0;
2193         u32 iif;
2194         int err;
2195         int mark;
2196         struct sk_buff *skb;
2197
2198         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2199         if (err < 0)
2200                 goto errout;
2201
2202         rtm = nlmsg_data(nlh);
2203
2204         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2205         if (skb == NULL) {
2206                 err = -ENOBUFS;
2207                 goto errout;
2208         }
2209
2210         /* Reserve room for dummy headers, this skb can pass
2211            through good chunk of routing engine.
2212          */
2213         skb_reset_mac_header(skb);
2214         skb_reset_network_header(skb);
2215
2216         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2217         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2218         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2219
2220         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2221         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2222         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2223         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2224
2225         memset(&fl4, 0, sizeof(fl4));
2226         fl4.daddr = dst;
2227         fl4.saddr = src;
2228         fl4.flowi4_tos = rtm->rtm_tos;
2229         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2230         fl4.flowi4_mark = mark;
2231
2232         if (iif) {
2233                 struct net_device *dev;
2234
2235                 dev = __dev_get_by_index(net, iif);
2236                 if (dev == NULL) {
2237                         err = -ENODEV;
2238                         goto errout_free;
2239                 }
2240
2241                 skb->protocol   = htons(ETH_P_IP);
2242                 skb->dev        = dev;
2243                 skb->mark       = mark;
2244                 local_bh_disable();
2245                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2246                 local_bh_enable();
2247
2248                 rt = skb_rtable(skb);
2249                 if (err == 0 && rt->dst.error)
2250                         err = -rt->dst.error;
2251         } else {
2252                 rt = ip_route_output_key(net, &fl4);
2253
2254                 err = 0;
2255                 if (IS_ERR(rt))
2256                         err = PTR_ERR(rt);
2257         }
2258
2259         if (err)
2260                 goto errout_free;
2261
2262         skb_dst_set(skb, &rt->dst);
2263         if (rtm->rtm_flags & RTM_F_NOTIFY)
2264                 rt->rt_flags |= RTCF_NOTIFY;
2265
2266         err = rt_fill_info(net, dst, src, &fl4, skb,
2267                            NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2268                            RTM_NEWROUTE, 0, 0);
2269         if (err <= 0)
2270                 goto errout_free;
2271
2272         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2273 errout:
2274         return err;
2275
2276 errout_free:
2277         kfree_skb(skb);
2278         goto errout;
2279 }
2280
2281 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2282 {
2283         return skb->len;
2284 }
2285
2286 void ip_rt_multicast_event(struct in_device *in_dev)
2287 {
2288         rt_cache_flush(dev_net(in_dev->dev), 0);
2289 }
2290
2291 #ifdef CONFIG_SYSCTL
2292 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
2293                                         void __user *buffer,
2294                                         size_t *lenp, loff_t *ppos)
2295 {
2296         if (write) {
2297                 int flush_delay;
2298                 ctl_table ctl;
2299                 struct net *net;
2300
2301                 memcpy(&ctl, __ctl, sizeof(ctl));
2302                 ctl.data = &flush_delay;
2303                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
2304
2305                 net = (struct net *)__ctl->extra1;
2306                 rt_cache_flush(net, flush_delay);
2307                 return 0;
2308         }
2309
2310         return -EINVAL;
2311 }
2312
2313 static ctl_table ipv4_route_table[] = {
2314         {
2315                 .procname       = "gc_thresh",
2316                 .data           = &ipv4_dst_ops.gc_thresh,
2317                 .maxlen         = sizeof(int),
2318                 .mode           = 0644,
2319                 .proc_handler   = proc_dointvec,
2320         },
2321         {
2322                 .procname       = "max_size",
2323                 .data           = &ip_rt_max_size,
2324                 .maxlen         = sizeof(int),
2325                 .mode           = 0644,
2326                 .proc_handler   = proc_dointvec,
2327         },
2328         {
2329                 /*  Deprecated. Use gc_min_interval_ms */
2330
2331                 .procname       = "gc_min_interval",
2332                 .data           = &ip_rt_gc_min_interval,
2333                 .maxlen         = sizeof(int),
2334                 .mode           = 0644,
2335                 .proc_handler   = proc_dointvec_jiffies,
2336         },
2337         {
2338                 .procname       = "gc_min_interval_ms",
2339                 .data           = &ip_rt_gc_min_interval,
2340                 .maxlen         = sizeof(int),
2341                 .mode           = 0644,
2342                 .proc_handler   = proc_dointvec_ms_jiffies,
2343         },
2344         {
2345                 .procname       = "gc_timeout",
2346                 .data           = &ip_rt_gc_timeout,
2347                 .maxlen         = sizeof(int),
2348                 .mode           = 0644,
2349                 .proc_handler   = proc_dointvec_jiffies,
2350         },
2351         {
2352                 .procname       = "gc_interval",
2353                 .data           = &ip_rt_gc_interval,
2354                 .maxlen         = sizeof(int),
2355                 .mode           = 0644,
2356                 .proc_handler   = proc_dointvec_jiffies,
2357         },
2358         {
2359                 .procname       = "redirect_load",
2360                 .data           = &ip_rt_redirect_load,
2361                 .maxlen         = sizeof(int),
2362                 .mode           = 0644,
2363                 .proc_handler   = proc_dointvec,
2364         },
2365         {
2366                 .procname       = "redirect_number",
2367                 .data           = &ip_rt_redirect_number,
2368                 .maxlen         = sizeof(int),
2369                 .mode           = 0644,
2370                 .proc_handler   = proc_dointvec,
2371         },
2372         {
2373                 .procname       = "redirect_silence",
2374                 .data           = &ip_rt_redirect_silence,
2375                 .maxlen         = sizeof(int),
2376                 .mode           = 0644,
2377                 .proc_handler   = proc_dointvec,
2378         },
2379         {
2380                 .procname       = "error_cost",
2381                 .data           = &ip_rt_error_cost,
2382                 .maxlen         = sizeof(int),
2383                 .mode           = 0644,
2384                 .proc_handler   = proc_dointvec,
2385         },
2386         {
2387                 .procname       = "error_burst",
2388                 .data           = &ip_rt_error_burst,
2389                 .maxlen         = sizeof(int),
2390                 .mode           = 0644,
2391                 .proc_handler   = proc_dointvec,
2392         },
2393         {
2394                 .procname       = "gc_elasticity",
2395                 .data           = &ip_rt_gc_elasticity,
2396                 .maxlen         = sizeof(int),
2397                 .mode           = 0644,
2398                 .proc_handler   = proc_dointvec,
2399         },
2400         {
2401                 .procname       = "mtu_expires",
2402                 .data           = &ip_rt_mtu_expires,
2403                 .maxlen         = sizeof(int),
2404                 .mode           = 0644,
2405                 .proc_handler   = proc_dointvec_jiffies,
2406         },
2407         {
2408                 .procname       = "min_pmtu",
2409                 .data           = &ip_rt_min_pmtu,
2410                 .maxlen         = sizeof(int),
2411                 .mode           = 0644,
2412                 .proc_handler   = proc_dointvec,
2413         },
2414         {
2415                 .procname       = "min_adv_mss",
2416                 .data           = &ip_rt_min_advmss,
2417                 .maxlen         = sizeof(int),
2418                 .mode           = 0644,
2419                 .proc_handler   = proc_dointvec,
2420         },
2421         { }
2422 };
2423
2424 static struct ctl_table ipv4_route_flush_table[] = {
2425         {
2426                 .procname       = "flush",
2427                 .maxlen         = sizeof(int),
2428                 .mode           = 0200,
2429                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2430         },
2431         { },
2432 };
2433
2434 static __net_init int sysctl_route_net_init(struct net *net)
2435 {
2436         struct ctl_table *tbl;
2437
2438         tbl = ipv4_route_flush_table;
2439         if (!net_eq(net, &init_net)) {
2440                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2441                 if (tbl == NULL)
2442                         goto err_dup;
2443         }
2444         tbl[0].extra1 = net;
2445
2446         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2447         if (net->ipv4.route_hdr == NULL)
2448                 goto err_reg;
2449         return 0;
2450
2451 err_reg:
2452         if (tbl != ipv4_route_flush_table)
2453                 kfree(tbl);
2454 err_dup:
2455         return -ENOMEM;
2456 }
2457
2458 static __net_exit void sysctl_route_net_exit(struct net *net)
2459 {
2460         struct ctl_table *tbl;
2461
2462         tbl = net->ipv4.route_hdr->ctl_table_arg;
2463         unregister_net_sysctl_table(net->ipv4.route_hdr);
2464         BUG_ON(tbl == ipv4_route_flush_table);
2465         kfree(tbl);
2466 }
2467
2468 static __net_initdata struct pernet_operations sysctl_route_ops = {
2469         .init = sysctl_route_net_init,
2470         .exit = sysctl_route_net_exit,
2471 };
2472 #endif
2473
2474 static __net_init int rt_genid_init(struct net *net)
2475 {
2476         get_random_bytes(&net->ipv4.rt_genid,
2477                          sizeof(net->ipv4.rt_genid));
2478         get_random_bytes(&net->ipv4.dev_addr_genid,
2479                          sizeof(net->ipv4.dev_addr_genid));
2480         return 0;
2481 }
2482
2483 static __net_initdata struct pernet_operations rt_genid_ops = {
2484         .init = rt_genid_init,
2485 };
2486
2487 static int __net_init ipv4_inetpeer_init(struct net *net)
2488 {
2489         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2490
2491         if (!bp)
2492                 return -ENOMEM;
2493         inet_peer_base_init(bp);
2494         net->ipv4.peers = bp;
2495         return 0;
2496 }
2497
2498 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2499 {
2500         struct inet_peer_base *bp = net->ipv4.peers;
2501
2502         net->ipv4.peers = NULL;
2503         inetpeer_invalidate_tree(bp);
2504         kfree(bp);
2505 }
2506
2507 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2508         .init   =       ipv4_inetpeer_init,
2509         .exit   =       ipv4_inetpeer_exit,
2510 };
2511
2512 #ifdef CONFIG_IP_ROUTE_CLASSID
2513 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2514 #endif /* CONFIG_IP_ROUTE_CLASSID */
2515
2516 int __init ip_rt_init(void)
2517 {
2518         int rc = 0;
2519
2520 #ifdef CONFIG_IP_ROUTE_CLASSID
2521         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2522         if (!ip_rt_acct)
2523                 panic("IP: failed to allocate ip_rt_acct\n");
2524 #endif
2525
2526         ipv4_dst_ops.kmem_cachep =
2527                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2528                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2529
2530         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2531
2532         if (dst_entries_init(&ipv4_dst_ops) < 0)
2533                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2534
2535         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2536                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2537
2538         ipv4_dst_ops.gc_thresh = ~0;
2539         ip_rt_max_size = INT_MAX;
2540
2541         devinet_init();
2542         ip_fib_init();
2543
2544         if (ip_rt_proc_init())
2545                 pr_err("Unable to create route proc files\n");
2546 #ifdef CONFIG_XFRM
2547         xfrm_init();
2548         xfrm4_init(ip_rt_max_size);
2549 #endif
2550         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2551
2552 #ifdef CONFIG_SYSCTL
2553         register_pernet_subsys(&sysctl_route_ops);
2554 #endif
2555         register_pernet_subsys(&rt_genid_ops);
2556         register_pernet_subsys(&ipv4_inetpeer_ops);
2557         return rc;
2558 }
2559
2560 #ifdef CONFIG_SYSCTL
2561 /*
2562  * We really need to sanitize the damn ipv4 init order, then all
2563  * this nonsense will go away.
2564  */
2565 void __init ip_static_sysctl_init(void)
2566 {
2567         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2568 }
2569 #endif