net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/bootmem.h>
  74 #include <linux/string.h>
  75 #include <linux/socket.h>
  76 #include <linux/sockios.h>
  77 #include <linux/errno.h>
  78 #include <linux/in.h>
  79 #include <linux/inet.h>
  80 #include <linux/netdevice.h>
  81 #include <linux/proc_fs.h>
  82 #include <linux/init.h>
  83 #include <linux/workqueue.h>
  84 #include <linux/skbuff.h>
  85 #include <linux/inetdevice.h>
  86 #include <linux/igmp.h>
  87 #include <linux/pkt_sched.h>
  88 #include <linux/mroute.h>
  89 #include <linux/netfilter_ipv4.h>
  90 #include <linux/random.h>
  91 #include <linux/jhash.h>
  92 #include <linux/rcupdate.h>
  93 #include <linux/times.h>
  94 #include <linux/slab.h>
  95 #include <linux/prefetch.h>
  96 #include <net/dst.h>
  97 #include <net/net_namespace.h>
  98 #include <net/protocol.h>
  99 #include <net/ip.h>
 100 #include <net/route.h>
 101 #include <net/inetpeer.h>
 102 #include <net/sock.h>
 103 #include <net/ip_fib.h>
 104 #include <net/arp.h>
 105 #include <net/tcp.h>
 106 #include <net/icmp.h>
 107 #include <net/xfrm.h>
 108 #include <net/netevent.h>
 109 #include <net/rtnetlink.h>
 110 #ifdef CONFIG_SYSCTL
 111 #include <linux/sysctl.h>
 112 #include <linux/kmemleak.h>
 113 #endif
 114 #include <net/secure_seq.h>
 115
 116 #define RT_FL_TOS(oldflp4) \
 117         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 118
 119 #define IP_MAX_MTU      0xFFF0
 120
 121 #define RT_GC_TIMEOUT (300*HZ)
 122
 123 static int ip_rt_max_size;
 124 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 125 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
 126 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
 127 static int ip_rt_redirect_number __read_mostly  = 9;
 128 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 129 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 130 static int ip_rt_error_cost __read_mostly       = HZ;
 131 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 132 static int ip_rt_gc_elasticity __read_mostly    = 8;
 133 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 134 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 135 static int ip_rt_min_advmss __read_mostly       = 256;
 136 static int rt_chain_length_max __read_mostly    = 20;
 137
 138 static struct delayed_work expires_work;
 139 static unsigned long expires_ljiffies;
 140
 141 /*
 142  *      Interface to generic destination cache.
 143  */
 144
 145 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 146 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 147 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 148 static void              ipv4_dst_destroy(struct dst_entry *dst);
 149 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 150 static void              ipv4_link_failure(struct sk_buff *skb);
 151 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
 152 static int rt_garbage_collect(struct dst_ops *ops);
 153
 154 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
 155                             int how)
 156 {
 157 }
 158
 159 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 160 {
 161         struct rtable *rt = (struct rtable *) dst;
 162         struct inet_peer *peer;
 163         u32 *p = NULL;
 164
 165         peer = rt_get_peer_create(rt, rt->rt_dst);
 166         if (peer) {
 167                 u32 *old_p = __DST_METRICS_PTR(old);
 168                 unsigned long prev, new;
 169
 170                 p = peer->metrics;
 171                 if (inet_metrics_new(peer))
 172                         memcpy(p, old_p, sizeof(u32) * RTAX_MAX);
 173
 174                 new = (unsigned long) p;
 175                 prev = cmpxchg(&dst->_metrics, old, new);
 176
 177                 if (prev != old) {
 178                         p = __DST_METRICS_PTR(prev);
 179                         if (prev & DST_METRICS_READ_ONLY)
 180                                 p = NULL;
 181                 } else {
 182                         if (rt->fi) {
 183                                 fib_info_put(rt->fi);
 184                                 rt->fi = NULL;
 185                         }
 186                 }
 187         }
 188         return p;
 189 }
 190
 191 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 192                                            struct sk_buff *skb,
 193                                            const void *daddr);
 194
 195 static struct dst_ops ipv4_dst_ops = {
 196         .family =               AF_INET,
 197         .protocol =             cpu_to_be16(ETH_P_IP),
 198         .gc =                   rt_garbage_collect,
 199         .check =                ipv4_dst_check,
 200         .default_advmss =       ipv4_default_advmss,
 201         .mtu =                  ipv4_mtu,
 202         .cow_metrics =          ipv4_cow_metrics,
 203         .destroy =              ipv4_dst_destroy,
 204         .ifdown =               ipv4_dst_ifdown,
 205         .negative_advice =      ipv4_negative_advice,
 206         .link_failure =         ipv4_link_failure,
 207         .update_pmtu =          ip_rt_update_pmtu,
 208         .local_out =            __ip_local_out,
 209         .neigh_lookup =         ipv4_neigh_lookup,
 210 };
 211
 212 #define ECN_OR_COST(class)      TC_PRIO_##class
 213
 214 const __u8 ip_tos2prio[16] = {
 215         TC_PRIO_BESTEFFORT,
 216         ECN_OR_COST(BESTEFFORT),
 217         TC_PRIO_BESTEFFORT,
 218         ECN_OR_COST(BESTEFFORT),
 219         TC_PRIO_BULK,
 220         ECN_OR_COST(BULK),
 221         TC_PRIO_BULK,
 222         ECN_OR_COST(BULK),
 223         TC_PRIO_INTERACTIVE,
 224         ECN_OR_COST(INTERACTIVE),
 225         TC_PRIO_INTERACTIVE,
 226         ECN_OR_COST(INTERACTIVE),
 227         TC_PRIO_INTERACTIVE_BULK,
 228         ECN_OR_COST(INTERACTIVE_BULK),
 229         TC_PRIO_INTERACTIVE_BULK,
 230         ECN_OR_COST(INTERACTIVE_BULK)
 231 };
 232 EXPORT_SYMBOL(ip_tos2prio);
 233
 234 /*
 235  * Route cache.
 236  */
 237
 238 /* The locking scheme is rather straight forward:
 239  *
 240  * 1) Read-Copy Update protects the buckets of the central route hash.
 241  * 2) Only writers remove entries, and they hold the lock
 242  *    as they look at rtable reference counts.
 243  * 3) Only readers acquire references to rtable entries,
 244  *    they do so with atomic increments and with the
 245  *    lock held.
 246  */
 247
 248 struct rt_hash_bucket {
 249         struct rtable __rcu     *chain;
 250 };
 251
 252 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
 253         defined(CONFIG_PROVE_LOCKING)
 254 /*
 255  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
 256  * The size of this table is a power of two and depends on the number of CPUS.
 257  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
 258  */
 259 #ifdef CONFIG_LOCKDEP
 260 # define RT_HASH_LOCK_SZ        256
 261 #else
 262 # if NR_CPUS >= 32
 263 #  define RT_HASH_LOCK_SZ       4096
 264 # elif NR_CPUS >= 16
 265 #  define RT_HASH_LOCK_SZ       2048
 266 # elif NR_CPUS >= 8
 267 #  define RT_HASH_LOCK_SZ       1024
 268 # elif NR_CPUS >= 4
 269 #  define RT_HASH_LOCK_SZ       512
 270 # else
 271 #  define RT_HASH_LOCK_SZ       256
 272 # endif
 273 #endif
 274
 275 static spinlock_t       *rt_hash_locks;
 276 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
 277
 278 static __init void rt_hash_lock_init(void)
 279 {
 280         int i;
 281
 282         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
 283                         GFP_KERNEL);
 284         if (!rt_hash_locks)
 285                 panic("IP: failed to allocate rt_hash_locks\n");
 286
 287         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
 288                 spin_lock_init(&rt_hash_locks[i]);
 289 }
 290 #else
 291 # define rt_hash_lock_addr(slot) NULL
 292
 293 static inline void rt_hash_lock_init(void)
 294 {
 295 }
 296 #endif
 297
 298 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
 299 static unsigned int             rt_hash_mask __read_mostly;
 300 static unsigned int             rt_hash_log  __read_mostly;
 301
 302 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 303 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 304
 305 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
 306                                    int genid)
 307 {
 308         return jhash_3words((__force u32)daddr, (__force u32)saddr,
 309                             idx, genid)
 310                 & rt_hash_mask;
 311 }
 312
 313 static inline int rt_genid(struct net *net)
 314 {
 315         return atomic_read(&net->ipv4.rt_genid);
 316 }
 317
 318 #ifdef CONFIG_PROC_FS
 319 struct rt_cache_iter_state {
 320         struct seq_net_private p;
 321         int bucket;
 322         int genid;
 323 };
 324
 325 static struct rtable *rt_cache_get_first(struct seq_file *seq)
 326 {
 327         struct rt_cache_iter_state *st = seq->private;
 328         struct rtable *r = NULL;
 329
 330         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
 331                 if (!rcu_access_pointer(rt_hash_table[st->bucket].chain))
 332                         continue;
 333                 rcu_read_lock_bh();
 334                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
 335                 while (r) {
 336                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
 337                             r->rt_genid == st->genid)
 338                                 return r;
 339                         r = rcu_dereference_bh(r->dst.rt_next);
 340                 }
 341                 rcu_read_unlock_bh();
 342         }
 343         return r;
 344 }
 345
 346 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
 347                                           struct rtable *r)
 348 {
 349         struct rt_cache_iter_state *st = seq->private;
 350
 351         r = rcu_dereference_bh(r->dst.rt_next);
 352         while (!r) {
 353                 rcu_read_unlock_bh();
 354                 do {
 355                         if (--st->bucket < 0)
 356                                 return NULL;
 357                 } while (!rcu_access_pointer(rt_hash_table[st->bucket].chain));
 358                 rcu_read_lock_bh();
 359                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
 360         }
 361         return r;
 362 }
 363
 364 static struct rtable *rt_cache_get_next(struct seq_file *seq,
 365                                         struct rtable *r)
 366 {
 367         struct rt_cache_iter_state *st = seq->private;
 368         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
 369                 if (dev_net(r->dst.dev) != seq_file_net(seq))
 370                         continue;
 371                 if (r->rt_genid == st->genid)
 372                         break;
 373         }
 374         return r;
 375 }
 376
 377 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
 378 {
 379         struct rtable *r = rt_cache_get_first(seq);
 380
 381         if (r)
 382                 while (pos && (r = rt_cache_get_next(seq, r)))
 383                         --pos;
 384         return pos ? NULL : r;
 385 }
 386
 387 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 388 {
 389         struct rt_cache_iter_state *st = seq->private;
 390         if (*pos)
 391                 return rt_cache_get_idx(seq, *pos - 1);
 392         st->genid = rt_genid(seq_file_net(seq));
 393         return SEQ_START_TOKEN;
 394 }
 395
 396 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 397 {
 398         struct rtable *r;
 399
 400         if (v == SEQ_START_TOKEN)
 401                 r = rt_cache_get_first(seq);
 402         else
 403                 r = rt_cache_get_next(seq, v);
 404         ++*pos;
 405         return r;
 406 }
 407
 408 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 409 {
 410         if (v && v != SEQ_START_TOKEN)
 411                 rcu_read_unlock_bh();
 412 }
 413
 414 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 415 {
 416         if (v == SEQ_START_TOKEN)
 417                 seq_printf(seq, "%-127s\n",
 418                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 419                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 420                            "HHUptod\tSpecDst");
 421         else {
 422                 struct rtable *r = v;
 423                 int len;
 424
 425                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
 426                            "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
 427                            r->dst.dev ? r->dst.dev->name : "*",
 428                            (__force u32)r->rt_dst,
 429                            (__force u32)r->rt_gateway,
 430                            r->rt_flags, atomic_read(&r->dst.__refcnt),
 431                            r->dst.__use, 0, (__force u32)r->rt_src,
 432                            dst_metric_advmss(&r->dst) + 40,
 433                            dst_metric(&r->dst, RTAX_WINDOW), 0,
 434                            r->rt_key_tos,
 435                            -1, 0, 0, &len);
 436
 437                 seq_printf(seq, "%*s\n", 127 - len, "");
 438         }
 439         return 0;
 440 }
 441
 442 static const struct seq_operations rt_cache_seq_ops = {
 443         .start  = rt_cache_seq_start,
 444         .next   = rt_cache_seq_next,
 445         .stop   = rt_cache_seq_stop,
 446         .show   = rt_cache_seq_show,
 447 };
 448
 449 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 450 {
 451         return seq_open_net(inode, file, &rt_cache_seq_ops,
 452                         sizeof(struct rt_cache_iter_state));
 453 }
 454
 455 static const struct file_operations rt_cache_seq_fops = {
 456         .owner   = THIS_MODULE,
 457         .open    = rt_cache_seq_open,
 458         .read    = seq_read,
 459         .llseek  = seq_lseek,
 460         .release = seq_release_net,
 461 };
 462
 463
 464 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 465 {
 466         int cpu;
 467
 468         if (*pos == 0)
 469                 return SEQ_START_TOKEN;
 470
 471         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 472                 if (!cpu_possible(cpu))
 473                         continue;
 474                 *pos = cpu+1;
 475                 return &per_cpu(rt_cache_stat, cpu);
 476         }
 477         return NULL;
 478 }
 479
 480 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 481 {
 482         int cpu;
 483
 484         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 485                 if (!cpu_possible(cpu))
 486                         continue;
 487                 *pos = cpu+1;
 488                 return &per_cpu(rt_cache_stat, cpu);
 489         }
 490         return NULL;
 491
 492 }
 493
 494 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 495 {
 496
 497 }
 498
 499 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 500 {
 501         struct rt_cache_stat *st = v;
 502
 503         if (v == SEQ_START_TOKEN) {
 504                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 505                 return 0;
 506         }
 507
 508         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 509                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 510                    dst_entries_get_slow(&ipv4_dst_ops),
 511                    st->in_hit,
 512                    st->in_slow_tot,
 513                    st->in_slow_mc,
 514                    st->in_no_route,
 515                    st->in_brd,
 516                    st->in_martian_dst,
 517                    st->in_martian_src,
 518
 519                    st->out_hit,
 520                    st->out_slow_tot,
 521                    st->out_slow_mc,
 522
 523                    st->gc_total,
 524                    st->gc_ignored,
 525                    st->gc_goal_miss,
 526                    st->gc_dst_overflow,
 527                    st->in_hlist_search,
 528                    st->out_hlist_search
 529                 );
 530         return 0;
 531 }
 532
 533 static const struct seq_operations rt_cpu_seq_ops = {
 534         .start  = rt_cpu_seq_start,
 535         .next   = rt_cpu_seq_next,
 536         .stop   = rt_cpu_seq_stop,
 537         .show   = rt_cpu_seq_show,
 538 };
 539
 540
 541 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 542 {
 543         return seq_open(file, &rt_cpu_seq_ops);
 544 }
 545
 546 static const struct file_operations rt_cpu_seq_fops = {
 547         .owner   = THIS_MODULE,
 548         .open    = rt_cpu_seq_open,
 549         .read    = seq_read,
 550         .llseek  = seq_lseek,
 551         .release = seq_release,
 552 };
 553
 554 #ifdef CONFIG_IP_ROUTE_CLASSID
 555 static int rt_acct_proc_show(struct seq_file *m, void *v)
 556 {
 557         struct ip_rt_acct *dst, *src;
 558         unsigned int i, j;
 559
 560         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 561         if (!dst)
 562                 return -ENOMEM;
 563
 564         for_each_possible_cpu(i) {
 565                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 566                 for (j = 0; j < 256; j++) {
 567                         dst[j].o_bytes   += src[j].o_bytes;
 568                         dst[j].o_packets += src[j].o_packets;
 569                         dst[j].i_bytes   += src[j].i_bytes;
 570                         dst[j].i_packets += src[j].i_packets;
 571                 }
 572         }
 573
 574         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 575         kfree(dst);
 576         return 0;
 577 }
 578
 579 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 580 {
 581         return single_open(file, rt_acct_proc_show, NULL);
 582 }
 583
 584 static const struct file_operations rt_acct_proc_fops = {
 585         .owner          = THIS_MODULE,
 586         .open           = rt_acct_proc_open,
 587         .read           = seq_read,
 588         .llseek         = seq_lseek,
 589         .release        = single_release,
 590 };
 591 #endif
 592
 593 static int __net_init ip_rt_do_proc_init(struct net *net)
 594 {
 595         struct proc_dir_entry *pde;
 596
 597         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
 598                         &rt_cache_seq_fops);
 599         if (!pde)
 600                 goto err1;
 601
 602         pde = proc_create("rt_cache", S_IRUGO,
 603                           net->proc_net_stat, &rt_cpu_seq_fops);
 604         if (!pde)
 605                 goto err2;
 606
 607 #ifdef CONFIG_IP_ROUTE_CLASSID
 608         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 609         if (!pde)
 610                 goto err3;
 611 #endif
 612         return 0;
 613
 614 #ifdef CONFIG_IP_ROUTE_CLASSID
 615 err3:
 616         remove_proc_entry("rt_cache", net->proc_net_stat);
 617 #endif
 618 err2:
 619         remove_proc_entry("rt_cache", net->proc_net);
 620 err1:
 621         return -ENOMEM;
 622 }
 623
 624 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 625 {
 626         remove_proc_entry("rt_cache", net->proc_net_stat);
 627         remove_proc_entry("rt_cache", net->proc_net);
 628 #ifdef CONFIG_IP_ROUTE_CLASSID
 629         remove_proc_entry("rt_acct", net->proc_net);
 630 #endif
 631 }
 632
 633 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 634         .init = ip_rt_do_proc_init,
 635         .exit = ip_rt_do_proc_exit,
 636 };
 637
 638 static int __init ip_rt_proc_init(void)
 639 {
 640         return register_pernet_subsys(&ip_rt_proc_ops);
 641 }
 642
 643 #else
 644 static inline int ip_rt_proc_init(void)
 645 {
 646         return 0;
 647 }
 648 #endif /* CONFIG_PROC_FS */
 649
 650 static inline void rt_free(struct rtable *rt)
 651 {
 652         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
 653 }
 654
 655 static inline void rt_drop(struct rtable *rt)
 656 {
 657         ip_rt_put(rt);
 658         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
 659 }
 660
 661 static inline int rt_fast_clean(struct rtable *rth)
 662 {
 663         /* Kill broadcast/multicast entries very aggresively, if they
 664            collide in hash table with more useful entries */
 665         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
 666                 rt_is_input_route(rth) && rth->dst.rt_next;
 667 }
 668
 669 static inline int rt_valuable(struct rtable *rth)
 670 {
 671         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
 672                 (rt_has_peer(rth) && rt_peer_ptr(rth)->pmtu_expires);
 673 }
 674
 675 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
 676 {
 677         unsigned long age;
 678         int ret = 0;
 679
 680         if (atomic_read(&rth->dst.__refcnt))
 681                 goto out;
 682
 683         age = jiffies - rth->dst.lastuse;
 684         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
 685             (age <= tmo2 && rt_valuable(rth)))
 686                 goto out;
 687         ret = 1;
 688 out:    return ret;
 689 }
 690
 691 /* Bits of score are:
 692  * 31: very valuable
 693  * 30: not quite useless
 694  * 29..0: usage counter
 695  */
 696 static inline u32 rt_score(struct rtable *rt)
 697 {
 698         u32 score = jiffies - rt->dst.lastuse;
 699
 700         score = ~score & ~(3<<30);
 701
 702         if (rt_valuable(rt))
 703                 score |= (1<<31);
 704
 705         if (rt_is_output_route(rt) ||
 706             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
 707                 score |= (1<<30);
 708
 709         return score;
 710 }
 711
 712 static inline bool rt_caching(const struct net *net)
 713 {
 714         return net->ipv4.current_rt_cache_rebuild_count <=
 715                 net->ipv4.sysctl_rt_cache_rebuild_count;
 716 }
 717
 718 static inline bool compare_hash_inputs(const struct rtable *rt1,
 719                                        const struct rtable *rt2)
 720 {
 721         return ((((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
 722                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
 723                 (rt1->rt_route_iif ^ rt2->rt_route_iif)) == 0);
 724 }
 725
 726 static inline int compare_keys(struct rtable *rt1, struct rtable *rt2)
 727 {
 728         return (((__force u32)rt1->rt_key_dst ^ (__force u32)rt2->rt_key_dst) |
 729                 ((__force u32)rt1->rt_key_src ^ (__force u32)rt2->rt_key_src) |
 730                 (rt1->rt_mark ^ rt2->rt_mark) |
 731                 (rt1->rt_key_tos ^ rt2->rt_key_tos) |
 732                 (rt1->rt_route_iif ^ rt2->rt_route_iif) |
 733                 (rt1->rt_oif ^ rt2->rt_oif)) == 0;
 734 }
 735
 736 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
 737 {
 738         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
 739 }
 740
 741 static inline int rt_is_expired(struct rtable *rth)
 742 {
 743         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 744 }
 745
 746 /*
 747  * Perform a full scan of hash table and free all entries.
 748  * Can be called by a softirq or a process.
 749  * In the later case, we want to be reschedule if necessary
 750  */
 751 static void rt_do_flush(struct net *net, int process_context)
 752 {
 753         unsigned int i;
 754         struct rtable *rth, *next;
 755
 756         for (i = 0; i <= rt_hash_mask; i++) {
 757                 struct rtable __rcu **pprev;
 758                 struct rtable *list;
 759
 760                 if (process_context && need_resched())
 761                         cond_resched();
 762                 rth = rcu_access_pointer(rt_hash_table[i].chain);
 763                 if (!rth)
 764                         continue;
 765
 766                 spin_lock_bh(rt_hash_lock_addr(i));
 767
 768                 list = NULL;
 769                 pprev = &rt_hash_table[i].chain;
 770                 rth = rcu_dereference_protected(*pprev,
 771                         lockdep_is_held(rt_hash_lock_addr(i)));
 772
 773                 while (rth) {
 774                         next = rcu_dereference_protected(rth->dst.rt_next,
 775                                 lockdep_is_held(rt_hash_lock_addr(i)));
 776
 777                         if (!net ||
 778                             net_eq(dev_net(rth->dst.dev), net)) {
 779                                 rcu_assign_pointer(*pprev, next);
 780                                 rcu_assign_pointer(rth->dst.rt_next, list);
 781                                 list = rth;
 782                         } else {
 783                                 pprev = &rth->dst.rt_next;
 784                         }
 785                         rth = next;
 786                 }
 787
 788                 spin_unlock_bh(rt_hash_lock_addr(i));
 789
 790                 for (; list; list = next) {
 791                         next = rcu_dereference_protected(list->dst.rt_next, 1);
 792                         rt_free(list);
 793                 }
 794         }
 795 }
 796
 797 /*
 798  * While freeing expired entries, we compute average chain length
 799  * and standard deviation, using fixed-point arithmetic.
 800  * This to have an estimation of rt_chain_length_max
 801  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
 802  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
 803  */
 804
 805 #define FRACT_BITS 3
 806 #define ONE (1UL << FRACT_BITS)
 807
 808 /*
 809  * Given a hash chain and an item in this hash chain,
 810  * find if a previous entry has the same hash_inputs
 811  * (but differs on tos, mark or oif)
 812  * Returns 0 if an alias is found.
 813  * Returns ONE if rth has no alias before itself.
 814  */
 815 static int has_noalias(const struct rtable *head, const struct rtable *rth)
 816 {
 817         const struct rtable *aux = head;
 818
 819         while (aux != rth) {
 820                 if (compare_hash_inputs(aux, rth))
 821                         return 0;
 822                 aux = rcu_dereference_protected(aux->dst.rt_next, 1);
 823         }
 824         return ONE;
 825 }
 826
 827 static void rt_check_expire(void)
 828 {
 829         static unsigned int rover;
 830         unsigned int i = rover, goal;
 831         struct rtable *rth;
 832         struct rtable __rcu **rthp;
 833         unsigned long samples = 0;
 834         unsigned long sum = 0, sum2 = 0;
 835         unsigned long delta;
 836         u64 mult;
 837
 838         delta = jiffies - expires_ljiffies;
 839         expires_ljiffies = jiffies;
 840         mult = ((u64)delta) << rt_hash_log;
 841         if (ip_rt_gc_timeout > 1)
 842                 do_div(mult, ip_rt_gc_timeout);
 843         goal = (unsigned int)mult;
 844         if (goal > rt_hash_mask)
 845                 goal = rt_hash_mask + 1;
 846         for (; goal > 0; goal--) {
 847                 unsigned long tmo = ip_rt_gc_timeout;
 848                 unsigned long length;
 849
 850                 i = (i + 1) & rt_hash_mask;
 851                 rthp = &rt_hash_table[i].chain;
 852
 853                 if (need_resched())
 854                         cond_resched();
 855
 856                 samples++;
 857
 858                 if (rcu_dereference_raw(*rthp) == NULL)
 859                         continue;
 860                 length = 0;
 861                 spin_lock_bh(rt_hash_lock_addr(i));
 862                 while ((rth = rcu_dereference_protected(*rthp,
 863                                         lockdep_is_held(rt_hash_lock_addr(i)))) != NULL) {
 864                         prefetch(rth->dst.rt_next);
 865                         if (rt_is_expired(rth) ||
 866                             rt_may_expire(rth, tmo, ip_rt_gc_timeout)) {
 867                                 *rthp = rth->dst.rt_next;
 868                                 rt_free(rth);
 869                                 continue;
 870                         }
 871
 872                         /* We only count entries on a chain with equal
 873                          * hash inputs once so that entries for
 874                          * different QOS levels, and other non-hash
 875                          * input attributes don't unfairly skew the
 876                          * length computation
 877                          */
 878                         tmo >>= 1;
 879                         rthp = &rth->dst.rt_next;
 880                         length += has_noalias(rt_hash_table[i].chain, rth);
 881                 }
 882                 spin_unlock_bh(rt_hash_lock_addr(i));
 883                 sum += length;
 884                 sum2 += length*length;
 885         }
 886         if (samples) {
 887                 unsigned long avg = sum / samples;
 888                 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
 889                 rt_chain_length_max = max_t(unsigned long,
 890                                         ip_rt_gc_elasticity,
 891                                         (avg + 4*sd) >> FRACT_BITS);
 892         }
 893         rover = i;
 894 }
 895
 896 /*
 897  * rt_worker_func() is run in process context.
 898  * we call rt_check_expire() to scan part of the hash table
 899  */
 900 static void rt_worker_func(struct work_struct *work)
 901 {
 902         rt_check_expire();
 903         schedule_delayed_work(&expires_work, ip_rt_gc_interval);
 904 }
 905
 906 /*
 907  * Perturbation of rt_genid by a small quantity [1..256]
 908  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
 909  * many times (2^24) without giving recent rt_genid.
 910  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
 911  */
 912 static void rt_cache_invalidate(struct net *net)
 913 {
 914         unsigned char shuffle;
 915
 916         get_random_bytes(&shuffle, sizeof(shuffle));
 917         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
 918         inetpeer_invalidate_family(AF_INET);
 919 }
 920
 921 /*
 922  * delay < 0  : invalidate cache (fast : entries will be deleted later)
 923  * delay >= 0 : invalidate & flush cache (can be long)
 924  */
 925 void rt_cache_flush(struct net *net, int delay)
 926 {
 927         rt_cache_invalidate(net);
 928         if (delay >= 0)
 929                 rt_do_flush(net, !in_softirq());
 930 }
 931
 932 /* Flush previous cache invalidated entries from the cache */
 933 void rt_cache_flush_batch(struct net *net)
 934 {
 935         rt_do_flush(net, !in_softirq());
 936 }
 937
 938 static void rt_emergency_hash_rebuild(struct net *net)
 939 {
 940         net_warn_ratelimited("Route hash chain too long!\n");
 941         rt_cache_invalidate(net);
 942 }
 943
 944 /*
 945    Short description of GC goals.
 946
 947    We want to build algorithm, which will keep routing cache
 948    at some equilibrium point, when number of aged off entries
 949    is kept approximately equal to newly generated ones.
 950
 951    Current expiration strength is variable "expire".
 952    We try to adjust it dynamically, so that if networking
 953    is idle expires is large enough to keep enough of warm entries,
 954    and when load increases it reduces to limit cache size.
 955  */
 956
 957 static int rt_garbage_collect(struct dst_ops *ops)
 958 {
 959         static unsigned long expire = RT_GC_TIMEOUT;
 960         static unsigned long last_gc;
 961         static int rover;
 962         static int equilibrium;
 963         struct rtable *rth;
 964         struct rtable __rcu **rthp;
 965         unsigned long now = jiffies;
 966         int goal;
 967         int entries = dst_entries_get_fast(&ipv4_dst_ops);
 968
 969         /*
 970          * Garbage collection is pretty expensive,
 971          * do not make it too frequently.
 972          */
 973
 974         RT_CACHE_STAT_INC(gc_total);
 975
 976         if (now - last_gc < ip_rt_gc_min_interval &&
 977             entries < ip_rt_max_size) {
 978                 RT_CACHE_STAT_INC(gc_ignored);
 979                 goto out;
 980         }
 981
 982         entries = dst_entries_get_slow(&ipv4_dst_ops);
 983         /* Calculate number of entries, which we want to expire now. */
 984         goal = entries - (ip_rt_gc_elasticity << rt_hash_log);
 985         if (goal <= 0) {
 986                 if (equilibrium < ipv4_dst_ops.gc_thresh)
 987                         equilibrium = ipv4_dst_ops.gc_thresh;
 988                 goal = entries - equilibrium;
 989                 if (goal > 0) {
 990                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
 991                         goal = entries - equilibrium;
 992                 }
 993         } else {
 994                 /* We are in dangerous area. Try to reduce cache really
 995                  * aggressively.
 996                  */
 997                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
 998                 equilibrium = entries - goal;
 999         }
1000
1001         if (now - last_gc >= ip_rt_gc_min_interval)
1002                 last_gc = now;
1003
1004         if (goal <= 0) {
1005                 equilibrium += goal;
1006                 goto work_done;
1007         }
1008
1009         do {
1010                 int i, k;
1011
1012                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
1013                         unsigned long tmo = expire;
1014
1015                         k = (k + 1) & rt_hash_mask;
1016                         rthp = &rt_hash_table[k].chain;
1017                         spin_lock_bh(rt_hash_lock_addr(k));
1018                         while ((rth = rcu_dereference_protected(*rthp,
1019                                         lockdep_is_held(rt_hash_lock_addr(k)))) != NULL) {
1020                                 if (!rt_is_expired(rth) &&
1021                                         !rt_may_expire(rth, tmo, expire)) {
1022                                         tmo >>= 1;
1023                                         rthp = &rth->dst.rt_next;
1024                                         continue;
1025                                 }
1026                                 *rthp = rth->dst.rt_next;
1027                                 rt_free(rth);
1028                                 goal--;
1029                         }
1030                         spin_unlock_bh(rt_hash_lock_addr(k));
1031                         if (goal <= 0)
1032                                 break;
1033                 }
1034                 rover = k;
1035
1036                 if (goal <= 0)
1037                         goto work_done;
1038
1039                 /* Goal is not achieved. We stop process if:
1040
1041                    - if expire reduced to zero. Otherwise, expire is halfed.
1042                    - if table is not full.
1043                    - if we are called from interrupt.
1044                    - jiffies check is just fallback/debug loop breaker.
1045                      We will not spin here for long time in any case.
1046                  */
1047
1048                 RT_CACHE_STAT_INC(gc_goal_miss);
1049
1050                 if (expire == 0)
1051                         break;
1052
1053                 expire >>= 1;
1054
1055                 if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1056                         goto out;
1057         } while (!in_softirq() && time_before_eq(jiffies, now));
1058
1059         if (dst_entries_get_fast(&ipv4_dst_ops) < ip_rt_max_size)
1060                 goto out;
1061         if (dst_entries_get_slow(&ipv4_dst_ops) < ip_rt_max_size)
1062                 goto out;
1063         net_warn_ratelimited("dst cache overflow\n");
1064         RT_CACHE_STAT_INC(gc_dst_overflow);
1065         return 1;
1066
1067 work_done:
1068         expire += ip_rt_gc_min_interval;
1069         if (expire > ip_rt_gc_timeout ||
1070             dst_entries_get_fast(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh ||
1071             dst_entries_get_slow(&ipv4_dst_ops) < ipv4_dst_ops.gc_thresh)
1072                 expire = ip_rt_gc_timeout;
1073 out:    return 0;
1074 }
1075
1076 /*
1077  * Returns number of entries in a hash chain that have different hash_inputs
1078  */
1079 static int slow_chain_length(const struct rtable *head)
1080 {
1081         int length = 0;
1082         const struct rtable *rth = head;
1083
1084         while (rth) {
1085                 length += has_noalias(head, rth);
1086                 rth = rcu_dereference_protected(rth->dst.rt_next, 1);
1087         }
1088         return length >> FRACT_BITS;
1089 }
1090
1091 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
1092                                            struct sk_buff *skb,
1093                                            const void *daddr)
1094 {
1095         struct net_device *dev = dst->dev;
1096         const __be32 *pkey = daddr;
1097         const struct rtable *rt;
1098         struct neighbour *n;
1099
1100         rt = (const struct rtable *) dst;
1101         if (rt->rt_gateway)
1102                 pkey = (const __be32 *) &rt->rt_gateway;
1103         else if (skb)
1104                 pkey = &ip_hdr(skb)->daddr;
1105
1106         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
1107         if (n)
1108                 return n;
1109         return neigh_create(&arp_tbl, pkey, dev);
1110 }
1111
1112 static struct rtable *rt_intern_hash(unsigned int hash, struct rtable *rt,
1113                                      struct sk_buff *skb, int ifindex)
1114 {
1115         struct rtable   *rth, *cand;
1116         struct rtable __rcu **rthp, **candp;
1117         unsigned long   now;
1118         u32             min_score;
1119         int             chain_length;
1120
1121 restart:
1122         chain_length = 0;
1123         min_score = ~(u32)0;
1124         cand = NULL;
1125         candp = NULL;
1126         now = jiffies;
1127
1128         if (!rt_caching(dev_net(rt->dst.dev)) || (rt->dst.flags & DST_NOCACHE)) {
1129                 /*
1130                  * If we're not caching, just tell the caller we
1131                  * were successful and don't touch the route.  The
1132                  * caller hold the sole reference to the cache entry, and
1133                  * it will be released when the caller is done with it.
1134                  * If we drop it here, the callers have no way to resolve routes
1135                  * when we're not caching.  Instead, just point *rp at rt, so
1136                  * the caller gets a single use out of the route
1137                  * Note that we do rt_free on this new route entry, so that
1138                  * once its refcount hits zero, we are still able to reap it
1139                  * (Thanks Alexey)
1140                  * Note: To avoid expensive rcu stuff for this uncached dst,
1141                  * we set DST_NOCACHE so that dst_release() can free dst without
1142                  * waiting a grace period.
1143                  */
1144
1145                 rt->dst.flags |= DST_NOCACHE;
1146                 goto skip_hashing;
1147         }
1148
1149         rthp = &rt_hash_table[hash].chain;
1150
1151         spin_lock_bh(rt_hash_lock_addr(hash));
1152         while ((rth = rcu_dereference_protected(*rthp,
1153                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1154                 if (rt_is_expired(rth)) {
1155                         *rthp = rth->dst.rt_next;
1156                         rt_free(rth);
1157                         continue;
1158                 }
1159                 if (compare_keys(rth, rt) && compare_netns(rth, rt)) {
1160                         /* Put it first */
1161                         *rthp = rth->dst.rt_next;
1162                         /*
1163                          * Since lookup is lockfree, the deletion
1164                          * must be visible to another weakly ordered CPU before
1165                          * the insertion at the start of the hash chain.
1166                          */
1167                         rcu_assign_pointer(rth->dst.rt_next,
1168                                            rt_hash_table[hash].chain);
1169                         /*
1170                          * Since lookup is lockfree, the update writes
1171                          * must be ordered for consistency on SMP.
1172                          */
1173                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1174
1175                         dst_use(&rth->dst, now);
1176                         spin_unlock_bh(rt_hash_lock_addr(hash));
1177
1178                         rt_drop(rt);
1179                         if (skb)
1180                                 skb_dst_set(skb, &rth->dst);
1181                         return rth;
1182                 }
1183
1184                 if (!atomic_read(&rth->dst.__refcnt)) {
1185                         u32 score = rt_score(rth);
1186
1187                         if (score <= min_score) {
1188                                 cand = rth;
1189                                 candp = rthp;
1190                                 min_score = score;
1191                         }
1192                 }
1193
1194                 chain_length++;
1195
1196                 rthp = &rth->dst.rt_next;
1197         }
1198
1199         if (cand) {
1200                 /* ip_rt_gc_elasticity used to be average length of chain
1201                  * length, when exceeded gc becomes really aggressive.
1202                  *
1203                  * The second limit is less certain. At the moment it allows
1204                  * only 2 entries per bucket. We will see.
1205                  */
1206                 if (chain_length > ip_rt_gc_elasticity) {
1207                         *candp = cand->dst.rt_next;
1208                         rt_free(cand);
1209                 }
1210         } else {
1211                 if (chain_length > rt_chain_length_max &&
1212                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1213                         struct net *net = dev_net(rt->dst.dev);
1214                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1215                         if (!rt_caching(net)) {
1216                                 pr_warn("%s: %d rebuilds is over limit, route caching disabled\n",
1217                                         rt->dst.dev->name, num);
1218                         }
1219                         rt_emergency_hash_rebuild(net);
1220                         spin_unlock_bh(rt_hash_lock_addr(hash));
1221
1222                         hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1223                                         ifindex, rt_genid(net));
1224                         goto restart;
1225                 }
1226         }
1227
1228         rt->dst.rt_next = rt_hash_table[hash].chain;
1229
1230         /*
1231          * Since lookup is lockfree, we must make sure
1232          * previous writes to rt are committed to memory
1233          * before making rt visible to other CPUS.
1234          */
1235         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1236
1237         spin_unlock_bh(rt_hash_lock_addr(hash));
1238
1239 skip_hashing:
1240         if (skb)
1241                 skb_dst_set(skb, &rt->dst);
1242         return rt;
1243 }
1244
1245 static atomic_t __rt_peer_genid = ATOMIC_INIT(0);
1246
1247 static u32 rt_peer_genid(void)
1248 {
1249         return atomic_read(&__rt_peer_genid);
1250 }
1251
1252 void rt_bind_peer(struct rtable *rt, __be32 daddr, int create)
1253 {
1254         struct inet_peer_base *base;
1255         struct inet_peer *peer;
1256
1257         base = inetpeer_base_ptr(rt->_peer);
1258         if (!base)
1259                 return;
1260
1261         peer = inet_getpeer_v4(base, daddr, create);
1262         if (peer) {
1263                 if (!rt_set_peer(rt, peer))
1264                         inet_putpeer(peer);
1265                 else
1266                         rt->rt_peer_genid = rt_peer_genid();
1267         }
1268 }
1269
1270 /*
1271  * Peer allocation may fail only in serious out-of-memory conditions.  However
1272  * we still can generate some output.
1273  * Random ID selection looks a bit dangerous because we have no chances to
1274  * select ID being unique in a reasonable period of time.
1275  * But broken packet identifier may be better than no packet at all.
1276  */
1277 static void ip_select_fb_ident(struct iphdr *iph)
1278 {
1279         static DEFINE_SPINLOCK(ip_fb_id_lock);
1280         static u32 ip_fallback_id;
1281         u32 salt;
1282
1283         spin_lock_bh(&ip_fb_id_lock);
1284         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1285         iph->id = htons(salt & 0xFFFF);
1286         ip_fallback_id = salt;
1287         spin_unlock_bh(&ip_fb_id_lock);
1288 }
1289
1290 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1291 {
1292         struct net *net = dev_net(dst->dev);
1293         struct inet_peer *peer;
1294
1295         peer = inet_getpeer_v4(net->ipv4.peers, iph->daddr, 1);
1296         if (peer) {
1297                 iph->id = htons(inet_getid(peer, more));
1298                 inet_putpeer(peer);
1299                 return;
1300         }
1301
1302         ip_select_fb_ident(iph);
1303 }
1304 EXPORT_SYMBOL(__ip_select_ident);
1305
1306 static void rt_del(unsigned int hash, struct rtable *rt)
1307 {
1308         struct rtable __rcu **rthp;
1309         struct rtable *aux;
1310
1311         rthp = &rt_hash_table[hash].chain;
1312         spin_lock_bh(rt_hash_lock_addr(hash));
1313         ip_rt_put(rt);
1314         while ((aux = rcu_dereference_protected(*rthp,
1315                         lockdep_is_held(rt_hash_lock_addr(hash)))) != NULL) {
1316                 if (aux == rt || rt_is_expired(aux)) {
1317                         *rthp = aux->dst.rt_next;
1318                         rt_free(aux);
1319                         continue;
1320                 }
1321                 rthp = &aux->dst.rt_next;
1322         }
1323         spin_unlock_bh(rt_hash_lock_addr(hash));
1324 }
1325
1326 static void check_peer_redir(struct dst_entry *dst, struct inet_peer *peer)
1327 {
1328         struct rtable *rt = (struct rtable *) dst;
1329         __be32 orig_gw = rt->rt_gateway;
1330         struct neighbour *n;
1331
1332         dst_confirm(&rt->dst);
1333
1334         rt->rt_gateway = peer->redirect_learned.a4;
1335
1336         n = ipv4_neigh_lookup(&rt->dst, NULL, &rt->rt_gateway);
1337         if (!n) {
1338                 rt->rt_gateway = orig_gw;
1339                 return;
1340         }
1341         if (!(n->nud_state & NUD_VALID)) {
1342                 neigh_event_send(n, NULL);
1343         } else {
1344                 rt->rt_flags |= RTCF_REDIRECTED;
1345                 call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
1346         }
1347         neigh_release(n);
1348 }
1349
1350 /* called in rcu_read_lock() section */
1351 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1352                     __be32 saddr, struct net_device *dev)
1353 {
1354         int s, i;
1355         struct in_device *in_dev = __in_dev_get_rcu(dev);
1356         __be32 skeys[2] = { saddr, 0 };
1357         int    ikeys[2] = { dev->ifindex, 0 };
1358         struct inet_peer *peer;
1359         struct net *net;
1360
1361         if (!in_dev)
1362                 return;
1363
1364         net = dev_net(dev);
1365         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1366             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1367             ipv4_is_zeronet(new_gw))
1368                 goto reject_redirect;
1369
1370         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1371                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1372                         goto reject_redirect;
1373                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1374                         goto reject_redirect;
1375         } else {
1376                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1377                         goto reject_redirect;
1378         }
1379
1380         for (s = 0; s < 2; s++) {
1381                 for (i = 0; i < 2; i++) {
1382                         unsigned int hash;
1383                         struct rtable __rcu **rthp;
1384                         struct rtable *rt;
1385
1386                         hash = rt_hash(daddr, skeys[s], ikeys[i], rt_genid(net));
1387
1388                         rthp = &rt_hash_table[hash].chain;
1389
1390                         while ((rt = rcu_dereference(*rthp)) != NULL) {
1391                                 rthp = &rt->dst.rt_next;
1392
1393                                 if (rt->rt_key_dst != daddr ||
1394                                     rt->rt_key_src != skeys[s] ||
1395                                     rt->rt_oif != ikeys[i] ||
1396                                     rt_is_input_route(rt) ||
1397                                     rt_is_expired(rt) ||
1398                                     !net_eq(dev_net(rt->dst.dev), net) ||
1399                                     rt->dst.error ||
1400                                     rt->dst.dev != dev ||
1401                                     rt->rt_gateway != old_gw)
1402                                         continue;
1403
1404                                 peer = rt_get_peer_create(rt, rt->rt_dst);
1405                                 if (peer) {
1406                                         if (peer->redirect_learned.a4 != new_gw) {
1407                                                 peer->redirect_learned.a4 = new_gw;
1408                                                 atomic_inc(&__rt_peer_genid);
1409                                         }
1410                                         check_peer_redir(&rt->dst, peer);
1411                                 }
1412                         }
1413                 }
1414         }
1415         return;
1416
1417 reject_redirect:
1418 #ifdef CONFIG_IP_ROUTE_VERBOSE
1419         if (IN_DEV_LOG_MARTIANS(in_dev))
1420                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
1421                                      "  Advised path = %pI4 -> %pI4\n",
1422                                      &old_gw, dev->name, &new_gw,
1423                                      &saddr, &daddr);
1424 #endif
1425         ;
1426 }
1427
1428 static bool peer_pmtu_expired(struct inet_peer *peer)
1429 {
1430         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1431
1432         return orig &&
1433                time_after_eq(jiffies, orig) &&
1434                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1435 }
1436
1437 static bool peer_pmtu_cleaned(struct inet_peer *peer)
1438 {
1439         unsigned long orig = ACCESS_ONCE(peer->pmtu_expires);
1440
1441         return orig &&
1442                cmpxchg(&peer->pmtu_expires, orig, 0) == orig;
1443 }
1444
1445 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1446 {
1447         struct rtable *rt = (struct rtable *)dst;
1448         struct dst_entry *ret = dst;
1449
1450         if (rt) {
1451                 if (dst->obsolete > 0) {
1452                         ip_rt_put(rt);
1453                         ret = NULL;
1454                 } else if (rt->rt_flags & RTCF_REDIRECTED) {
1455                         unsigned int hash = rt_hash(rt->rt_key_dst, rt->rt_key_src,
1456                                                 rt->rt_oif,
1457                                                 rt_genid(dev_net(dst->dev)));
1458                         rt_del(hash, rt);
1459                         ret = NULL;
1460                 } else if (rt_has_peer(rt)) {
1461                         struct inet_peer *peer = rt_peer_ptr(rt);
1462                         if (peer_pmtu_expired(peer))
1463                                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1464                 }
1465         }
1466         return ret;
1467 }
1468
1469 /*
1470  * Algorithm:
1471  *      1. The first ip_rt_redirect_number redirects are sent
1472  *         with exponential backoff, then we stop sending them at all,
1473  *         assuming that the host ignores our redirects.
1474  *      2. If we did not see packets requiring redirects
1475  *         during ip_rt_redirect_silence, we assume that the host
1476  *         forgot redirected route and start to send redirects again.
1477  *
1478  * This algorithm is much cheaper and more intelligent than dumb load limiting
1479  * in icmp.c.
1480  *
1481  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1482  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1483  */
1484
1485 void ip_rt_send_redirect(struct sk_buff *skb)
1486 {
1487         struct rtable *rt = skb_rtable(skb);
1488         struct in_device *in_dev;
1489         struct inet_peer *peer;
1490         struct net *net;
1491         int log_martians;
1492
1493         rcu_read_lock();
1494         in_dev = __in_dev_get_rcu(rt->dst.dev);
1495         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1496                 rcu_read_unlock();
1497                 return;
1498         }
1499         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1500         rcu_read_unlock();
1501
1502         net = dev_net(rt->dst.dev);
1503         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
1504         if (!peer) {
1505                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1506                 return;
1507         }
1508
1509         /* No redirected packets during ip_rt_redirect_silence;
1510          * reset the algorithm.
1511          */
1512         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
1513                 peer->rate_tokens = 0;
1514
1515         /* Too many ignored redirects; do not send anything
1516          * set dst.rate_last to the last seen redirected packet.
1517          */
1518         if (peer->rate_tokens >= ip_rt_redirect_number) {
1519                 peer->rate_last = jiffies;
1520                 goto out_put_peer;
1521         }
1522
1523         /* Check for load limit; set rate_last to the latest sent
1524          * redirect.
1525          */
1526         if (peer->rate_tokens == 0 ||
1527             time_after(jiffies,
1528                        (peer->rate_last +
1529                         (ip_rt_redirect_load << peer->rate_tokens)))) {
1530                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1531                 peer->rate_last = jiffies;
1532                 ++peer->rate_tokens;
1533 #ifdef CONFIG_IP_ROUTE_VERBOSE
1534                 if (log_martians &&
1535                     peer->rate_tokens == ip_rt_redirect_number)
1536                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
1537                                              &ip_hdr(skb)->saddr, rt->rt_iif,
1538                                              &rt->rt_dst, &rt->rt_gateway);
1539 #endif
1540         }
1541 out_put_peer:
1542         inet_putpeer(peer);
1543 }
1544
1545 static int ip_error(struct sk_buff *skb)
1546 {
1547         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
1548         struct rtable *rt = skb_rtable(skb);
1549         struct inet_peer *peer;
1550         unsigned long now;
1551         struct net *net;
1552         bool send;
1553         int code;
1554
1555         net = dev_net(rt->dst.dev);
1556         if (!IN_DEV_FORWARD(in_dev)) {
1557                 switch (rt->dst.error) {
1558                 case EHOSTUNREACH:
1559                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
1560                         break;
1561
1562                 case ENETUNREACH:
1563                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1564                         break;
1565                 }
1566                 goto out;
1567         }
1568
1569         switch (rt->dst.error) {
1570         case EINVAL:
1571         default:
1572                 goto out;
1573         case EHOSTUNREACH:
1574                 code = ICMP_HOST_UNREACH;
1575                 break;
1576         case ENETUNREACH:
1577                 code = ICMP_NET_UNREACH;
1578                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
1579                 break;
1580         case EACCES:
1581                 code = ICMP_PKT_FILTERED;
1582                 break;
1583         }
1584
1585         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, 1);
1586
1587         send = true;
1588         if (peer) {
1589                 now = jiffies;
1590                 peer->rate_tokens += now - peer->rate_last;
1591                 if (peer->rate_tokens > ip_rt_error_burst)
1592                         peer->rate_tokens = ip_rt_error_burst;
1593                 peer->rate_last = now;
1594                 if (peer->rate_tokens >= ip_rt_error_cost)
1595                         peer->rate_tokens -= ip_rt_error_cost;
1596                 else
1597                         send = false;
1598                 inet_putpeer(peer);
1599         }
1600         if (send)
1601                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1602
1603 out:    kfree_skb(skb);
1604         return 0;
1605 }
1606
1607 static void check_peer_pmtu(struct dst_entry *dst, struct inet_peer *peer)
1608 {
1609         unsigned long expires = ACCESS_ONCE(peer->pmtu_expires);
1610
1611         if (!expires)
1612                 return;
1613         if (time_before(jiffies, expires)) {
1614                 u32 orig_dst_mtu = dst_mtu(dst);
1615                 if (peer->pmtu_learned < orig_dst_mtu) {
1616                         if (!peer->pmtu_orig)
1617                                 peer->pmtu_orig = dst_metric_raw(dst, RTAX_MTU);
1618                         dst_metric_set(dst, RTAX_MTU, peer->pmtu_learned);
1619                 }
1620         } else if (cmpxchg(&peer->pmtu_expires, expires, 0) == expires)
1621                 dst_metric_set(dst, RTAX_MTU, peer->pmtu_orig);
1622 }
1623
1624 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1625 {
1626         struct rtable *rt = (struct rtable *) dst;
1627         struct inet_peer *peer;
1628
1629         dst_confirm(dst);
1630
1631         peer = rt_get_peer_create(rt, rt->rt_dst);
1632         if (peer) {
1633                 unsigned long pmtu_expires = ACCESS_ONCE(peer->pmtu_expires);
1634
1635                 if (mtu < ip_rt_min_pmtu)
1636                         mtu = ip_rt_min_pmtu;
1637                 if (!pmtu_expires || mtu < peer->pmtu_learned) {
1638
1639                         pmtu_expires = jiffies + ip_rt_mtu_expires;
1640                         if (!pmtu_expires)
1641                                 pmtu_expires = 1UL;
1642
1643                         peer->pmtu_learned = mtu;
1644                         peer->pmtu_expires = pmtu_expires;
1645
1646                         atomic_inc(&__rt_peer_genid);
1647                         rt->rt_peer_genid = rt_peer_genid();
1648                 }
1649                 check_peer_pmtu(dst, peer);
1650         }
1651 }
1652
1653 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1654                       int oif, u32 mark, u8 protocol, int flow_flags)
1655 {
1656         const struct iphdr *iph = (const struct iphdr *)skb->data;
1657         struct flowi4 fl4;
1658         struct rtable *rt;
1659
1660         flowi4_init_output(&fl4, oif, mark, RT_TOS(iph->tos), RT_SCOPE_UNIVERSE,
1661                            protocol, flow_flags,
1662                            iph->daddr, iph->saddr, 0, 0);
1663         rt = __ip_route_output_key(net, &fl4);
1664         if (!IS_ERR(rt)) {
1665                 ip_rt_update_pmtu(&rt->dst, mtu);
1666                 ip_rt_put(rt);
1667         }
1668 }
1669 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1670
1671 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1672 {
1673         const struct inet_sock *inet = inet_sk(sk);
1674
1675         return ipv4_update_pmtu(skb, sock_net(sk), mtu,
1676                                 sk->sk_bound_dev_if, sk->sk_mark,
1677                                 inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
1678                                 inet_sk_flowi_flags(sk));
1679 }
1680 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1681
1682 static void ipv4_validate_peer(struct rtable *rt)
1683 {
1684         if (rt->rt_peer_genid != rt_peer_genid()) {
1685                 struct inet_peer *peer = rt_get_peer(rt, rt->rt_dst);
1686
1687                 if (peer) {
1688                         check_peer_pmtu(&rt->dst, peer);
1689
1690                         if (peer->redirect_learned.a4 &&
1691                             peer->redirect_learned.a4 != rt->rt_gateway)
1692                                 check_peer_redir(&rt->dst, peer);
1693                 }
1694
1695                 rt->rt_peer_genid = rt_peer_genid();
1696         }
1697 }
1698
1699 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1700 {
1701         struct rtable *rt = (struct rtable *) dst;
1702
1703         if (rt_is_expired(rt))
1704                 return NULL;
1705         ipv4_validate_peer(rt);
1706         return dst;
1707 }
1708
1709 static void ipv4_dst_destroy(struct dst_entry *dst)
1710 {
1711         struct rtable *rt = (struct rtable *) dst;
1712
1713         if (rt->fi) {
1714                 fib_info_put(rt->fi);
1715                 rt->fi = NULL;
1716         }
1717         if (rt_has_peer(rt)) {
1718                 struct inet_peer *peer = rt_peer_ptr(rt);
1719                 inet_putpeer(peer);
1720         }
1721 }
1722
1723
1724 static void ipv4_link_failure(struct sk_buff *skb)
1725 {
1726         struct rtable *rt;
1727
1728         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1729
1730         rt = skb_rtable(skb);
1731         if (rt && rt_has_peer(rt)) {
1732                 struct inet_peer *peer = rt_peer_ptr(rt);
1733                 if (peer_pmtu_cleaned(peer))
1734                         dst_metric_set(&rt->dst, RTAX_MTU, peer->pmtu_orig);
1735         }
1736 }
1737
1738 static int ip_rt_bug(struct sk_buff *skb)
1739 {
1740         pr_debug("%s: %pI4 -> %pI4, %s\n",
1741                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1742                  skb->dev ? skb->dev->name : "?");
1743         kfree_skb(skb);
1744         WARN_ON(1);
1745         return 0;
1746 }
1747
1748 /*
1749    We do not cache source address of outgoing interface,
1750    because it is used only by IP RR, TS and SRR options,
1751    so that it out of fast path.
1752
1753    BTW remember: "addr" is allowed to be not aligned
1754    in IP options!
1755  */
1756
1757 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1758 {
1759         __be32 src;
1760
1761         if (rt_is_output_route(rt))
1762                 src = ip_hdr(skb)->saddr;
1763         else {
1764                 struct fib_result res;
1765                 struct flowi4 fl4;
1766                 struct iphdr *iph;
1767
1768                 iph = ip_hdr(skb);
1769
1770                 memset(&fl4, 0, sizeof(fl4));
1771                 fl4.daddr = iph->daddr;
1772                 fl4.saddr = iph->saddr;
1773                 fl4.flowi4_tos = RT_TOS(iph->tos);
1774                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1775                 fl4.flowi4_iif = skb->dev->ifindex;
1776                 fl4.flowi4_mark = skb->mark;
1777
1778                 rcu_read_lock();
1779                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res) == 0)
1780                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1781                 else
1782                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1783                                         RT_SCOPE_UNIVERSE);
1784                 rcu_read_unlock();
1785         }
1786         memcpy(addr, &src, 4);
1787 }
1788
1789 #ifdef CONFIG_IP_ROUTE_CLASSID
1790 static void set_class_tag(struct rtable *rt, u32 tag)
1791 {
1792         if (!(rt->dst.tclassid & 0xFFFF))
1793                 rt->dst.tclassid |= tag & 0xFFFF;
1794         if (!(rt->dst.tclassid & 0xFFFF0000))
1795                 rt->dst.tclassid |= tag & 0xFFFF0000;
1796 }
1797 #endif
1798
1799 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1800 {
1801         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1802
1803         if (advmss == 0) {
1804                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1805                                ip_rt_min_advmss);
1806                 if (advmss > 65535 - 40)
1807                         advmss = 65535 - 40;
1808         }
1809         return advmss;
1810 }
1811
1812 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1813 {
1814         const struct rtable *rt = (const struct rtable *) dst;
1815         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
1816
1817         if (mtu && rt_is_output_route(rt))
1818                 return mtu;
1819
1820         mtu = dst->dev->mtu;
1821
1822         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1823
1824                 if (rt->rt_gateway != rt->rt_dst && mtu > 576)
1825                         mtu = 576;
1826         }
1827
1828         if (mtu > IP_MAX_MTU)
1829                 mtu = IP_MAX_MTU;
1830
1831         return mtu;
1832 }
1833
1834 static void rt_init_metrics(struct rtable *rt, const struct flowi4 *fl4,
1835                             struct fib_info *fi)
1836 {
1837         struct inet_peer_base *base;
1838         struct inet_peer *peer;
1839
1840         base = inetpeer_base_ptr(rt->_peer);
1841         BUG_ON(!base);
1842
1843         peer = inet_getpeer_v4(base, rt->rt_dst, 0);
1844         if (peer) {
1845                 __rt_set_peer(rt, peer);
1846                 rt->rt_peer_genid = rt_peer_genid();
1847                 if (inet_metrics_new(peer))
1848                         memcpy(peer->metrics, fi->fib_metrics,
1849                                sizeof(u32) * RTAX_MAX);
1850                 dst_init_metrics(&rt->dst, peer->metrics, false);
1851
1852                 check_peer_pmtu(&rt->dst, peer);
1853
1854                 if (peer->redirect_learned.a4 &&
1855                     peer->redirect_learned.a4 != rt->rt_gateway) {
1856                         rt->rt_gateway = peer->redirect_learned.a4;
1857                         rt->rt_flags |= RTCF_REDIRECTED;
1858                 }
1859         } else {
1860                 if (fi->fib_metrics != (u32 *) dst_default_metrics) {
1861                         rt->fi = fi;
1862                         atomic_inc(&fi->fib_clntref);
1863                 }
1864                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1865         }
1866 }
1867
1868 static void rt_set_nexthop(struct rtable *rt, const struct flowi4 *fl4,
1869                            const struct fib_result *res,
1870                            struct fib_info *fi, u16 type, u32 itag)
1871 {
1872         struct dst_entry *dst = &rt->dst;
1873
1874         if (fi) {
1875                 if (FIB_RES_GW(*res) &&
1876                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1877                         rt->rt_gateway = FIB_RES_GW(*res);
1878                 rt_init_metrics(rt, fl4, fi);
1879 #ifdef CONFIG_IP_ROUTE_CLASSID
1880                 dst->tclassid = FIB_RES_NH(*res).nh_tclassid;
1881 #endif
1882         }
1883
1884         if (dst_mtu(dst) > IP_MAX_MTU)
1885                 dst_metric_set(dst, RTAX_MTU, IP_MAX_MTU);
1886
1887 #ifdef CONFIG_IP_ROUTE_CLASSID
1888 #ifdef CONFIG_IP_MULTIPLE_TABLES
1889         set_class_tag(rt, fib_rules_tclass(res));
1890 #endif
1891         set_class_tag(rt, itag);
1892 #endif
1893 }
1894
1895 static struct rtable *rt_dst_alloc(struct net_device *dev,
1896                                    bool nopolicy, bool noxfrm)
1897 {
1898         return dst_alloc(&ipv4_dst_ops, dev, 1, -1,
1899                          DST_HOST |
1900                          (nopolicy ? DST_NOPOLICY : 0) |
1901                          (noxfrm ? DST_NOXFRM : 0));
1902 }
1903
1904 /* called in rcu_read_lock() section */
1905 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1906                                 u8 tos, struct net_device *dev, int our)
1907 {
1908         unsigned int hash;
1909         struct rtable *rth;
1910         struct in_device *in_dev = __in_dev_get_rcu(dev);
1911         u32 itag = 0;
1912         int err;
1913
1914         /* Primary sanity checks. */
1915
1916         if (in_dev == NULL)
1917                 return -EINVAL;
1918
1919         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1920             skb->protocol != htons(ETH_P_IP))
1921                 goto e_inval;
1922
1923         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
1924                 if (ipv4_is_loopback(saddr))
1925                         goto e_inval;
1926
1927         if (ipv4_is_zeronet(saddr)) {
1928                 if (!ipv4_is_local_multicast(daddr))
1929                         goto e_inval;
1930         } else {
1931                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1932                                           in_dev, &itag);
1933                 if (err < 0)
1934                         goto e_err;
1935         }
1936         rth = rt_dst_alloc(dev_net(dev)->loopback_dev,
1937                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
1938         if (!rth)
1939                 goto e_nobufs;
1940
1941 #ifdef CONFIG_IP_ROUTE_CLASSID
1942         rth->dst.tclassid = itag;
1943 #endif
1944         rth->dst.output = ip_rt_bug;
1945
1946         rth->rt_key_dst = daddr;
1947         rth->rt_key_src = saddr;
1948         rth->rt_genid   = rt_genid(dev_net(dev));
1949         rth->rt_flags   = RTCF_MULTICAST;
1950         rth->rt_type    = RTN_MULTICAST;
1951         rth->rt_key_tos = tos;
1952         rth->rt_dst     = daddr;
1953         rth->rt_src     = saddr;
1954         rth->rt_route_iif = dev->ifindex;
1955         rth->rt_iif     = dev->ifindex;
1956         rth->rt_oif     = 0;
1957         rth->rt_mark    = skb->mark;
1958         rth->rt_gateway = daddr;
1959         rth->rt_peer_genid = 0;
1960         rt_init_peer(rth, dev_net(dev)->ipv4.peers);
1961         rth->fi = NULL;
1962         if (our) {
1963                 rth->dst.input= ip_local_deliver;
1964                 rth->rt_flags |= RTCF_LOCAL;
1965         }
1966
1967 #ifdef CONFIG_IP_MROUTE
1968         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1969                 rth->dst.input = ip_mr_input;
1970 #endif
1971         RT_CACHE_STAT_INC(in_slow_mc);
1972
1973         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1974         rth = rt_intern_hash(hash, rth, skb, dev->ifindex);
1975         return IS_ERR(rth) ? PTR_ERR(rth) : 0;
1976
1977 e_nobufs:
1978         return -ENOBUFS;
1979 e_inval:
1980         return -EINVAL;
1981 e_err:
1982         return err;
1983 }
1984
1985
1986 static void ip_handle_martian_source(struct net_device *dev,
1987                                      struct in_device *in_dev,
1988                                      struct sk_buff *skb,
1989                                      __be32 daddr,
1990                                      __be32 saddr)
1991 {
1992         RT_CACHE_STAT_INC(in_martian_src);
1993 #ifdef CONFIG_IP_ROUTE_VERBOSE
1994         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1995                 /*
1996                  *      RFC1812 recommendation, if source is martian,
1997                  *      the only hint is MAC header.
1998                  */
1999                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
2000                         &daddr, &saddr, dev->name);
2001                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
2002                         print_hex_dump(KERN_WARNING, "ll header: ",
2003                                        DUMP_PREFIX_OFFSET, 16, 1,
2004                                        skb_mac_header(skb),
2005                                        dev->hard_header_len, true);
2006                 }
2007         }
2008 #endif
2009 }
2010
2011 /* called in rcu_read_lock() section */
2012 static int __mkroute_input(struct sk_buff *skb,
2013                            const struct fib_result *res,
2014                            struct in_device *in_dev,
2015                            __be32 daddr, __be32 saddr, u32 tos,
2016                            struct rtable **result)
2017 {
2018         struct rtable *rth;
2019         int err;
2020         struct in_device *out_dev;
2021         unsigned int flags = 0;
2022         u32 itag;
2023
2024         /* get a working reference to the output device */
2025         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
2026         if (out_dev == NULL) {
2027                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
2028                 return -EINVAL;
2029         }
2030
2031
2032         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
2033                                   in_dev->dev, in_dev, &itag);
2034         if (err < 0) {
2035                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
2036                                          saddr);
2037
2038                 goto cleanup;
2039         }
2040
2041         if (err)
2042                 flags |= RTCF_DIRECTSRC;
2043
2044         if (out_dev == in_dev && err &&
2045             (IN_DEV_SHARED_MEDIA(out_dev) ||
2046              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
2047                 flags |= RTCF_DOREDIRECT;
2048
2049         if (skb->protocol != htons(ETH_P_IP)) {
2050                 /* Not IP (i.e. ARP). Do not create route, if it is
2051                  * invalid for proxy arp. DNAT routes are always valid.
2052                  *
2053                  * Proxy arp feature have been extended to allow, ARP
2054                  * replies back to the same interface, to support
2055                  * Private VLAN switch technologies. See arp.c.
2056                  */
2057                 if (out_dev == in_dev &&
2058                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2059                         err = -EINVAL;
2060                         goto cleanup;
2061                 }
2062         }
2063
2064         rth = rt_dst_alloc(out_dev->dev,
2065                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2066                            IN_DEV_CONF_GET(out_dev, NOXFRM));
2067         if (!rth) {
2068                 err = -ENOBUFS;
2069                 goto cleanup;
2070         }
2071
2072         rth->rt_key_dst = daddr;
2073         rth->rt_key_src = saddr;
2074         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2075         rth->rt_flags = flags;
2076         rth->rt_type = res->type;
2077         rth->rt_key_tos = tos;
2078         rth->rt_dst     = daddr;
2079         rth->rt_src     = saddr;
2080         rth->rt_route_iif = in_dev->dev->ifindex;
2081         rth->rt_iif     = in_dev->dev->ifindex;
2082         rth->rt_oif     = 0;
2083         rth->rt_mark    = skb->mark;
2084         rth->rt_gateway = daddr;
2085         rth->rt_peer_genid = 0;
2086         rt_init_peer(rth, &res->table->tb_peers);
2087         rth->fi = NULL;
2088
2089         rth->dst.input = ip_forward;
2090         rth->dst.output = ip_output;
2091
2092         rt_set_nexthop(rth, NULL, res, res->fi, res->type, itag);
2093
2094         *result = rth;
2095         err = 0;
2096  cleanup:
2097         return err;
2098 }
2099
2100 static int ip_mkroute_input(struct sk_buff *skb,
2101                             struct fib_result *res,
2102                             const struct flowi4 *fl4,
2103                             struct in_device *in_dev,
2104                             __be32 daddr, __be32 saddr, u32 tos)
2105 {
2106         struct rtable *rth = NULL;
2107         int err;
2108         unsigned int hash;
2109
2110 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2111         if (res->fi && res->fi->fib_nhs > 1)
2112                 fib_select_multipath(res);
2113 #endif
2114
2115         /* create a routing cache entry */
2116         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2117         if (err)
2118                 return err;
2119
2120         /* put it into the cache */
2121         hash = rt_hash(daddr, saddr, fl4->flowi4_iif,
2122                        rt_genid(dev_net(rth->dst.dev)));
2123         rth = rt_intern_hash(hash, rth, skb, fl4->flowi4_iif);
2124         if (IS_ERR(rth))
2125                 return PTR_ERR(rth);
2126         return 0;
2127 }
2128
2129 /*
2130  *      NOTE. We drop all the packets that has local source
2131  *      addresses, because every properly looped back packet
2132  *      must have correct destination already attached by output routine.
2133  *
2134  *      Such approach solves two big problems:
2135  *      1. Not simplex devices are handled properly.
2136  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2137  *      called with rcu_read_lock()
2138  */
2139
2140 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2141                                u8 tos, struct net_device *dev)
2142 {
2143         struct fib_result res;
2144         struct in_device *in_dev = __in_dev_get_rcu(dev);
2145         struct flowi4   fl4;
2146         unsigned int    flags = 0;
2147         u32             itag = 0;
2148         struct rtable   *rth;
2149         unsigned int    hash;
2150         int             err = -EINVAL;
2151         struct net    *net = dev_net(dev);
2152
2153         /* IP on this device is disabled. */
2154
2155         if (!in_dev)
2156                 goto out;
2157
2158         /* Check for the most weird martians, which can be not detected
2159            by fib_lookup.
2160          */
2161
2162         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
2163                 goto martian_source;
2164
2165         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
2166                 goto brd_input;
2167
2168         /* Accept zero addresses only to limited broadcast;
2169          * I even do not know to fix it or not. Waiting for complains :-)
2170          */
2171         if (ipv4_is_zeronet(saddr))
2172                 goto martian_source;
2173
2174         if (ipv4_is_zeronet(daddr))
2175                 goto martian_destination;
2176
2177         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev))) {
2178                 if (ipv4_is_loopback(daddr))
2179                         goto martian_destination;
2180
2181                 if (ipv4_is_loopback(saddr))
2182                         goto martian_source;
2183         }
2184
2185         /*
2186          *      Now we are ready to route packet.
2187          */
2188         fl4.flowi4_oif = 0;
2189         fl4.flowi4_iif = dev->ifindex;
2190         fl4.flowi4_mark = skb->mark;
2191         fl4.flowi4_tos = tos;
2192         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
2193         fl4.daddr = daddr;
2194         fl4.saddr = saddr;
2195         err = fib_lookup(net, &fl4, &res);
2196         if (err != 0)
2197                 goto no_route;
2198
2199         RT_CACHE_STAT_INC(in_slow_tot);
2200
2201         if (res.type == RTN_BROADCAST)
2202                 goto brd_input;
2203
2204         if (res.type == RTN_LOCAL) {
2205                 err = fib_validate_source(skb, saddr, daddr, tos,
2206                                           net->loopback_dev->ifindex,
2207                                           dev, in_dev, &itag);
2208                 if (err < 0)
2209                         goto martian_source_keep_err;
2210                 if (err)
2211                         flags |= RTCF_DIRECTSRC;
2212                 goto local_input;
2213         }
2214
2215         if (!IN_DEV_FORWARD(in_dev))
2216                 goto no_route;
2217         if (res.type != RTN_UNICAST)
2218                 goto martian_destination;
2219
2220         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
2221 out:    return err;
2222
2223 brd_input:
2224         if (skb->protocol != htons(ETH_P_IP))
2225                 goto e_inval;
2226
2227         if (!ipv4_is_zeronet(saddr)) {
2228                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
2229                                           in_dev, &itag);
2230                 if (err < 0)
2231                         goto martian_source_keep_err;
2232                 if (err)
2233                         flags |= RTCF_DIRECTSRC;
2234         }
2235         flags |= RTCF_BROADCAST;
2236         res.type = RTN_BROADCAST;
2237         RT_CACHE_STAT_INC(in_brd);
2238
2239 local_input:
2240         rth = rt_dst_alloc(net->loopback_dev,
2241                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false);
2242         if (!rth)
2243                 goto e_nobufs;
2244
2245         rth->dst.input= ip_local_deliver;
2246         rth->dst.output= ip_rt_bug;
2247 #ifdef CONFIG_IP_ROUTE_CLASSID
2248         rth->dst.tclassid = itag;
2249 #endif
2250
2251         rth->rt_key_dst = daddr;
2252         rth->rt_key_src = saddr;
2253         rth->rt_genid = rt_genid(net);
2254         rth->rt_flags   = flags|RTCF_LOCAL;
2255         rth->rt_type    = res.type;
2256         rth->rt_key_tos = tos;
2257         rth->rt_dst     = daddr;
2258         rth->rt_src     = saddr;
2259         rth->rt_route_iif = dev->ifindex;
2260         rth->rt_iif     = dev->ifindex;
2261         rth->rt_oif     = 0;
2262         rth->rt_mark    = skb->mark;
2263         rth->rt_gateway = daddr;
2264         rth->rt_peer_genid = 0;
2265         rt_init_peer(rth, net->ipv4.peers);
2266         rth->fi = NULL;
2267         if (res.type == RTN_UNREACHABLE) {
2268                 rth->dst.input= ip_error;
2269                 rth->dst.error= -err;
2270                 rth->rt_flags   &= ~RTCF_LOCAL;
2271         }
2272         hash = rt_hash(daddr, saddr, fl4.flowi4_iif, rt_genid(net));
2273         rth = rt_intern_hash(hash, rth, skb, fl4.flowi4_iif);
2274         err = 0;
2275         if (IS_ERR(rth))
2276                 err = PTR_ERR(rth);
2277         goto out;
2278
2279 no_route:
2280         RT_CACHE_STAT_INC(in_no_route);
2281         res.type = RTN_UNREACHABLE;
2282         if (err == -ESRCH)
2283                 err = -ENETUNREACH;
2284         goto local_input;
2285
2286         /*
2287          *      Do not cache martian addresses: they should be logged (RFC1812)
2288          */
2289 martian_destination:
2290         RT_CACHE_STAT_INC(in_martian_dst);
2291 #ifdef CONFIG_IP_ROUTE_VERBOSE
2292         if (IN_DEV_LOG_MARTIANS(in_dev))
2293                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
2294                                      &daddr, &saddr, dev->name);
2295 #endif
2296
2297 e_inval:
2298         err = -EINVAL;
2299         goto out;
2300
2301 e_nobufs:
2302         err = -ENOBUFS;
2303         goto out;
2304
2305 martian_source:
2306         err = -EINVAL;
2307 martian_source_keep_err:
2308         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2309         goto out;
2310 }
2311
2312 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2313                            u8 tos, struct net_device *dev, bool noref)
2314 {
2315         struct rtable   *rth;
2316         unsigned int    hash;
2317         int iif = dev->ifindex;
2318         struct net *net;
2319         int res;
2320
2321         net = dev_net(dev);
2322
2323         rcu_read_lock();
2324
2325         if (!rt_caching(net))
2326                 goto skip_cache;
2327
2328         tos &= IPTOS_RT_MASK;
2329         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2330
2331         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2332              rth = rcu_dereference(rth->dst.rt_next)) {
2333                 if ((((__force u32)rth->rt_key_dst ^ (__force u32)daddr) |
2334                      ((__force u32)rth->rt_key_src ^ (__force u32)saddr) |
2335                      (rth->rt_route_iif ^ iif) |
2336                      (rth->rt_key_tos ^ tos)) == 0 &&
2337                     rth->rt_mark == skb->mark &&
2338                     net_eq(dev_net(rth->dst.dev), net) &&
2339                     !rt_is_expired(rth)) {
2340                         ipv4_validate_peer(rth);
2341                         if (noref) {
2342                                 dst_use_noref(&rth->dst, jiffies);
2343                                 skb_dst_set_noref(skb, &rth->dst);
2344                         } else {
2345                                 dst_use(&rth->dst, jiffies);
2346                                 skb_dst_set(skb, &rth->dst);
2347                         }
2348                         RT_CACHE_STAT_INC(in_hit);
2349                         rcu_read_unlock();
2350                         return 0;
2351                 }
2352                 RT_CACHE_STAT_INC(in_hlist_search);
2353         }
2354
2355 skip_cache:
2356         /* Multicast recognition logic is moved from route cache to here.
2357            The problem was that too many Ethernet cards have broken/missing
2358            hardware multicast filters :-( As result the host on multicasting
2359            network acquires a lot of useless route cache entries, sort of
2360            SDR messages from all the world. Now we try to get rid of them.
2361            Really, provided software IP multicast filter is organized
2362            reasonably (at least, hashed), it does not result in a slowdown
2363            comparing with route cache reject entries.
2364            Note, that multicast routers are not affected, because
2365            route cache entry is created eventually.
2366          */
2367         if (ipv4_is_multicast(daddr)) {
2368                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2369
2370                 if (in_dev) {
2371                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
2372                                                   ip_hdr(skb)->protocol);
2373                         if (our
2374 #ifdef CONFIG_IP_MROUTE
2375                                 ||
2376                             (!ipv4_is_local_multicast(daddr) &&
2377                              IN_DEV_MFORWARD(in_dev))
2378 #endif
2379                            ) {
2380                                 int res = ip_route_input_mc(skb, daddr, saddr,
2381                                                             tos, dev, our);
2382                                 rcu_read_unlock();
2383                                 return res;
2384                         }
2385                 }
2386                 rcu_read_unlock();
2387                 return -EINVAL;
2388         }
2389         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2390         rcu_read_unlock();
2391         return res;
2392 }
2393 EXPORT_SYMBOL(ip_route_input_common);
2394
2395 /* called with rcu_read_lock() */
2396 static struct rtable *__mkroute_output(const struct fib_result *res,
2397                                        const struct flowi4 *fl4,
2398                                        __be32 orig_daddr, __be32 orig_saddr,
2399                                        int orig_oif, __u8 orig_rtos,
2400                                        struct net_device *dev_out,
2401                                        unsigned int flags)
2402 {
2403         struct fib_info *fi = res->fi;
2404         struct in_device *in_dev;
2405         u16 type = res->type;
2406         struct rtable *rth;
2407
2408         in_dev = __in_dev_get_rcu(dev_out);
2409         if (!in_dev)
2410                 return ERR_PTR(-EINVAL);
2411
2412         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2413                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2414                         return ERR_PTR(-EINVAL);
2415
2416         if (ipv4_is_lbcast(fl4->daddr))
2417                 type = RTN_BROADCAST;
2418         else if (ipv4_is_multicast(fl4->daddr))
2419                 type = RTN_MULTICAST;
2420         else if (ipv4_is_zeronet(fl4->daddr))
2421                 return ERR_PTR(-EINVAL);
2422
2423         if (dev_out->flags & IFF_LOOPBACK)
2424                 flags |= RTCF_LOCAL;
2425
2426         if (type == RTN_BROADCAST) {
2427                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2428                 fi = NULL;
2429         } else if (type == RTN_MULTICAST) {
2430                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2431                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2432                                      fl4->flowi4_proto))
2433                         flags &= ~RTCF_LOCAL;
2434                 /* If multicast route do not exist use
2435                  * default one, but do not gateway in this case.
2436                  * Yes, it is hack.
2437                  */
2438                 if (fi && res->prefixlen < 4)
2439                         fi = NULL;
2440         }
2441
2442         rth = rt_dst_alloc(dev_out,
2443                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2444                            IN_DEV_CONF_GET(in_dev, NOXFRM));
2445         if (!rth)
2446                 return ERR_PTR(-ENOBUFS);
2447
2448         rth->dst.output = ip_output;
2449
2450         rth->rt_key_dst = orig_daddr;
2451         rth->rt_key_src = orig_saddr;
2452         rth->rt_genid = rt_genid(dev_net(dev_out));
2453         rth->rt_flags   = flags;
2454         rth->rt_type    = type;
2455         rth->rt_key_tos = orig_rtos;
2456         rth->rt_dst     = fl4->daddr;
2457         rth->rt_src     = fl4->saddr;
2458         rth->rt_route_iif = 0;
2459         rth->rt_iif     = orig_oif ? : dev_out->ifindex;
2460         rth->rt_oif     = orig_oif;
2461         rth->rt_mark    = fl4->flowi4_mark;
2462         rth->rt_gateway = fl4->daddr;
2463         rth->rt_peer_genid = 0;
2464         rt_init_peer(rth, (res->table ?
2465                            &res->table->tb_peers :
2466                            dev_net(dev_out)->ipv4.peers));
2467         rth->fi = NULL;
2468
2469         RT_CACHE_STAT_INC(out_slow_tot);
2470
2471         if (flags & RTCF_LOCAL)
2472                 rth->dst.input = ip_local_deliver;
2473         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2474                 if (flags & RTCF_LOCAL &&
2475                     !(dev_out->flags & IFF_LOOPBACK)) {
2476                         rth->dst.output = ip_mc_output;
2477                         RT_CACHE_STAT_INC(out_slow_mc);
2478                 }
2479 #ifdef CONFIG_IP_MROUTE
2480                 if (type == RTN_MULTICAST) {
2481                         if (IN_DEV_MFORWARD(in_dev) &&
2482                             !ipv4_is_local_multicast(fl4->daddr)) {
2483                                 rth->dst.input = ip_mr_input;
2484                                 rth->dst.output = ip_mc_output;
2485                         }
2486                 }
2487 #endif
2488         }
2489
2490         rt_set_nexthop(rth, fl4, res, fi, type, 0);
2491
2492         if (fl4->flowi4_flags & FLOWI_FLAG_RT_NOCACHE)
2493                 rth->dst.flags |= DST_NOCACHE;
2494
2495         return rth;
2496 }
2497
2498 /*
2499  * Major route resolver routine.
2500  * called with rcu_read_lock();
2501  */
2502
2503 static struct rtable *ip_route_output_slow(struct net *net, struct flowi4 *fl4)
2504 {
2505         struct net_device *dev_out = NULL;
2506         __u8 tos = RT_FL_TOS(fl4);
2507         unsigned int flags = 0;
2508         struct fib_result res;
2509         struct rtable *rth;
2510         __be32 orig_daddr;
2511         __be32 orig_saddr;
2512         int orig_oif;
2513
2514         res.fi          = NULL;
2515         res.table       = NULL;
2516 #ifdef CONFIG_IP_MULTIPLE_TABLES
2517         res.r           = NULL;
2518 #endif
2519
2520         orig_daddr = fl4->daddr;
2521         orig_saddr = fl4->saddr;
2522         orig_oif = fl4->flowi4_oif;
2523
2524         fl4->flowi4_iif = net->loopback_dev->ifindex;
2525         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2526         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2527                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2528
2529         rcu_read_lock();
2530         if (fl4->saddr) {
2531                 rth = ERR_PTR(-EINVAL);
2532                 if (ipv4_is_multicast(fl4->saddr) ||
2533                     ipv4_is_lbcast(fl4->saddr) ||
2534                     ipv4_is_zeronet(fl4->saddr))
2535                         goto out;
2536
2537                 /* I removed check for oif == dev_out->oif here.
2538                    It was wrong for two reasons:
2539                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2540                       is assigned to multiple interfaces.
2541                    2. Moreover, we are allowed to send packets with saddr
2542                       of another iface. --ANK
2543                  */
2544
2545                 if (fl4->flowi4_oif == 0 &&
2546                     (ipv4_is_multicast(fl4->daddr) ||
2547                      ipv4_is_lbcast(fl4->daddr))) {
2548                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2549                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2550                         if (dev_out == NULL)
2551                                 goto out;
2552
2553                         /* Special hack: user can direct multicasts
2554                            and limited broadcast via necessary interface
2555                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2556                            This hack is not just for fun, it allows
2557                            vic,vat and friends to work.
2558                            They bind socket to loopback, set ttl to zero
2559                            and expect that it will work.
2560                            From the viewpoint of routing cache they are broken,
2561                            because we are not allowed to build multicast path
2562                            with loopback source addr (look, routing cache
2563                            cannot know, that ttl is zero, so that packet
2564                            will not leave this host and route is valid).
2565                            Luckily, this hack is good workaround.
2566                          */
2567
2568                         fl4->flowi4_oif = dev_out->ifindex;
2569                         goto make_route;
2570                 }
2571
2572                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2573                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2574                         if (!__ip_dev_find(net, fl4->saddr, false))
2575                                 goto out;
2576                 }
2577         }
2578
2579
2580         if (fl4->flowi4_oif) {
2581                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2582                 rth = ERR_PTR(-ENODEV);
2583                 if (dev_out == NULL)
2584                         goto out;
2585
2586                 /* RACE: Check return value of inet_select_addr instead. */
2587                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2588                         rth = ERR_PTR(-ENETUNREACH);
2589                         goto out;
2590                 }
2591                 if (ipv4_is_local_multicast(fl4->daddr) ||
2592                     ipv4_is_lbcast(fl4->daddr)) {
2593                         if (!fl4->saddr)
2594                                 fl4->saddr = inet_select_addr(dev_out, 0,
2595                                                               RT_SCOPE_LINK);
2596                         goto make_route;
2597                 }
2598                 if (fl4->saddr) {
2599                         if (ipv4_is_multicast(fl4->daddr))
2600                                 fl4->saddr = inet_select_addr(dev_out, 0,
2601                                                               fl4->flowi4_scope);
2602                         else if (!fl4->daddr)
2603                                 fl4->saddr = inet_select_addr(dev_out, 0,
2604                                                               RT_SCOPE_HOST);
2605                 }
2606         }
2607
2608         if (!fl4->daddr) {
2609                 fl4->daddr = fl4->saddr;
2610                 if (!fl4->daddr)
2611                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2612                 dev_out = net->loopback_dev;
2613                 fl4->flowi4_oif = net->loopback_dev->ifindex;
2614                 res.type = RTN_LOCAL;
2615                 flags |= RTCF_LOCAL;
2616                 goto make_route;
2617         }
2618
2619         if (fib_lookup(net, fl4, &res)) {
2620                 res.fi = NULL;
2621                 res.table = NULL;
2622                 if (fl4->flowi4_oif) {
2623                         /* Apparently, routing tables are wrong. Assume,
2624                            that the destination is on link.
2625
2626                            WHY? DW.
2627                            Because we are allowed to send to iface
2628                            even if it has NO routes and NO assigned
2629                            addresses. When oif is specified, routing
2630                            tables are looked up with only one purpose:
2631                            to catch if destination is gatewayed, rather than
2632                            direct. Moreover, if MSG_DONTROUTE is set,
2633                            we send packet, ignoring both routing tables
2634                            and ifaddr state. --ANK
2635
2636
2637                            We could make it even if oif is unknown,
2638                            likely IPv6, but we do not.
2639                          */
2640
2641                         if (fl4->saddr == 0)
2642                                 fl4->saddr = inet_select_addr(dev_out, 0,
2643                                                               RT_SCOPE_LINK);
2644                         res.type = RTN_UNICAST;
2645                         goto make_route;
2646                 }
2647                 rth = ERR_PTR(-ENETUNREACH);
2648                 goto out;
2649         }
2650
2651         if (res.type == RTN_LOCAL) {
2652                 if (!fl4->saddr) {
2653                         if (res.fi->fib_prefsrc)
2654                                 fl4->saddr = res.fi->fib_prefsrc;
2655                         else
2656                                 fl4->saddr = fl4->daddr;
2657                 }
2658                 dev_out = net->loopback_dev;
2659                 fl4->flowi4_oif = dev_out->ifindex;
2660                 res.fi = NULL;
2661                 flags |= RTCF_LOCAL;
2662                 goto make_route;
2663         }
2664
2665 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2666         if (res.fi->fib_nhs > 1 && fl4->flowi4_oif == 0)
2667                 fib_select_multipath(&res);
2668         else
2669 #endif
2670         if (!res.prefixlen &&
2671             res.table->tb_num_default > 1 &&
2672             res.type == RTN_UNICAST && !fl4->flowi4_oif)
2673                 fib_select_default(&res);
2674
2675         if (!fl4->saddr)
2676                 fl4->saddr = FIB_RES_PREFSRC(net, res);
2677
2678         dev_out = FIB_RES_DEV(res);
2679         fl4->flowi4_oif = dev_out->ifindex;
2680
2681
2682 make_route:
2683         rth = __mkroute_output(&res, fl4, orig_daddr, orig_saddr, orig_oif,
2684                                tos, dev_out, flags);
2685         if (!IS_ERR(rth)) {
2686                 unsigned int hash;
2687
2688                 hash = rt_hash(orig_daddr, orig_saddr, orig_oif,
2689                                rt_genid(dev_net(dev_out)));
2690                 rth = rt_intern_hash(hash, rth, NULL, orig_oif);
2691         }
2692
2693 out:
2694         rcu_read_unlock();
2695         return rth;
2696 }
2697
2698 struct rtable *__ip_route_output_key(struct net *net, struct flowi4 *flp4)
2699 {
2700         struct rtable *rth;
2701         unsigned int hash;
2702
2703         if (!rt_caching(net))
2704                 goto slow_output;
2705
2706         hash = rt_hash(flp4->daddr, flp4->saddr, flp4->flowi4_oif, rt_genid(net));
2707
2708         rcu_read_lock_bh();
2709         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2710                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2711                 if (rth->rt_key_dst == flp4->daddr &&
2712                     rth->rt_key_src == flp4->saddr &&
2713                     rt_is_output_route(rth) &&
2714                     rth->rt_oif == flp4->flowi4_oif &&
2715                     rth->rt_mark == flp4->flowi4_mark &&
2716                     !((rth->rt_key_tos ^ flp4->flowi4_tos) &
2717                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2718                     net_eq(dev_net(rth->dst.dev), net) &&
2719                     !rt_is_expired(rth)) {
2720                         ipv4_validate_peer(rth);
2721                         dst_use(&rth->dst, jiffies);
2722                         RT_CACHE_STAT_INC(out_hit);
2723                         rcu_read_unlock_bh();
2724                         if (!flp4->saddr)
2725                                 flp4->saddr = rth->rt_src;
2726                         if (!flp4->daddr)
2727                                 flp4->daddr = rth->rt_dst;
2728                         return rth;
2729                 }
2730                 RT_CACHE_STAT_INC(out_hlist_search);
2731         }
2732         rcu_read_unlock_bh();
2733
2734 slow_output:
2735         return ip_route_output_slow(net, flp4);
2736 }
2737 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2738
2739 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2740 {
2741         return NULL;
2742 }
2743
2744 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2745 {
2746         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2747
2748         return mtu ? : dst->dev->mtu;
2749 }
2750
2751 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2752 {
2753 }
2754
2755 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2756                                           unsigned long old)
2757 {
2758         return NULL;
2759 }
2760
2761 static struct dst_ops ipv4_dst_blackhole_ops = {
2762         .family                 =       AF_INET,
2763         .protocol               =       cpu_to_be16(ETH_P_IP),
2764         .destroy                =       ipv4_dst_destroy,
2765         .check                  =       ipv4_blackhole_dst_check,
2766         .mtu                    =       ipv4_blackhole_mtu,
2767         .default_advmss         =       ipv4_default_advmss,
2768         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2769         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2770         .neigh_lookup           =       ipv4_neigh_lookup,
2771 };
2772
2773 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2774 {
2775         struct rtable *rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, 0, 0);
2776         struct rtable *ort = (struct rtable *) dst_orig;
2777
2778         if (rt) {
2779                 struct dst_entry *new = &rt->dst;
2780
2781                 new->__use = 1;
2782                 new->input = dst_discard;
2783                 new->output = dst_discard;
2784                 dst_copy_metrics(new, &ort->dst);
2785
2786                 new->dev = ort->dst.dev;
2787                 if (new->dev)
2788                         dev_hold(new->dev);
2789
2790                 rt->rt_key_dst = ort->rt_key_dst;
2791                 rt->rt_key_src = ort->rt_key_src;
2792                 rt->rt_key_tos = ort->rt_key_tos;
2793                 rt->rt_route_iif = ort->rt_route_iif;
2794                 rt->rt_iif = ort->rt_iif;
2795                 rt->rt_oif = ort->rt_oif;
2796                 rt->rt_mark = ort->rt_mark;
2797
2798                 rt->rt_genid = rt_genid(net);
2799                 rt->rt_flags = ort->rt_flags;
2800                 rt->rt_type = ort->rt_type;
2801                 rt->rt_dst = ort->rt_dst;
2802                 rt->rt_src = ort->rt_src;
2803                 rt->rt_gateway = ort->rt_gateway;
2804                 rt_transfer_peer(rt, ort);
2805                 rt->fi = ort->fi;
2806                 if (rt->fi)
2807                         atomic_inc(&rt->fi->fib_clntref);
2808
2809                 dst_free(new);
2810         }
2811
2812         dst_release(dst_orig);
2813
2814         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2815 }
2816
2817 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2818                                     struct sock *sk)
2819 {
2820         struct rtable *rt = __ip_route_output_key(net, flp4);
2821
2822         if (IS_ERR(rt))
2823                 return rt;
2824
2825         if (flp4->flowi4_proto)
2826                 rt = (struct rtable *) xfrm_lookup(net, &rt->dst,
2827                                                    flowi4_to_flowi(flp4),
2828                                                    sk, 0);
2829
2830         return rt;
2831 }
2832 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2833
2834 static int rt_fill_info(struct net *net,
2835                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2836                         int nowait, unsigned int flags)
2837 {
2838         struct rtable *rt = skb_rtable(skb);
2839         struct rtmsg *r;
2840         struct nlmsghdr *nlh;
2841         unsigned long expires = 0;
2842         u32 id = 0, error;
2843
2844         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2845         if (nlh == NULL)
2846                 return -EMSGSIZE;
2847
2848         r = nlmsg_data(nlh);
2849         r->rtm_family    = AF_INET;
2850         r->rtm_dst_len  = 32;
2851         r->rtm_src_len  = 0;
2852         r->rtm_tos      = rt->rt_key_tos;
2853         r->rtm_table    = RT_TABLE_MAIN;
2854         if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN))
2855                 goto nla_put_failure;
2856         r->rtm_type     = rt->rt_type;
2857         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2858         r->rtm_protocol = RTPROT_UNSPEC;
2859         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2860         if (rt->rt_flags & RTCF_NOTIFY)
2861                 r->rtm_flags |= RTM_F_NOTIFY;
2862
2863         if (nla_put_be32(skb, RTA_DST, rt->rt_dst))
2864                 goto nla_put_failure;
2865         if (rt->rt_key_src) {
2866                 r->rtm_src_len = 32;
2867                 if (nla_put_be32(skb, RTA_SRC, rt->rt_key_src))
2868                         goto nla_put_failure;
2869         }
2870         if (rt->dst.dev &&
2871             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2872                 goto nla_put_failure;
2873 #ifdef CONFIG_IP_ROUTE_CLASSID
2874         if (rt->dst.tclassid &&
2875             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2876                 goto nla_put_failure;
2877 #endif
2878         if (!rt_is_input_route(rt) &&
2879             rt->rt_src != rt->rt_key_src) {
2880                 if (nla_put_be32(skb, RTA_PREFSRC, rt->rt_src))
2881                         goto nla_put_failure;
2882         }
2883         if (rt->rt_dst != rt->rt_gateway &&
2884             nla_put_be32(skb, RTA_GATEWAY, rt->rt_gateway))
2885                 goto nla_put_failure;
2886
2887         if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
2888                 goto nla_put_failure;
2889
2890         if (rt->rt_mark &&
2891             nla_put_be32(skb, RTA_MARK, rt->rt_mark))
2892                 goto nla_put_failure;
2893
2894         error = rt->dst.error;
2895         if (rt_has_peer(rt)) {
2896                 const struct inet_peer *peer = rt_peer_ptr(rt);
2897                 inet_peer_refcheck(peer);
2898                 id = atomic_read(&peer->ip_id_count) & 0xffff;
2899                 expires = ACCESS_ONCE(peer->pmtu_expires);
2900                 if (expires) {
2901                         if (time_before(jiffies, expires))
2902                                 expires -= jiffies;
2903                         else
2904                                 expires = 0;
2905                 }
2906         }
2907
2908         if (rt_is_input_route(rt)) {
2909 #ifdef CONFIG_IP_MROUTE
2910                 __be32 dst = rt->rt_dst;
2911
2912                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2913                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2914                         int err = ipmr_get_route(net, skb,
2915                                                  rt->rt_src, rt->rt_dst,
2916                                                  r, nowait);
2917                         if (err <= 0) {
2918                                 if (!nowait) {
2919                                         if (err == 0)
2920                                                 return 0;
2921                                         goto nla_put_failure;
2922                                 } else {
2923                                         if (err == -EMSGSIZE)
2924                                                 goto nla_put_failure;
2925                                         error = err;
2926                                 }
2927                         }
2928                 } else
2929 #endif
2930                         if (nla_put_u32(skb, RTA_IIF, rt->rt_iif))
2931                                 goto nla_put_failure;
2932         }
2933
2934         if (rtnl_put_cacheinfo(skb, &rt->dst, id, 0, 0,
2935                                expires, error) < 0)
2936                 goto nla_put_failure;
2937
2938         return nlmsg_end(skb, nlh);
2939
2940 nla_put_failure:
2941         nlmsg_cancel(skb, nlh);
2942         return -EMSGSIZE;
2943 }
2944
2945 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, void *arg)
2946 {
2947         struct net *net = sock_net(in_skb->sk);
2948         struct rtmsg *rtm;
2949         struct nlattr *tb[RTA_MAX+1];
2950         struct rtable *rt = NULL;
2951         __be32 dst = 0;
2952         __be32 src = 0;
2953         u32 iif;
2954         int err;
2955         int mark;
2956         struct sk_buff *skb;
2957
2958         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2959         if (err < 0)
2960                 goto errout;
2961
2962         rtm = nlmsg_data(nlh);
2963
2964         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2965         if (skb == NULL) {
2966                 err = -ENOBUFS;
2967                 goto errout;
2968         }
2969
2970         /* Reserve room for dummy headers, this skb can pass
2971            through good chunk of routing engine.
2972          */
2973         skb_reset_mac_header(skb);
2974         skb_reset_network_header(skb);
2975
2976         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2977         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2978         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2979
2980         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2981         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2982         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2983         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2984
2985         if (iif) {
2986                 struct net_device *dev;
2987
2988                 dev = __dev_get_by_index(net, iif);
2989                 if (dev == NULL) {
2990                         err = -ENODEV;
2991                         goto errout_free;
2992                 }
2993
2994                 skb->protocol   = htons(ETH_P_IP);
2995                 skb->dev        = dev;
2996                 skb->mark       = mark;
2997                 local_bh_disable();
2998                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2999                 local_bh_enable();
3000
3001                 rt = skb_rtable(skb);
3002                 if (err == 0 && rt->dst.error)
3003                         err = -rt->dst.error;
3004         } else {
3005                 struct flowi4 fl4 = {
3006                         .daddr = dst,
3007                         .saddr = src,
3008                         .flowi4_tos = rtm->rtm_tos,
3009                         .flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
3010                         .flowi4_mark = mark,
3011                 };
3012                 rt = ip_route_output_key(net, &fl4);
3013
3014                 err = 0;
3015                 if (IS_ERR(rt))
3016                         err = PTR_ERR(rt);
3017         }
3018
3019         if (err)
3020                 goto errout_free;
3021
3022         skb_dst_set(skb, &rt->dst);
3023         if (rtm->rtm_flags & RTM_F_NOTIFY)
3024                 rt->rt_flags |= RTCF_NOTIFY;
3025
3026         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
3027                            RTM_NEWROUTE, 0, 0);
3028         if (err <= 0)
3029                 goto errout_free;
3030
3031         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
3032 errout:
3033         return err;
3034
3035 errout_free:
3036         kfree_skb(skb);
3037         goto errout;
3038 }
3039
3040 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
3041 {
3042         struct rtable *rt;
3043         int h, s_h;
3044         int idx, s_idx;
3045         struct net *net;
3046
3047         net = sock_net(skb->sk);
3048
3049         s_h = cb->args[0];
3050         if (s_h < 0)
3051                 s_h = 0;
3052         s_idx = idx = cb->args[1];
3053         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3054                 if (!rt_hash_table[h].chain)
3055                         continue;
3056                 rcu_read_lock_bh();
3057                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
3058                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3059                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3060                                 continue;
3061                         if (rt_is_expired(rt))
3062                                 continue;
3063                         skb_dst_set_noref(skb, &rt->dst);
3064                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3065                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3066                                          1, NLM_F_MULTI) <= 0) {
3067                                 skb_dst_drop(skb);
3068                                 rcu_read_unlock_bh();
3069                                 goto done;
3070                         }
3071                         skb_dst_drop(skb);
3072                 }
3073                 rcu_read_unlock_bh();
3074         }
3075
3076 done:
3077         cb->args[0] = h;
3078         cb->args[1] = idx;
3079         return skb->len;
3080 }
3081
3082 void ip_rt_multicast_event(struct in_device *in_dev)
3083 {
3084         rt_cache_flush(dev_net(in_dev->dev), 0);
3085 }
3086
3087 #ifdef CONFIG_SYSCTL
3088 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3089                                         void __user *buffer,
3090                                         size_t *lenp, loff_t *ppos)
3091 {
3092         if (write) {
3093                 int flush_delay;
3094                 ctl_table ctl;
3095                 struct net *net;
3096
3097                 memcpy(&ctl, __ctl, sizeof(ctl));
3098                 ctl.data = &flush_delay;
3099                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3100
3101                 net = (struct net *)__ctl->extra1;
3102                 rt_cache_flush(net, flush_delay);
3103                 return 0;
3104         }
3105
3106         return -EINVAL;
3107 }
3108
3109 static ctl_table ipv4_route_table[] = {
3110         {
3111                 .procname       = "gc_thresh",
3112                 .data           = &ipv4_dst_ops.gc_thresh,
3113                 .maxlen         = sizeof(int),
3114                 .mode           = 0644,
3115                 .proc_handler   = proc_dointvec,
3116         },
3117         {
3118                 .procname       = "max_size",
3119                 .data           = &ip_rt_max_size,
3120                 .maxlen         = sizeof(int),
3121                 .mode           = 0644,
3122                 .proc_handler   = proc_dointvec,
3123         },
3124         {
3125                 /*  Deprecated. Use gc_min_interval_ms */
3126
3127                 .procname       = "gc_min_interval",
3128                 .data           = &ip_rt_gc_min_interval,
3129                 .maxlen         = sizeof(int),
3130                 .mode           = 0644,
3131                 .proc_handler   = proc_dointvec_jiffies,
3132         },
3133         {
3134                 .procname       = "gc_min_interval_ms",
3135                 .data           = &ip_rt_gc_min_interval,
3136                 .maxlen         = sizeof(int),
3137                 .mode           = 0644,
3138                 .proc_handler   = proc_dointvec_ms_jiffies,
3139         },
3140         {
3141                 .procname       = "gc_timeout",
3142                 .data           = &ip_rt_gc_timeout,
3143                 .maxlen         = sizeof(int),
3144                 .mode           = 0644,
3145                 .proc_handler   = proc_dointvec_jiffies,
3146         },
3147         {
3148                 .procname       = "gc_interval",
3149                 .data           = &ip_rt_gc_interval,
3150                 .maxlen         = sizeof(int),
3151                 .mode           = 0644,
3152                 .proc_handler   = proc_dointvec_jiffies,
3153         },
3154         {
3155                 .procname       = "redirect_load",
3156                 .data           = &ip_rt_redirect_load,
3157                 .maxlen         = sizeof(int),
3158                 .mode           = 0644,
3159                 .proc_handler   = proc_dointvec,
3160         },
3161         {
3162                 .procname       = "redirect_number",
3163                 .data           = &ip_rt_redirect_number,
3164                 .maxlen         = sizeof(int),
3165                 .mode           = 0644,
3166                 .proc_handler   = proc_dointvec,
3167         },
3168         {
3169                 .procname       = "redirect_silence",
3170                 .data           = &ip_rt_redirect_silence,
3171                 .maxlen         = sizeof(int),
3172                 .mode           = 0644,
3173                 .proc_handler   = proc_dointvec,
3174         },
3175         {
3176                 .procname       = "error_cost",
3177                 .data           = &ip_rt_error_cost,
3178                 .maxlen         = sizeof(int),
3179                 .mode           = 0644,
3180                 .proc_handler   = proc_dointvec,
3181         },
3182         {
3183                 .procname       = "error_burst",
3184                 .data           = &ip_rt_error_burst,
3185                 .maxlen         = sizeof(int),
3186                 .mode           = 0644,
3187                 .proc_handler   = proc_dointvec,
3188         },
3189         {
3190                 .procname       = "gc_elasticity",
3191                 .data           = &ip_rt_gc_elasticity,
3192                 .maxlen         = sizeof(int),
3193                 .mode           = 0644,
3194                 .proc_handler   = proc_dointvec,
3195         },
3196         {
3197                 .procname       = "mtu_expires",
3198                 .data           = &ip_rt_mtu_expires,
3199                 .maxlen         = sizeof(int),
3200                 .mode           = 0644,
3201                 .proc_handler   = proc_dointvec_jiffies,
3202         },
3203         {
3204                 .procname       = "min_pmtu",
3205                 .data           = &ip_rt_min_pmtu,
3206                 .maxlen         = sizeof(int),
3207                 .mode           = 0644,
3208                 .proc_handler   = proc_dointvec,
3209         },
3210         {
3211                 .procname       = "min_adv_mss",
3212                 .data           = &ip_rt_min_advmss,
3213                 .maxlen         = sizeof(int),
3214                 .mode           = 0644,
3215                 .proc_handler   = proc_dointvec,
3216         },
3217         { }
3218 };
3219
3220 static struct ctl_table ipv4_route_flush_table[] = {
3221         {
3222                 .procname       = "flush",
3223                 .maxlen         = sizeof(int),
3224                 .mode           = 0200,
3225                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3226         },
3227         { },
3228 };
3229
3230 static __net_init int sysctl_route_net_init(struct net *net)
3231 {
3232         struct ctl_table *tbl;
3233
3234         tbl = ipv4_route_flush_table;
3235         if (!net_eq(net, &init_net)) {
3236                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3237                 if (tbl == NULL)
3238                         goto err_dup;
3239         }
3240         tbl[0].extra1 = net;
3241
3242         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
3243         if (net->ipv4.route_hdr == NULL)
3244                 goto err_reg;
3245         return 0;
3246
3247 err_reg:
3248         if (tbl != ipv4_route_flush_table)
3249                 kfree(tbl);
3250 err_dup:
3251         return -ENOMEM;
3252 }
3253
3254 static __net_exit void sysctl_route_net_exit(struct net *net)
3255 {
3256         struct ctl_table *tbl;
3257
3258         tbl = net->ipv4.route_hdr->ctl_table_arg;
3259         unregister_net_sysctl_table(net->ipv4.route_hdr);
3260         BUG_ON(tbl == ipv4_route_flush_table);
3261         kfree(tbl);
3262 }
3263
3264 static __net_initdata struct pernet_operations sysctl_route_ops = {
3265         .init = sysctl_route_net_init,
3266         .exit = sysctl_route_net_exit,
3267 };
3268 #endif
3269
3270 static __net_init int rt_genid_init(struct net *net)
3271 {
3272         get_random_bytes(&net->ipv4.rt_genid,
3273                          sizeof(net->ipv4.rt_genid));
3274         get_random_bytes(&net->ipv4.dev_addr_genid,
3275                          sizeof(net->ipv4.dev_addr_genid));
3276         return 0;
3277 }
3278
3279 static __net_initdata struct pernet_operations rt_genid_ops = {
3280         .init = rt_genid_init,
3281 };
3282
3283 static int __net_init ipv4_inetpeer_init(struct net *net)
3284 {
3285         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
3286
3287         if (!bp)
3288                 return -ENOMEM;
3289         inet_peer_base_init(bp);
3290         net->ipv4.peers = bp;
3291         return 0;
3292 }
3293
3294 static void __net_exit ipv4_inetpeer_exit(struct net *net)
3295 {
3296         struct inet_peer_base *bp = net->ipv4.peers;
3297
3298         net->ipv4.peers = NULL;
3299         inetpeer_invalidate_tree(bp);
3300         kfree(bp);
3301 }
3302
3303 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
3304         .init   =       ipv4_inetpeer_init,
3305         .exit   =       ipv4_inetpeer_exit,
3306 };
3307
3308 #ifdef CONFIG_IP_ROUTE_CLASSID
3309 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3310 #endif /* CONFIG_IP_ROUTE_CLASSID */
3311
3312 static __initdata unsigned long rhash_entries;
3313 static int __init set_rhash_entries(char *str)
3314 {
3315         ssize_t ret;
3316
3317         if (!str)
3318                 return 0;
3319
3320         ret = kstrtoul(str, 0, &rhash_entries);
3321         if (ret)
3322                 return 0;
3323
3324         return 1;
3325 }
3326 __setup("rhash_entries=", set_rhash_entries);
3327
3328 int __init ip_rt_init(void)
3329 {
3330         int rc = 0;
3331
3332 #ifdef CONFIG_IP_ROUTE_CLASSID
3333         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3334         if (!ip_rt_acct)
3335                 panic("IP: failed to allocate ip_rt_acct\n");
3336 #endif
3337
3338         ipv4_dst_ops.kmem_cachep =
3339                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3340                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3341
3342         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3343
3344         if (dst_entries_init(&ipv4_dst_ops) < 0)
3345                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
3346
3347         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
3348                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
3349
3350         rt_hash_table = (struct rt_hash_bucket *)
3351                 alloc_large_system_hash("IP route cache",
3352                                         sizeof(struct rt_hash_bucket),
3353                                         rhash_entries,
3354                                         (totalram_pages >= 128 * 1024) ?
3355                                         15 : 17,
3356                                         0,
3357                                         &rt_hash_log,
3358                                         &rt_hash_mask,
3359                                         0,
3360                                         rhash_entries ? 0 : 512 * 1024);
3361         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3362         rt_hash_lock_init();
3363
3364         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3365         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3366
3367         devinet_init();
3368         ip_fib_init();
3369
3370         INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3371         expires_ljiffies = jiffies;
3372         schedule_delayed_work(&expires_work,
3373                 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3374
3375         if (ip_rt_proc_init())
3376                 pr_err("Unable to create route proc files\n");
3377 #ifdef CONFIG_XFRM
3378         xfrm_init();
3379         xfrm4_init(ip_rt_max_size);
3380 #endif
3381         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
3382
3383 #ifdef CONFIG_SYSCTL
3384         register_pernet_subsys(&sysctl_route_ops);
3385 #endif
3386         register_pernet_subsys(&rt_genid_ops);
3387         register_pernet_subsys(&ipv4_inetpeer_ops);
3388         return rc;
3389 }
3390
3391 #ifdef CONFIG_SYSCTL
3392 /*
3393  * We really need to sanitize the damn ipv4 init order, then all
3394  * this nonsense will go away.
3395  */
3396 void __init ip_static_sysctl_init(void)
3397 {
3398         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
3399 }
3400 #endif