drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <net/netevent.h>
  44 #include <net/neighbour.h>
  45 #include <net/arp.h>
  46 #include <net/ip_fib.h>
  47
  48 #include "spectrum.h"
  49 #include "core.h"
  50 #include "reg.h"
  51
  52 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  53         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  54
  55 static bool
  56 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  57                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  58 {
  59         unsigned char prefix;
  60
  61         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  62                 if (!test_bit(prefix, prefix_usage2->b))
  63                         return false;
  64         }
  65         return true;
  66 }
  67
  68 static bool
  69 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  70                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  71 {
  72         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  73 }
  74
  75 static bool
  76 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  77 {
  78         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  79
  80         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
  81 }
  82
  83 static void
  84 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
  85                           struct mlxsw_sp_prefix_usage *prefix_usage2)
  86 {
  87         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  88 }
  89
  90 static void
  91 mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
  92 {
  93         memset(prefix_usage, 0, sizeof(*prefix_usage));
  94 }
  95
  96 static void
  97 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
  98                           unsigned char prefix_len)
  99 {
 100         set_bit(prefix_len, prefix_usage->b);
 101 }
 102
 103 static void
 104 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 105                             unsigned char prefix_len)
 106 {
 107         clear_bit(prefix_len, prefix_usage->b);
 108 }
 109
 110 struct mlxsw_sp_fib_key {
 111         struct net_device *dev;
 112         unsigned char addr[sizeof(struct in6_addr)];
 113         unsigned char prefix_len;
 114 };
 115
 116 enum mlxsw_sp_fib_entry_type {
 117         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 118         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 119         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 120 };
 121
 122 struct mlxsw_sp_nexthop_group;
 123
 124 struct mlxsw_sp_fib_entry {
 125         struct rhash_head ht_node;
 126         struct list_head list;
 127         struct mlxsw_sp_fib_key key;
 128         enum mlxsw_sp_fib_entry_type type;
 129         unsigned int ref_count;
 130         struct mlxsw_sp_vr *vr;
 131         struct list_head nexthop_group_node;
 132         struct mlxsw_sp_nexthop_group *nh_group;
 133         bool offloaded;
 134 };
 135
 136 struct mlxsw_sp_fib {
 137         struct rhashtable ht;
 138         struct list_head entry_list;
 139         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 140         struct mlxsw_sp_prefix_usage prefix_usage;
 141 };
 142
 143 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
 144         .key_offset = offsetof(struct mlxsw_sp_fib_entry, key),
 145         .head_offset = offsetof(struct mlxsw_sp_fib_entry, ht_node),
 146         .key_len = sizeof(struct mlxsw_sp_fib_key),
 147         .automatic_shrinking = true,
 148 };
 149
 150 static int mlxsw_sp_fib_entry_insert(struct mlxsw_sp_fib *fib,
 151                                      struct mlxsw_sp_fib_entry *fib_entry)
 152 {
 153         unsigned char prefix_len = fib_entry->key.prefix_len;
 154         int err;
 155
 156         err = rhashtable_insert_fast(&fib->ht, &fib_entry->ht_node,
 157                                      mlxsw_sp_fib_ht_params);
 158         if (err)
 159                 return err;
 160         list_add_tail(&fib_entry->list, &fib->entry_list);
 161         if (fib->prefix_ref_count[prefix_len]++ == 0)
 162                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
 163         return 0;
 164 }
 165
 166 static void mlxsw_sp_fib_entry_remove(struct mlxsw_sp_fib *fib,
 167                                       struct mlxsw_sp_fib_entry *fib_entry)
 168 {
 169         unsigned char prefix_len = fib_entry->key.prefix_len;
 170
 171         if (--fib->prefix_ref_count[prefix_len] == 0)
 172                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
 173         list_del(&fib_entry->list);
 174         rhashtable_remove_fast(&fib->ht, &fib_entry->ht_node,
 175                                mlxsw_sp_fib_ht_params);
 176 }
 177
 178 static struct mlxsw_sp_fib_entry *
 179 mlxsw_sp_fib_entry_create(struct mlxsw_sp_fib *fib, const void *addr,
 180                           size_t addr_len, unsigned char prefix_len,
 181                           struct net_device *dev)
 182 {
 183         struct mlxsw_sp_fib_entry *fib_entry;
 184
 185         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
 186         if (!fib_entry)
 187                 return NULL;
 188         fib_entry->key.dev = dev;
 189         memcpy(fib_entry->key.addr, addr, addr_len);
 190         fib_entry->key.prefix_len = prefix_len;
 191         return fib_entry;
 192 }
 193
 194 static void mlxsw_sp_fib_entry_destroy(struct mlxsw_sp_fib_entry *fib_entry)
 195 {
 196         kfree(fib_entry);
 197 }
 198
 199 static struct mlxsw_sp_fib_entry *
 200 mlxsw_sp_fib_entry_lookup(struct mlxsw_sp_fib *fib, const void *addr,
 201                           size_t addr_len, unsigned char prefix_len,
 202                           struct net_device *dev)
 203 {
 204         struct mlxsw_sp_fib_key key;
 205
 206         memset(&key, 0, sizeof(key));
 207         key.dev = dev;
 208         memcpy(key.addr, addr, addr_len);
 209         key.prefix_len = prefix_len;
 210         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
 211 }
 212
 213 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
 214 {
 215         struct mlxsw_sp_fib *fib;
 216         int err;
 217
 218         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 219         if (!fib)
 220                 return ERR_PTR(-ENOMEM);
 221         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 222         if (err)
 223                 goto err_rhashtable_init;
 224         INIT_LIST_HEAD(&fib->entry_list);
 225         return fib;
 226
 227 err_rhashtable_init:
 228         kfree(fib);
 229         return ERR_PTR(err);
 230 }
 231
 232 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 233 {
 234         rhashtable_destroy(&fib->ht);
 235         kfree(fib);
 236 }
 237
 238 static struct mlxsw_sp_lpm_tree *
 239 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
 240 {
 241         static struct mlxsw_sp_lpm_tree *lpm_tree;
 242         int i;
 243
 244         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 245                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 246                 if (lpm_tree->ref_count == 0) {
 247                         if (one_reserved)
 248                                 one_reserved = false;
 249                         else
 250                                 return lpm_tree;
 251                 }
 252         }
 253         return NULL;
 254 }
 255
 256 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 257                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 258 {
 259         char ralta_pl[MLXSW_REG_RALTA_LEN];
 260
 261         mlxsw_reg_ralta_pack(ralta_pl, true,
 262                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 263                              lpm_tree->id);
 264         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 265 }
 266
 267 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 268                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 269 {
 270         char ralta_pl[MLXSW_REG_RALTA_LEN];
 271
 272         mlxsw_reg_ralta_pack(ralta_pl, false,
 273                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 274                              lpm_tree->id);
 275         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 276 }
 277
 278 static int
 279 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 280                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 281                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 282 {
 283         char ralst_pl[MLXSW_REG_RALST_LEN];
 284         u8 root_bin = 0;
 285         u8 prefix;
 286         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 287
 288         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 289                 root_bin = prefix;
 290
 291         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 292         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 293                 if (prefix == 0)
 294                         continue;
 295                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 296                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 297                 last_prefix = prefix;
 298         }
 299         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 300 }
 301
 302 static struct mlxsw_sp_lpm_tree *
 303 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 304                          struct mlxsw_sp_prefix_usage *prefix_usage,
 305                          enum mlxsw_sp_l3proto proto, bool one_reserved)
 306 {
 307         struct mlxsw_sp_lpm_tree *lpm_tree;
 308         int err;
 309
 310         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
 311         if (!lpm_tree)
 312                 return ERR_PTR(-EBUSY);
 313         lpm_tree->proto = proto;
 314         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 315         if (err)
 316                 return ERR_PTR(err);
 317
 318         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 319                                                 lpm_tree);
 320         if (err)
 321                 goto err_left_struct_set;
 322         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 323                sizeof(lpm_tree->prefix_usage));
 324         return lpm_tree;
 325
 326 err_left_struct_set:
 327         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 328         return ERR_PTR(err);
 329 }
 330
 331 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 332                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 333 {
 334         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 335 }
 336
 337 static struct mlxsw_sp_lpm_tree *
 338 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 339                       struct mlxsw_sp_prefix_usage *prefix_usage,
 340                       enum mlxsw_sp_l3proto proto, bool one_reserved)
 341 {
 342         struct mlxsw_sp_lpm_tree *lpm_tree;
 343         int i;
 344
 345         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 346                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 347                 if (lpm_tree->ref_count != 0 &&
 348                     lpm_tree->proto == proto &&
 349                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 350                                              prefix_usage))
 351                         goto inc_ref_count;
 352         }
 353         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 354                                             proto, one_reserved);
 355         if (IS_ERR(lpm_tree))
 356                 return lpm_tree;
 357
 358 inc_ref_count:
 359         lpm_tree->ref_count++;
 360         return lpm_tree;
 361 }
 362
 363 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 364                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 365 {
 366         if (--lpm_tree->ref_count == 0)
 367                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 368         return 0;
 369 }
 370
 371 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 372 {
 373         struct mlxsw_sp_lpm_tree *lpm_tree;
 374         int i;
 375
 376         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 377                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 378                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 379         }
 380 }
 381
 382 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 383 {
 384         struct mlxsw_sp_vr *vr;
 385         int i;
 386
 387         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 388                 vr = &mlxsw_sp->router.vrs[i];
 389                 if (!vr->used)
 390                         return vr;
 391         }
 392         return NULL;
 393 }
 394
 395 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 396                                      struct mlxsw_sp_vr *vr)
 397 {
 398         char raltb_pl[MLXSW_REG_RALTB_LEN];
 399
 400         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 401                              (enum mlxsw_reg_ralxx_protocol) vr->proto,
 402                              vr->lpm_tree->id);
 403         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 404 }
 405
 406 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 407                                        struct mlxsw_sp_vr *vr)
 408 {
 409         char raltb_pl[MLXSW_REG_RALTB_LEN];
 410
 411         /* Bind to tree 0 which is default */
 412         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 413                              (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
 414         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 415 }
 416
 417 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 418 {
 419         /* For our purpose, squash main and local table into one */
 420         if (tb_id == RT_TABLE_LOCAL)
 421                 tb_id = RT_TABLE_MAIN;
 422         return tb_id;
 423 }
 424
 425 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 426                                             u32 tb_id,
 427                                             enum mlxsw_sp_l3proto proto)
 428 {
 429         struct mlxsw_sp_vr *vr;
 430         int i;
 431
 432         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 433
 434         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 435                 vr = &mlxsw_sp->router.vrs[i];
 436                 if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
 437                         return vr;
 438         }
 439         return NULL;
 440 }
 441
 442 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 443                                               unsigned char prefix_len,
 444                                               u32 tb_id,
 445                                               enum mlxsw_sp_l3proto proto)
 446 {
 447         struct mlxsw_sp_prefix_usage req_prefix_usage;
 448         struct mlxsw_sp_lpm_tree *lpm_tree;
 449         struct mlxsw_sp_vr *vr;
 450         int err;
 451
 452         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 453         if (!vr)
 454                 return ERR_PTR(-EBUSY);
 455         vr->fib = mlxsw_sp_fib_create();
 456         if (IS_ERR(vr->fib))
 457                 return ERR_CAST(vr->fib);
 458
 459         vr->proto = proto;
 460         vr->tb_id = tb_id;
 461         mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
 462         mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 463         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 464                                          proto, true);
 465         if (IS_ERR(lpm_tree)) {
 466                 err = PTR_ERR(lpm_tree);
 467                 goto err_tree_get;
 468         }
 469         vr->lpm_tree = lpm_tree;
 470         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 471         if (err)
 472                 goto err_tree_bind;
 473
 474         vr->used = true;
 475         return vr;
 476
 477 err_tree_bind:
 478         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 479 err_tree_get:
 480         mlxsw_sp_fib_destroy(vr->fib);
 481
 482         return ERR_PTR(err);
 483 }
 484
 485 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 486                                 struct mlxsw_sp_vr *vr)
 487 {
 488         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
 489         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 490         mlxsw_sp_fib_destroy(vr->fib);
 491         vr->used = false;
 492 }
 493
 494 static int
 495 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
 496                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 497 {
 498         struct mlxsw_sp_lpm_tree *lpm_tree;
 499
 500         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage,
 501                                      &vr->lpm_tree->prefix_usage))
 502                 return 0;
 503
 504         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 505                                          vr->proto, false);
 506         if (IS_ERR(lpm_tree)) {
 507                 /* We failed to get a tree according to the required
 508                  * prefix usage. However, the current tree might be still good
 509                  * for us if our requirement is subset of the prefixes used
 510                  * in the tree.
 511                  */
 512                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 513                                                  &vr->lpm_tree->prefix_usage))
 514                         return 0;
 515                 return PTR_ERR(lpm_tree);
 516         }
 517
 518         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
 519         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 520         vr->lpm_tree = lpm_tree;
 521         return mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 522 }
 523
 524 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
 525                                            unsigned char prefix_len,
 526                                            u32 tb_id,
 527                                            enum mlxsw_sp_l3proto proto)
 528 {
 529         struct mlxsw_sp_vr *vr;
 530         int err;
 531
 532         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 533         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
 534         if (!vr) {
 535                 vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
 536                 if (IS_ERR(vr))
 537                         return vr;
 538         } else {
 539                 struct mlxsw_sp_prefix_usage req_prefix_usage;
 540
 541                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
 542                                           &vr->fib->prefix_usage);
 543                 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 544                 /* Need to replace LPM tree in case new prefix is required. */
 545                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 546                                                  &req_prefix_usage);
 547                 if (err)
 548                         return ERR_PTR(err);
 549         }
 550         return vr;
 551 }
 552
 553 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 554 {
 555         /* Destroy virtual router entity in case the associated FIB is empty
 556          * and allow it to be used for other tables in future. Otherwise,
 557          * check if some prefix usage did not disappear and change tree if
 558          * that is the case. Note that in case new, smaller tree cannot be
 559          * allocated, the original one will be kept being used.
 560          */
 561         if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
 562                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 563         else
 564                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 565                                            &vr->fib->prefix_usage);
 566 }
 567
 568 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 569 {
 570         struct mlxsw_sp_vr *vr;
 571         u64 max_vrs;
 572         int i;
 573
 574         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 575                 return -EIO;
 576
 577         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 578         mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 579                                        GFP_KERNEL);
 580         if (!mlxsw_sp->router.vrs)
 581                 return -ENOMEM;
 582
 583         for (i = 0; i < max_vrs; i++) {
 584                 vr = &mlxsw_sp->router.vrs[i];
 585                 vr->id = i;
 586         }
 587
 588         return 0;
 589 }
 590
 591 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 592
 593 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 594 {
 595         /* At this stage we're guaranteed not to have new incoming
 596          * FIB notifications and the work queue is free from FIBs
 597          * sitting on top of mlxsw netdevs. However, we can still
 598          * have other FIBs queued. Flush the queue before flushing
 599          * the device's tables. No need for locks, as we're the only
 600          * writer.
 601          */
 602         mlxsw_core_flush_owq();
 603         mlxsw_sp_router_fib_flush(mlxsw_sp);
 604         kfree(mlxsw_sp->router.vrs);
 605 }
 606
 607 struct mlxsw_sp_neigh_key {
 608         struct neighbour *n;
 609 };
 610
 611 struct mlxsw_sp_neigh_entry {
 612         struct rhash_head ht_node;
 613         struct mlxsw_sp_neigh_key key;
 614         u16 rif;
 615         bool connected;
 616         unsigned char ha[ETH_ALEN];
 617         struct list_head nexthop_list; /* list of nexthops using
 618                                         * this neigh entry
 619                                         */
 620         struct list_head nexthop_neighs_list_node;
 621 };
 622
 623 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 624         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 625         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 626         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 627 };
 628
 629 static struct mlxsw_sp_neigh_entry *
 630 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 631                            u16 rif)
 632 {
 633         struct mlxsw_sp_neigh_entry *neigh_entry;
 634
 635         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 636         if (!neigh_entry)
 637                 return NULL;
 638
 639         neigh_entry->key.n = n;
 640         neigh_entry->rif = rif;
 641         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 642
 643         return neigh_entry;
 644 }
 645
 646 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 647 {
 648         kfree(neigh_entry);
 649 }
 650
 651 static int
 652 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 653                             struct mlxsw_sp_neigh_entry *neigh_entry)
 654 {
 655         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 656                                       &neigh_entry->ht_node,
 657                                       mlxsw_sp_neigh_ht_params);
 658 }
 659
 660 static void
 661 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 662                             struct mlxsw_sp_neigh_entry *neigh_entry)
 663 {
 664         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 665                                &neigh_entry->ht_node,
 666                                mlxsw_sp_neigh_ht_params);
 667 }
 668
 669 static struct mlxsw_sp_neigh_entry *
 670 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 671 {
 672         struct mlxsw_sp_neigh_entry *neigh_entry;
 673         struct mlxsw_sp_rif *r;
 674         int err;
 675
 676         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 677         if (!r)
 678                 return ERR_PTR(-EINVAL);
 679
 680         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, r->rif);
 681         if (!neigh_entry)
 682                 return ERR_PTR(-ENOMEM);
 683
 684         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 685         if (err)
 686                 goto err_neigh_entry_insert;
 687
 688         return neigh_entry;
 689
 690 err_neigh_entry_insert:
 691         mlxsw_sp_neigh_entry_free(neigh_entry);
 692         return ERR_PTR(err);
 693 }
 694
 695 static void
 696 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 697                              struct mlxsw_sp_neigh_entry *neigh_entry)
 698 {
 699         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 700         mlxsw_sp_neigh_entry_free(neigh_entry);
 701 }
 702
 703 static struct mlxsw_sp_neigh_entry *
 704 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 705 {
 706         struct mlxsw_sp_neigh_key key;
 707
 708         key.n = n;
 709         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 710                                       &key, mlxsw_sp_neigh_ht_params);
 711 }
 712
 713 static void
 714 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 715 {
 716         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 717
 718         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 719 }
 720
 721 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 722                                                    char *rauhtd_pl,
 723                                                    int ent_index)
 724 {
 725         struct net_device *dev;
 726         struct neighbour *n;
 727         __be32 dipn;
 728         u32 dip;
 729         u16 rif;
 730
 731         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 732
 733         if (!mlxsw_sp->rifs[rif]) {
 734                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 735                 return;
 736         }
 737
 738         dipn = htonl(dip);
 739         dev = mlxsw_sp->rifs[rif]->dev;
 740         n = neigh_lookup(&arp_tbl, &dipn, dev);
 741         if (!n) {
 742                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 743                            &dip);
 744                 return;
 745         }
 746
 747         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 748         neigh_event_send(n, NULL);
 749         neigh_release(n);
 750 }
 751
 752 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 753                                                    char *rauhtd_pl,
 754                                                    int rec_index)
 755 {
 756         u8 num_entries;
 757         int i;
 758
 759         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 760                                                                 rec_index);
 761         /* Hardware starts counting at 0, so add 1. */
 762         num_entries++;
 763
 764         /* Each record consists of several neighbour entries. */
 765         for (i = 0; i < num_entries; i++) {
 766                 int ent_index;
 767
 768                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 769                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 770                                                        ent_index);
 771         }
 772
 773 }
 774
 775 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 776                                               char *rauhtd_pl, int rec_index)
 777 {
 778         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 779         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 780                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 781                                                        rec_index);
 782                 break;
 783         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 784                 WARN_ON_ONCE(1);
 785                 break;
 786         }
 787 }
 788
 789 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 790 {
 791         u8 num_rec, last_rec_index, num_entries;
 792
 793         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 794         last_rec_index = num_rec - 1;
 795
 796         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 797                 return false;
 798         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 799             MLXSW_REG_RAUHTD_TYPE_IPV6)
 800                 return true;
 801
 802         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 803                                                                 last_rec_index);
 804         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 805                 return true;
 806         return false;
 807 }
 808
 809 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 810 {
 811         char *rauhtd_pl;
 812         u8 num_rec;
 813         int i, err;
 814
 815         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 816         if (!rauhtd_pl)
 817                 return -ENOMEM;
 818
 819         /* Make sure the neighbour's netdev isn't removed in the
 820          * process.
 821          */
 822         rtnl_lock();
 823         do {
 824                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 825                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 826                                       rauhtd_pl);
 827                 if (err) {
 828                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 829                         break;
 830                 }
 831                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 832                 for (i = 0; i < num_rec; i++)
 833                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 834                                                           i);
 835         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 836         rtnl_unlock();
 837
 838         kfree(rauhtd_pl);
 839         return err;
 840 }
 841
 842 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 843 {
 844         struct mlxsw_sp_neigh_entry *neigh_entry;
 845
 846         /* Take RTNL mutex here to prevent lists from changes */
 847         rtnl_lock();
 848         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 849                             nexthop_neighs_list_node)
 850                 /* If this neigh have nexthops, make the kernel think this neigh
 851                  * is active regardless of the traffic.
 852                  */
 853                 neigh_event_send(neigh_entry->key.n, NULL);
 854         rtnl_unlock();
 855 }
 856
 857 static void
 858 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 859 {
 860         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 861
 862         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 863                                msecs_to_jiffies(interval));
 864 }
 865
 866 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 867 {
 868         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 869                                                  router.neighs_update.dw.work);
 870         int err;
 871
 872         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 873         if (err)
 874                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 875
 876         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 877
 878         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 879 }
 880
 881 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 882 {
 883         struct mlxsw_sp_neigh_entry *neigh_entry;
 884         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 885                                                  router.nexthop_probe_dw.work);
 886
 887         /* Iterate over nexthop neighbours, find those who are unresolved and
 888          * send arp on them. This solves the chicken-egg problem when
 889          * the nexthop wouldn't get offloaded until the neighbor is resolved
 890          * but it wouldn't get resolved ever in case traffic is flowing in HW
 891          * using different nexthop.
 892          *
 893          * Take RTNL mutex here to prevent lists from changes.
 894          */
 895         rtnl_lock();
 896         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 897                             nexthop_neighs_list_node)
 898                 if (!neigh_entry->connected)
 899                         neigh_event_send(neigh_entry->key.n, NULL);
 900         rtnl_unlock();
 901
 902         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 903                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 904 }
 905
 906 static void
 907 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 908                               struct mlxsw_sp_neigh_entry *neigh_entry,
 909                               bool removing);
 910
 911 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
 912 {
 913         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
 914                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
 915 }
 916
 917 static void
 918 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 919                                 struct mlxsw_sp_neigh_entry *neigh_entry,
 920                                 enum mlxsw_reg_rauht_op op)
 921 {
 922         struct neighbour *n = neigh_entry->key.n;
 923         u32 dip = ntohl(*((__be32 *) n->primary_key));
 924         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 925
 926         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 927                               dip);
 928         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 929 }
 930
 931 static void
 932 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 933                             struct mlxsw_sp_neigh_entry *neigh_entry,
 934                             bool adding)
 935 {
 936         if (!adding && !neigh_entry->connected)
 937                 return;
 938         neigh_entry->connected = adding;
 939         if (neigh_entry->key.n->tbl == &arp_tbl)
 940                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 941                                                 mlxsw_sp_rauht_op(adding));
 942         else
 943                 WARN_ON_ONCE(1);
 944 }
 945
 946 struct mlxsw_sp_neigh_event_work {
 947         struct work_struct work;
 948         struct mlxsw_sp *mlxsw_sp;
 949         struct neighbour *n;
 950 };
 951
 952 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
 953 {
 954         struct mlxsw_sp_neigh_event_work *neigh_work =
 955                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
 956         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
 957         struct mlxsw_sp_neigh_entry *neigh_entry;
 958         struct neighbour *n = neigh_work->n;
 959         unsigned char ha[ETH_ALEN];
 960         bool entry_connected;
 961         u8 nud_state, dead;
 962
 963         /* If these parameters are changed after we release the lock,
 964          * then we are guaranteed to receive another event letting us
 965          * know about it.
 966          */
 967         read_lock_bh(&n->lock);
 968         memcpy(ha, n->ha, ETH_ALEN);
 969         nud_state = n->nud_state;
 970         dead = n->dead;
 971         read_unlock_bh(&n->lock);
 972
 973         rtnl_lock();
 974         entry_connected = nud_state & NUD_VALID && !dead;
 975         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 976         if (!entry_connected && !neigh_entry)
 977                 goto out;
 978         if (!neigh_entry) {
 979                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
 980                 if (IS_ERR(neigh_entry))
 981                         goto out;
 982         }
 983
 984         memcpy(neigh_entry->ha, ha, ETH_ALEN);
 985         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
 986         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
 987
 988         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 989                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
 990
 991 out:
 992         rtnl_unlock();
 993         neigh_release(n);
 994         kfree(neigh_work);
 995 }
 996
 997 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 998                                    unsigned long event, void *ptr)
 999 {
1000         struct mlxsw_sp_neigh_event_work *neigh_work;
1001         struct mlxsw_sp_port *mlxsw_sp_port;
1002         struct mlxsw_sp *mlxsw_sp;
1003         unsigned long interval;
1004         struct neigh_parms *p;
1005         struct neighbour *n;
1006
1007         switch (event) {
1008         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1009                 p = ptr;
1010
1011                 /* We don't care about changes in the default table. */
1012                 if (!p->dev || p->tbl != &arp_tbl)
1013                         return NOTIFY_DONE;
1014
1015                 /* We are in atomic context and can't take RTNL mutex,
1016                  * so use RCU variant to walk the device chain.
1017                  */
1018                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1019                 if (!mlxsw_sp_port)
1020                         return NOTIFY_DONE;
1021
1022                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1023                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1024                 mlxsw_sp->router.neighs_update.interval = interval;
1025
1026                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1027                 break;
1028         case NETEVENT_NEIGH_UPDATE:
1029                 n = ptr;
1030
1031                 if (n->tbl != &arp_tbl)
1032                         return NOTIFY_DONE;
1033
1034                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1035                 if (!mlxsw_sp_port)
1036                         return NOTIFY_DONE;
1037
1038                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1039                 if (!neigh_work) {
1040                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1041                         return NOTIFY_BAD;
1042                 }
1043
1044                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1045                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1046                 neigh_work->n = n;
1047
1048                 /* Take a reference to ensure the neighbour won't be
1049                  * destructed until we drop the reference in delayed
1050                  * work.
1051                  */
1052                 neigh_clone(n);
1053                 mlxsw_core_schedule_work(&neigh_work->work);
1054                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1055                 break;
1056         }
1057
1058         return NOTIFY_DONE;
1059 }
1060
1061 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1062 {
1063         int err;
1064
1065         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1066                               &mlxsw_sp_neigh_ht_params);
1067         if (err)
1068                 return err;
1069
1070         /* Initialize the polling interval according to the default
1071          * table.
1072          */
1073         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1074
1075         /* Create the delayed works for the activity_update */
1076         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1077                           mlxsw_sp_router_neighs_update_work);
1078         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1079                           mlxsw_sp_router_probe_unresolved_nexthops);
1080         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1081         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1082         return 0;
1083 }
1084
1085 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1086 {
1087         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1088         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1089         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1090 }
1091
1092 struct mlxsw_sp_nexthop_key {
1093         struct fib_nh *fib_nh;
1094 };
1095
1096 struct mlxsw_sp_nexthop {
1097         struct list_head neigh_list_node; /* member of neigh entry list */
1098         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1099                                                 * this belongs to
1100                                                 */
1101         struct rhash_head ht_node;
1102         struct mlxsw_sp_nexthop_key key;
1103         struct mlxsw_sp_rif *r;
1104         u8 should_offload:1, /* set indicates this neigh is connected and
1105                               * should be put to KVD linear area of this group.
1106                               */
1107            offloaded:1, /* set in case the neigh is actually put into
1108                          * KVD linear area of this group.
1109                          */
1110            update:1; /* set indicates that MAC of this neigh should be
1111                       * updated in HW
1112                       */
1113         struct mlxsw_sp_neigh_entry *neigh_entry;
1114 };
1115
1116 struct mlxsw_sp_nexthop_group_key {
1117         struct fib_info *fi;
1118 };
1119
1120 struct mlxsw_sp_nexthop_group {
1121         struct rhash_head ht_node;
1122         struct list_head fib_list; /* list of fib entries that use this group */
1123         struct mlxsw_sp_nexthop_group_key key;
1124         u8 adj_index_valid:1,
1125            gateway:1; /* routes using the group use a gateway */
1126         u32 adj_index;
1127         u16 ecmp_size;
1128         u16 count;
1129         struct mlxsw_sp_nexthop nexthops[0];
1130 #define nh_rif  nexthops[0].r
1131 };
1132
1133 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1134         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1135         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1136         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1137 };
1138
1139 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1140                                          struct mlxsw_sp_nexthop_group *nh_grp)
1141 {
1142         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1143                                       &nh_grp->ht_node,
1144                                       mlxsw_sp_nexthop_group_ht_params);
1145 }
1146
1147 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1148                                           struct mlxsw_sp_nexthop_group *nh_grp)
1149 {
1150         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1151                                &nh_grp->ht_node,
1152                                mlxsw_sp_nexthop_group_ht_params);
1153 }
1154
1155 static struct mlxsw_sp_nexthop_group *
1156 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1157                               struct mlxsw_sp_nexthop_group_key key)
1158 {
1159         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1160                                       mlxsw_sp_nexthop_group_ht_params);
1161 }
1162
1163 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1164         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1165         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1166         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1167 };
1168
1169 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1170                                    struct mlxsw_sp_nexthop *nh)
1171 {
1172         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1173                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1174 }
1175
1176 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1177                                     struct mlxsw_sp_nexthop *nh)
1178 {
1179         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1180                                mlxsw_sp_nexthop_ht_params);
1181 }
1182
1183 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1184                                              struct mlxsw_sp_vr *vr,
1185                                              u32 adj_index, u16 ecmp_size,
1186                                              u32 new_adj_index,
1187                                              u16 new_ecmp_size)
1188 {
1189         char raleu_pl[MLXSW_REG_RALEU_LEN];
1190
1191         mlxsw_reg_raleu_pack(raleu_pl,
1192                              (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
1193                              adj_index, ecmp_size, new_adj_index,
1194                              new_ecmp_size);
1195         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1196 }
1197
1198 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1199                                           struct mlxsw_sp_nexthop_group *nh_grp,
1200                                           u32 old_adj_index, u16 old_ecmp_size)
1201 {
1202         struct mlxsw_sp_fib_entry *fib_entry;
1203         struct mlxsw_sp_vr *vr = NULL;
1204         int err;
1205
1206         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1207                 if (vr == fib_entry->vr)
1208                         continue;
1209                 vr = fib_entry->vr;
1210                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
1211                                                         old_adj_index,
1212                                                         old_ecmp_size,
1213                                                         nh_grp->adj_index,
1214                                                         nh_grp->ecmp_size);
1215                 if (err)
1216                         return err;
1217         }
1218         return 0;
1219 }
1220
1221 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1222                                        struct mlxsw_sp_nexthop *nh)
1223 {
1224         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1225         char ratr_pl[MLXSW_REG_RATR_LEN];
1226
1227         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1228                             true, adj_index, neigh_entry->rif);
1229         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1230         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1231 }
1232
1233 static int
1234 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1235                                   struct mlxsw_sp_nexthop_group *nh_grp,
1236                                   bool reallocate)
1237 {
1238         u32 adj_index = nh_grp->adj_index; /* base */
1239         struct mlxsw_sp_nexthop *nh;
1240         int i;
1241         int err;
1242
1243         for (i = 0; i < nh_grp->count; i++) {
1244                 nh = &nh_grp->nexthops[i];
1245
1246                 if (!nh->should_offload) {
1247                         nh->offloaded = 0;
1248                         continue;
1249                 }
1250
1251                 if (nh->update || reallocate) {
1252                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1253                                                           adj_index, nh);
1254                         if (err)
1255                                 return err;
1256                         nh->update = 0;
1257                         nh->offloaded = 1;
1258                 }
1259                 adj_index++;
1260         }
1261         return 0;
1262 }
1263
1264 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1265                                      struct mlxsw_sp_fib_entry *fib_entry);
1266
1267 static int
1268 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1269                                     struct mlxsw_sp_nexthop_group *nh_grp)
1270 {
1271         struct mlxsw_sp_fib_entry *fib_entry;
1272         int err;
1273
1274         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1275                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1276                 if (err)
1277                         return err;
1278         }
1279         return 0;
1280 }
1281
1282 static void
1283 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1284                                struct mlxsw_sp_nexthop_group *nh_grp)
1285 {
1286         struct mlxsw_sp_nexthop *nh;
1287         bool offload_change = false;
1288         u32 adj_index;
1289         u16 ecmp_size = 0;
1290         bool old_adj_index_valid;
1291         u32 old_adj_index;
1292         u16 old_ecmp_size;
1293         int ret;
1294         int i;
1295         int err;
1296
1297         if (!nh_grp->gateway) {
1298                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1299                 return;
1300         }
1301
1302         for (i = 0; i < nh_grp->count; i++) {
1303                 nh = &nh_grp->nexthops[i];
1304
1305                 if (nh->should_offload ^ nh->offloaded) {
1306                         offload_change = true;
1307                         if (nh->should_offload)
1308                                 nh->update = 1;
1309                 }
1310                 if (nh->should_offload)
1311                         ecmp_size++;
1312         }
1313         if (!offload_change) {
1314                 /* Nothing was added or removed, so no need to reallocate. Just
1315                  * update MAC on existing adjacency indexes.
1316                  */
1317                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1318                                                         false);
1319                 if (err) {
1320                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1321                         goto set_trap;
1322                 }
1323                 return;
1324         }
1325         if (!ecmp_size)
1326                 /* No neigh of this group is connected so we just set
1327                  * the trap and let everthing flow through kernel.
1328                  */
1329                 goto set_trap;
1330
1331         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1332         if (ret < 0) {
1333                 /* We ran out of KVD linear space, just set the
1334                  * trap and let everything flow through kernel.
1335                  */
1336                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1337                 goto set_trap;
1338         }
1339         adj_index = ret;
1340         old_adj_index_valid = nh_grp->adj_index_valid;
1341         old_adj_index = nh_grp->adj_index;
1342         old_ecmp_size = nh_grp->ecmp_size;
1343         nh_grp->adj_index_valid = 1;
1344         nh_grp->adj_index = adj_index;
1345         nh_grp->ecmp_size = ecmp_size;
1346         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1347         if (err) {
1348                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1349                 goto set_trap;
1350         }
1351
1352         if (!old_adj_index_valid) {
1353                 /* The trap was set for fib entries, so we have to call
1354                  * fib entry update to unset it and use adjacency index.
1355                  */
1356                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1357                 if (err) {
1358                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1359                         goto set_trap;
1360                 }
1361                 return;
1362         }
1363
1364         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1365                                              old_adj_index, old_ecmp_size);
1366         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1367         if (err) {
1368                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1369                 goto set_trap;
1370         }
1371         return;
1372
1373 set_trap:
1374         old_adj_index_valid = nh_grp->adj_index_valid;
1375         nh_grp->adj_index_valid = 0;
1376         for (i = 0; i < nh_grp->count; i++) {
1377                 nh = &nh_grp->nexthops[i];
1378                 nh->offloaded = 0;
1379         }
1380         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1381         if (err)
1382                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1383         if (old_adj_index_valid)
1384                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1385 }
1386
1387 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1388                                             bool removing)
1389 {
1390         if (!removing && !nh->should_offload)
1391                 nh->should_offload = 1;
1392         else if (removing && nh->offloaded)
1393                 nh->should_offload = 0;
1394         nh->update = 1;
1395 }
1396
1397 static void
1398 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1399                               struct mlxsw_sp_neigh_entry *neigh_entry,
1400                               bool removing)
1401 {
1402         struct mlxsw_sp_nexthop *nh;
1403
1404         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1405                             neigh_list_node) {
1406                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1407                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1408         }
1409 }
1410
1411 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1412                                        struct mlxsw_sp_nexthop *nh)
1413 {
1414         struct mlxsw_sp_neigh_entry *neigh_entry;
1415         struct fib_nh *fib_nh = nh->key.fib_nh;
1416         struct neighbour *n;
1417         u8 nud_state, dead;
1418         int err;
1419
1420         if (!nh->nh_grp->gateway)
1421                 return 0;
1422
1423         /* Take a reference of neigh here ensuring that neigh would
1424          * not be detructed before the nexthop entry is finished.
1425          * The reference is taken either in neigh_lookup() or
1426          * in neigh_create() in case n is not found.
1427          */
1428         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1429         if (!n) {
1430                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1431                 if (IS_ERR(n))
1432                         return PTR_ERR(n);
1433                 neigh_event_send(n, NULL);
1434         }
1435         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1436         if (!neigh_entry) {
1437                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1438                 if (IS_ERR(neigh_entry)) {
1439                         err = -EINVAL;
1440                         goto err_neigh_entry_create;
1441                 }
1442         }
1443
1444         /* If that is the first nexthop connected to that neigh, add to
1445          * nexthop_neighs_list
1446          */
1447         if (list_empty(&neigh_entry->nexthop_list))
1448                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1449                               &mlxsw_sp->router.nexthop_neighs_list);
1450
1451         nh->neigh_entry = neigh_entry;
1452         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1453         read_lock_bh(&n->lock);
1454         nud_state = n->nud_state;
1455         dead = n->dead;
1456         read_unlock_bh(&n->lock);
1457         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1458
1459         return 0;
1460
1461 err_neigh_entry_create:
1462         neigh_release(n);
1463         return err;
1464 }
1465
1466 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1467                                         struct mlxsw_sp_nexthop *nh)
1468 {
1469         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1470         struct neighbour *n;
1471
1472         if (!neigh_entry)
1473                 return;
1474         n = neigh_entry->key.n;
1475
1476         __mlxsw_sp_nexthop_neigh_update(nh, true);
1477         list_del(&nh->neigh_list_node);
1478         nh->neigh_entry = NULL;
1479
1480         /* If that is the last nexthop connected to that neigh, remove from
1481          * nexthop_neighs_list
1482          */
1483         if (list_empty(&neigh_entry->nexthop_list))
1484                 list_del(&neigh_entry->nexthop_neighs_list_node);
1485
1486         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1487                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1488
1489         neigh_release(n);
1490 }
1491
1492 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1493                                  struct mlxsw_sp_nexthop_group *nh_grp,
1494                                  struct mlxsw_sp_nexthop *nh,
1495                                  struct fib_nh *fib_nh)
1496 {
1497         struct net_device *dev = fib_nh->nh_dev;
1498         struct mlxsw_sp_rif *r;
1499         int err;
1500
1501         nh->nh_grp = nh_grp;
1502         nh->key.fib_nh = fib_nh;
1503         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1504         if (err)
1505                 return err;
1506
1507         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1508         if (!r)
1509                 return 0;
1510         nh->r = r;
1511
1512         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1513         if (err)
1514                 goto err_nexthop_neigh_init;
1515
1516         return 0;
1517
1518 err_nexthop_neigh_init:
1519         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1520         return err;
1521 }
1522
1523 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1524                                   struct mlxsw_sp_nexthop *nh)
1525 {
1526         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1527         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1528 }
1529
1530 static struct mlxsw_sp_nexthop_group *
1531 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1532 {
1533         struct mlxsw_sp_nexthop_group *nh_grp;
1534         struct mlxsw_sp_nexthop *nh;
1535         struct fib_nh *fib_nh;
1536         size_t alloc_size;
1537         int i;
1538         int err;
1539
1540         alloc_size = sizeof(*nh_grp) +
1541                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1542         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1543         if (!nh_grp)
1544                 return ERR_PTR(-ENOMEM);
1545         INIT_LIST_HEAD(&nh_grp->fib_list);
1546         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1547         nh_grp->count = fi->fib_nhs;
1548         nh_grp->key.fi = fi;
1549         for (i = 0; i < nh_grp->count; i++) {
1550                 nh = &nh_grp->nexthops[i];
1551                 fib_nh = &fi->fib_nh[i];
1552                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1553                 if (err)
1554                         goto err_nexthop_init;
1555         }
1556         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1557         if (err)
1558                 goto err_nexthop_group_insert;
1559         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1560         return nh_grp;
1561
1562 err_nexthop_group_insert:
1563 err_nexthop_init:
1564         for (i--; i >= 0; i--)
1565                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1566         kfree(nh_grp);
1567         return ERR_PTR(err);
1568 }
1569
1570 static void
1571 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1572                                struct mlxsw_sp_nexthop_group *nh_grp)
1573 {
1574         struct mlxsw_sp_nexthop *nh;
1575         int i;
1576
1577         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1578         for (i = 0; i < nh_grp->count; i++) {
1579                 nh = &nh_grp->nexthops[i];
1580                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1581         }
1582         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1583         WARN_ON_ONCE(nh_grp->adj_index_valid);
1584         kfree(nh_grp);
1585 }
1586
1587 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1588                                       struct mlxsw_sp_fib_entry *fib_entry,
1589                                       struct fib_info *fi)
1590 {
1591         struct mlxsw_sp_nexthop_group_key key;
1592         struct mlxsw_sp_nexthop_group *nh_grp;
1593
1594         key.fi = fi;
1595         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1596         if (!nh_grp) {
1597                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1598                 if (IS_ERR(nh_grp))
1599                         return PTR_ERR(nh_grp);
1600         }
1601         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1602         fib_entry->nh_group = nh_grp;
1603         return 0;
1604 }
1605
1606 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1607                                        struct mlxsw_sp_fib_entry *fib_entry)
1608 {
1609         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1610
1611         list_del(&fib_entry->nexthop_group_node);
1612         if (!list_empty(&nh_grp->fib_list))
1613                 return;
1614         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1615 }
1616
1617 static bool
1618 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1619 {
1620         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1621
1622         switch (fib_entry->type) {
1623         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1624                 return !!nh_group->adj_index_valid;
1625         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1626                 return true;
1627         default:
1628                 return false;
1629         }
1630 }
1631
1632 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1633 {
1634         fib_entry->offloaded = true;
1635
1636         switch (fib_entry->vr->proto) {
1637         case MLXSW_SP_L3_PROTO_IPV4:
1638                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1639                 break;
1640         case MLXSW_SP_L3_PROTO_IPV6:
1641                 WARN_ON_ONCE(1);
1642         }
1643 }
1644
1645 static void
1646 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1647 {
1648         switch (fib_entry->vr->proto) {
1649         case MLXSW_SP_L3_PROTO_IPV4:
1650                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1651                 break;
1652         case MLXSW_SP_L3_PROTO_IPV6:
1653                 WARN_ON_ONCE(1);
1654         }
1655
1656         fib_entry->offloaded = false;
1657 }
1658
1659 static void
1660 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1661                                    enum mlxsw_reg_ralue_op op, int err)
1662 {
1663         switch (op) {
1664         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1665                 if (!fib_entry->offloaded)
1666                         return;
1667                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1668         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1669                 if (err)
1670                         return;
1671                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1672                     !fib_entry->offloaded)
1673                         mlxsw_sp_fib_entry_offload_set(fib_entry);
1674                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1675                          fib_entry->offloaded)
1676                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
1677                 return;
1678         default:
1679                 return;
1680         }
1681 }
1682
1683 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1684                                          struct mlxsw_sp_fib_entry *fib_entry,
1685                                          enum mlxsw_reg_ralue_op op)
1686 {
1687         char ralue_pl[MLXSW_REG_RALUE_LEN];
1688         u32 *p_dip = (u32 *) fib_entry->key.addr;
1689         struct mlxsw_sp_vr *vr = fib_entry->vr;
1690         enum mlxsw_reg_ralue_trap_action trap_action;
1691         u16 trap_id = 0;
1692         u32 adjacency_index = 0;
1693         u16 ecmp_size = 0;
1694
1695         /* In case the nexthop group adjacency index is valid, use it
1696          * with provided ECMP size. Otherwise, setup trap and pass
1697          * traffic to kernel.
1698          */
1699         if (fib_entry->nh_group->adj_index_valid) {
1700                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1701                 adjacency_index = fib_entry->nh_group->adj_index;
1702                 ecmp_size = fib_entry->nh_group->ecmp_size;
1703         } else {
1704                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1705                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1706         }
1707
1708         mlxsw_reg_ralue_pack4(ralue_pl,
1709                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1710                               vr->id, fib_entry->key.prefix_len, *p_dip);
1711         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1712                                         adjacency_index, ecmp_size);
1713         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1714 }
1715
1716 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1717                                         struct mlxsw_sp_fib_entry *fib_entry,
1718                                         enum mlxsw_reg_ralue_op op)
1719 {
1720         struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
1721         char ralue_pl[MLXSW_REG_RALUE_LEN];
1722         u32 *p_dip = (u32 *) fib_entry->key.addr;
1723         struct mlxsw_sp_vr *vr = fib_entry->vr;
1724
1725         mlxsw_reg_ralue_pack4(ralue_pl,
1726                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1727                               vr->id, fib_entry->key.prefix_len, *p_dip);
1728         mlxsw_reg_ralue_act_local_pack(ralue_pl,
1729                                        MLXSW_REG_RALUE_TRAP_ACTION_NOP, 0,
1730                                        r->rif);
1731         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1732 }
1733
1734 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1735                                        struct mlxsw_sp_fib_entry *fib_entry,
1736                                        enum mlxsw_reg_ralue_op op)
1737 {
1738         char ralue_pl[MLXSW_REG_RALUE_LEN];
1739         u32 *p_dip = (u32 *) fib_entry->key.addr;
1740         struct mlxsw_sp_vr *vr = fib_entry->vr;
1741
1742         mlxsw_reg_ralue_pack4(ralue_pl,
1743                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1744                               vr->id, fib_entry->key.prefix_len, *p_dip);
1745         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1746         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1747 }
1748
1749 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1750                                   struct mlxsw_sp_fib_entry *fib_entry,
1751                                   enum mlxsw_reg_ralue_op op)
1752 {
1753         switch (fib_entry->type) {
1754         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1755                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1756         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1757                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1758         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1759                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1760         }
1761         return -EINVAL;
1762 }
1763
1764 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1765                                  struct mlxsw_sp_fib_entry *fib_entry,
1766                                  enum mlxsw_reg_ralue_op op)
1767 {
1768         int err = -EINVAL;
1769
1770         switch (fib_entry->vr->proto) {
1771         case MLXSW_SP_L3_PROTO_IPV4:
1772                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1773                 break;
1774         case MLXSW_SP_L3_PROTO_IPV6:
1775                 return err;
1776         }
1777         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1778         return err;
1779 }
1780
1781 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1782                                      struct mlxsw_sp_fib_entry *fib_entry)
1783 {
1784         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1785                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1786 }
1787
1788 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1789                                   struct mlxsw_sp_fib_entry *fib_entry)
1790 {
1791         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1792                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1793 }
1794
1795 static int
1796 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1797                              const struct fib_entry_notifier_info *fen_info,
1798                              struct mlxsw_sp_fib_entry *fib_entry)
1799 {
1800         struct fib_info *fi = fen_info->fi;
1801         struct mlxsw_sp_rif *r = NULL;
1802         int nhsel;
1803
1804         if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
1805                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1806                 return 0;
1807         }
1808         if (fen_info->type != RTN_UNICAST)
1809                 return -EINVAL;
1810
1811         for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
1812                 const struct fib_nh *nh = &fi->fib_nh[nhsel];
1813
1814                 if (!nh->nh_dev)
1815                         continue;
1816                 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, nh->nh_dev);
1817                 if (!r) {
1818                         /* In case router interface is not found for
1819                          * at least one of the nexthops, that means
1820                          * the nexthop points to some device unrelated
1821                          * to us. Set trap and pass the packets for
1822                          * this prefix to kernel.
1823                          */
1824                         break;
1825                 }
1826         }
1827
1828         if (!r) {
1829                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1830                 return 0;
1831         }
1832
1833         if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1834                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1835         else
1836                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1837         return 0;
1838 }
1839
1840 static struct mlxsw_sp_fib_entry *
1841 mlxsw_sp_fib_entry_get(struct mlxsw_sp *mlxsw_sp,
1842                        const struct fib_entry_notifier_info *fen_info)
1843 {
1844         struct mlxsw_sp_fib_entry *fib_entry;
1845         struct fib_info *fi = fen_info->fi;
1846         struct mlxsw_sp_vr *vr;
1847         int err;
1848
1849         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
1850                              MLXSW_SP_L3_PROTO_IPV4);
1851         if (IS_ERR(vr))
1852                 return ERR_CAST(vr);
1853
1854         fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1855                                               sizeof(fen_info->dst),
1856                                               fen_info->dst_len, fi->fib_dev);
1857         if (fib_entry) {
1858                 /* Already exists, just take a reference */
1859                 fib_entry->ref_count++;
1860                 return fib_entry;
1861         }
1862         fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fen_info->dst,
1863                                               sizeof(fen_info->dst),
1864                                               fen_info->dst_len, fi->fib_dev);
1865         if (!fib_entry) {
1866                 err = -ENOMEM;
1867                 goto err_fib_entry_create;
1868         }
1869         fib_entry->vr = vr;
1870         fib_entry->ref_count = 1;
1871
1872         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1873         if (err)
1874                 goto err_fib4_entry_type_set;
1875
1876         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
1877         if (err)
1878                 goto err_nexthop_group_get;
1879
1880         return fib_entry;
1881
1882 err_nexthop_group_get:
1883 err_fib4_entry_type_set:
1884         mlxsw_sp_fib_entry_destroy(fib_entry);
1885 err_fib_entry_create:
1886         mlxsw_sp_vr_put(mlxsw_sp, vr);
1887
1888         return ERR_PTR(err);
1889 }
1890
1891 static struct mlxsw_sp_fib_entry *
1892 mlxsw_sp_fib_entry_find(struct mlxsw_sp *mlxsw_sp,
1893                         const struct fib_entry_notifier_info *fen_info)
1894 {
1895         struct mlxsw_sp_vr *vr;
1896
1897         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id,
1898                               MLXSW_SP_L3_PROTO_IPV4);
1899         if (!vr)
1900                 return NULL;
1901
1902         return mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1903                                          sizeof(fen_info->dst),
1904                                          fen_info->dst_len,
1905                                          fen_info->fi->fib_dev);
1906 }
1907
1908 static void mlxsw_sp_fib_entry_put(struct mlxsw_sp *mlxsw_sp,
1909                                    struct mlxsw_sp_fib_entry *fib_entry)
1910 {
1911         struct mlxsw_sp_vr *vr = fib_entry->vr;
1912
1913         if (--fib_entry->ref_count == 0) {
1914                 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1915                 mlxsw_sp_fib_entry_destroy(fib_entry);
1916         }
1917         mlxsw_sp_vr_put(mlxsw_sp, vr);
1918 }
1919
1920 static void mlxsw_sp_fib_entry_put_all(struct mlxsw_sp *mlxsw_sp,
1921                                        struct mlxsw_sp_fib_entry *fib_entry)
1922 {
1923         unsigned int last_ref_count;
1924
1925         do {
1926                 last_ref_count = fib_entry->ref_count;
1927                 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1928         } while (last_ref_count != 1);
1929 }
1930
1931 static int mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
1932                                     struct fib_entry_notifier_info *fen_info)
1933 {
1934         struct mlxsw_sp_fib_entry *fib_entry;
1935         struct mlxsw_sp_vr *vr;
1936         int err;
1937
1938         if (mlxsw_sp->router.aborted)
1939                 return 0;
1940
1941         fib_entry = mlxsw_sp_fib_entry_get(mlxsw_sp, fen_info);
1942         if (IS_ERR(fib_entry)) {
1943                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB4 entry being added.\n");
1944                 return PTR_ERR(fib_entry);
1945         }
1946
1947         if (fib_entry->ref_count != 1)
1948                 return 0;
1949
1950         vr = fib_entry->vr;
1951         err = mlxsw_sp_fib_entry_insert(vr->fib, fib_entry);
1952         if (err) {
1953                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to insert FIB4 entry being added.\n");
1954                 goto err_fib_entry_insert;
1955         }
1956         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1957         if (err)
1958                 goto err_fib_entry_add;
1959         return 0;
1960
1961 err_fib_entry_add:
1962         mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
1963 err_fib_entry_insert:
1964         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1965         return err;
1966 }
1967
1968 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
1969                                      struct fib_entry_notifier_info *fen_info)
1970 {
1971         struct mlxsw_sp_fib_entry *fib_entry;
1972
1973         if (mlxsw_sp->router.aborted)
1974                 return;
1975
1976         fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info);
1977         if (!fib_entry)
1978                 return;
1979
1980         if (fib_entry->ref_count == 1) {
1981                 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1982                 mlxsw_sp_fib_entry_remove(fib_entry->vr->fib, fib_entry);
1983         }
1984
1985         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1986 }
1987
1988 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
1989 {
1990         char ralta_pl[MLXSW_REG_RALTA_LEN];
1991         char ralst_pl[MLXSW_REG_RALST_LEN];
1992         char raltb_pl[MLXSW_REG_RALTB_LEN];
1993         char ralue_pl[MLXSW_REG_RALUE_LEN];
1994         int err;
1995
1996         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1997                              MLXSW_SP_LPM_TREE_MIN);
1998         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
1999         if (err)
2000                 return err;
2001
2002         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2003         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2004         if (err)
2005                 return err;
2006
2007         mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2008                              MLXSW_SP_LPM_TREE_MIN);
2009         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
2010         if (err)
2011                 return err;
2012
2013         mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2014                               MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
2015         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2016         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2017 }
2018
2019 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2020 {
2021         struct mlxsw_sp_fib_entry *fib_entry;
2022         struct mlxsw_sp_fib_entry *tmp;
2023         struct mlxsw_sp_vr *vr;
2024         int i;
2025
2026         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2027                 vr = &mlxsw_sp->router.vrs[i];
2028
2029                 if (!vr->used)
2030                         continue;
2031
2032                 list_for_each_entry_safe(fib_entry, tmp,
2033                                          &vr->fib->entry_list, list) {
2034                         bool do_break = &tmp->list == &vr->fib->entry_list;
2035
2036                         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2037                         mlxsw_sp_fib_entry_remove(fib_entry->vr->fib,
2038                                                   fib_entry);
2039                         mlxsw_sp_fib_entry_put_all(mlxsw_sp, fib_entry);
2040                         if (do_break)
2041                                 break;
2042                 }
2043         }
2044 }
2045
2046 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2047 {
2048         int err;
2049
2050         if (mlxsw_sp->router.aborted)
2051                 return;
2052         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2053         mlxsw_sp_router_fib_flush(mlxsw_sp);
2054         mlxsw_sp->router.aborted = true;
2055         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2056         if (err)
2057                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2058 }
2059
2060 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
2061 {
2062         char rgcr_pl[MLXSW_REG_RGCR_LEN];
2063         u64 max_rifs;
2064         int err;
2065
2066         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
2067                 return -EIO;
2068
2069         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2070         mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
2071                                  GFP_KERNEL);
2072         if (!mlxsw_sp->rifs)
2073                 return -ENOMEM;
2074
2075         mlxsw_reg_rgcr_pack(rgcr_pl, true);
2076         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
2077         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
2078         if (err)
2079                 goto err_rgcr_fail;
2080
2081         return 0;
2082
2083 err_rgcr_fail:
2084         kfree(mlxsw_sp->rifs);
2085         return err;
2086 }
2087
2088 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2089 {
2090         char rgcr_pl[MLXSW_REG_RGCR_LEN];
2091         int i;
2092
2093         mlxsw_reg_rgcr_pack(rgcr_pl, false);
2094         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
2095
2096         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2097                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
2098
2099         kfree(mlxsw_sp->rifs);
2100 }
2101
2102 struct mlxsw_sp_fib_event_work {
2103         struct work_struct work;
2104         struct fib_entry_notifier_info fen_info;
2105         struct mlxsw_sp *mlxsw_sp;
2106         unsigned long event;
2107 };
2108
2109 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2110 {
2111         struct mlxsw_sp_fib_event_work *fib_work =
2112                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2113         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2114         int err;
2115
2116         /* Protect internal structures from changes */
2117         rtnl_lock();
2118         switch (fib_work->event) {
2119         case FIB_EVENT_ENTRY_ADD:
2120                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info);
2121                 if (err)
2122                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2123                 fib_info_put(fib_work->fen_info.fi);
2124                 break;
2125         case FIB_EVENT_ENTRY_DEL:
2126                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2127                 fib_info_put(fib_work->fen_info.fi);
2128                 break;
2129         case FIB_EVENT_RULE_ADD: /* fall through */
2130         case FIB_EVENT_RULE_DEL:
2131                 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2132                 break;
2133         }
2134         rtnl_unlock();
2135         kfree(fib_work);
2136 }
2137
2138 /* Called with rcu_read_lock() */
2139 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2140                                      unsigned long event, void *ptr)
2141 {
2142         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2143         struct mlxsw_sp_fib_event_work *fib_work;
2144         struct fib_notifier_info *info = ptr;
2145
2146         if (!net_eq(info->net, &init_net))
2147                 return NOTIFY_DONE;
2148
2149         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2150         if (WARN_ON(!fib_work))
2151                 return NOTIFY_BAD;
2152
2153         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2154         fib_work->mlxsw_sp = mlxsw_sp;
2155         fib_work->event = event;
2156
2157         switch (event) {
2158         case FIB_EVENT_ENTRY_ADD: /* fall through */
2159         case FIB_EVENT_ENTRY_DEL:
2160                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2161                 /* Take referece on fib_info to prevent it from being
2162                  * freed while work is queued. Release it afterwards.
2163                  */
2164                 fib_info_hold(fib_work->fen_info.fi);
2165                 break;
2166         }
2167
2168         mlxsw_core_schedule_work(&fib_work->work);
2169
2170         return NOTIFY_DONE;
2171 }
2172
2173 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
2174 {
2175         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2176
2177         /* Flush pending FIB notifications and then flush the device's
2178          * table before requesting another dump. The FIB notification
2179          * block is unregistered, so no need to take RTNL.
2180          */
2181         mlxsw_core_flush_owq();
2182         mlxsw_sp_router_fib_flush(mlxsw_sp);
2183 }
2184
2185 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
2186 {
2187         int err;
2188
2189         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
2190         err = __mlxsw_sp_router_init(mlxsw_sp);
2191         if (err)
2192                 return err;
2193
2194         err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
2195                               &mlxsw_sp_nexthop_ht_params);
2196         if (err)
2197                 goto err_nexthop_ht_init;
2198
2199         err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
2200                               &mlxsw_sp_nexthop_group_ht_params);
2201         if (err)
2202                 goto err_nexthop_group_ht_init;
2203
2204         mlxsw_sp_lpm_init(mlxsw_sp);
2205         err = mlxsw_sp_vrs_init(mlxsw_sp);
2206         if (err)
2207                 goto err_vrs_init;
2208
2209         err = mlxsw_sp_neigh_init(mlxsw_sp);
2210         if (err)
2211                 goto err_neigh_init;
2212
2213         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
2214         err = register_fib_notifier(&mlxsw_sp->fib_nb,
2215                                     mlxsw_sp_router_fib_dump_flush);
2216         if (err)
2217                 goto err_register_fib_notifier;
2218
2219         return 0;
2220
2221 err_register_fib_notifier:
2222         mlxsw_sp_neigh_fini(mlxsw_sp);
2223 err_neigh_init:
2224         mlxsw_sp_vrs_fini(mlxsw_sp);
2225 err_vrs_init:
2226         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
2227 err_nexthop_group_ht_init:
2228         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
2229 err_nexthop_ht_init:
2230         __mlxsw_sp_router_fini(mlxsw_sp);
2231         return err;
2232 }
2233
2234 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2235 {
2236         unregister_fib_notifier(&mlxsw_sp->fib_nb);
2237         mlxsw_sp_neigh_fini(mlxsw_sp);
2238         mlxsw_sp_vrs_fini(mlxsw_sp);
2239         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
2240         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
2241         __mlxsw_sp_router_fini(mlxsw_sp);
2242 }