drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <net/netevent.h>
  46 #include <net/neighbour.h>
  47 #include <net/arp.h>
  48 #include <net/ip_fib.h>
  49 #include <net/fib_rules.h>
  50 #include <net/l3mdev.h>
  51
  52 #include "spectrum.h"
  53 #include "core.h"
  54 #include "reg.h"
  55
  56 struct mlxsw_sp_rif {
  57         struct list_head nexthop_list;
  58         struct list_head neigh_list;
  59         struct net_device *dev;
  60         struct mlxsw_sp_fid *f;
  61         unsigned char addr[ETH_ALEN];
  62         int mtu;
  63         u16 rif_index;
  64         u16 vr_id;
  65 };
  66
  67 static struct mlxsw_sp_rif *
  68 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
  69                          const struct net_device *dev);
  70
  71 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  72         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  73
  74 static bool
  75 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  76                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  77 {
  78         unsigned char prefix;
  79
  80         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  81                 if (!test_bit(prefix, prefix_usage2->b))
  82                         return false;
  83         }
  84         return true;
  85 }
  86
  87 static bool
  88 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  89                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  90 {
  91         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  92 }
  93
  94 static bool
  95 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  96 {
  97         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  98
  99         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 100 }
 101
 102 static void
 103 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 104                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 105 {
 106         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 107 }
 108
 109 static void
 110 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 111                           unsigned char prefix_len)
 112 {
 113         set_bit(prefix_len, prefix_usage->b);
 114 }
 115
 116 static void
 117 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 118                             unsigned char prefix_len)
 119 {
 120         clear_bit(prefix_len, prefix_usage->b);
 121 }
 122
 123 struct mlxsw_sp_fib_key {
 124         unsigned char addr[sizeof(struct in6_addr)];
 125         unsigned char prefix_len;
 126 };
 127
 128 enum mlxsw_sp_fib_entry_type {
 129         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 130         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 131         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 132 };
 133
 134 struct mlxsw_sp_nexthop_group;
 135
 136 struct mlxsw_sp_fib_node {
 137         struct list_head entry_list;
 138         struct list_head list;
 139         struct rhash_head ht_node;
 140         struct mlxsw_sp_fib *fib;
 141         struct mlxsw_sp_fib_key key;
 142 };
 143
 144 struct mlxsw_sp_fib_entry_params {
 145         u32 tb_id;
 146         u32 prio;
 147         u8 tos;
 148         u8 type;
 149 };
 150
 151 struct mlxsw_sp_fib_entry {
 152         struct list_head list;
 153         struct mlxsw_sp_fib_node *fib_node;
 154         enum mlxsw_sp_fib_entry_type type;
 155         struct list_head nexthop_group_node;
 156         struct mlxsw_sp_nexthop_group *nh_group;
 157         struct mlxsw_sp_fib_entry_params params;
 158         bool offloaded;
 159 };
 160
 161 struct mlxsw_sp_fib {
 162         struct rhashtable ht;
 163         struct list_head node_list;
 164         struct mlxsw_sp_vr *vr;
 165         struct mlxsw_sp_lpm_tree *lpm_tree;
 166         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 167         struct mlxsw_sp_prefix_usage prefix_usage;
 168         enum mlxsw_sp_l3proto proto;
 169 };
 170
 171 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 172
 173 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 174                                                 enum mlxsw_sp_l3proto proto)
 175 {
 176         struct mlxsw_sp_fib *fib;
 177         int err;
 178
 179         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 180         if (!fib)
 181                 return ERR_PTR(-ENOMEM);
 182         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 183         if (err)
 184                 goto err_rhashtable_init;
 185         INIT_LIST_HEAD(&fib->node_list);
 186         fib->proto = proto;
 187         fib->vr = vr;
 188         return fib;
 189
 190 err_rhashtable_init:
 191         kfree(fib);
 192         return ERR_PTR(err);
 193 }
 194
 195 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 196 {
 197         WARN_ON(!list_empty(&fib->node_list));
 198         WARN_ON(fib->lpm_tree);
 199         rhashtable_destroy(&fib->ht);
 200         kfree(fib);
 201 }
 202
 203 static struct mlxsw_sp_lpm_tree *
 204 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 205 {
 206         static struct mlxsw_sp_lpm_tree *lpm_tree;
 207         int i;
 208
 209         for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
 210                 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
 211                 if (lpm_tree->ref_count == 0)
 212                         return lpm_tree;
 213         }
 214         return NULL;
 215 }
 216
 217 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 218                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 219 {
 220         char ralta_pl[MLXSW_REG_RALTA_LEN];
 221
 222         mlxsw_reg_ralta_pack(ralta_pl, true,
 223                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 224                              lpm_tree->id);
 225         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 226 }
 227
 228 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 229                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 230 {
 231         char ralta_pl[MLXSW_REG_RALTA_LEN];
 232
 233         mlxsw_reg_ralta_pack(ralta_pl, false,
 234                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 235                              lpm_tree->id);
 236         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 237 }
 238
 239 static int
 240 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 241                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 242                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 243 {
 244         char ralst_pl[MLXSW_REG_RALST_LEN];
 245         u8 root_bin = 0;
 246         u8 prefix;
 247         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 248
 249         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 250                 root_bin = prefix;
 251
 252         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 253         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 254                 if (prefix == 0)
 255                         continue;
 256                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 257                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 258                 last_prefix = prefix;
 259         }
 260         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 261 }
 262
 263 static struct mlxsw_sp_lpm_tree *
 264 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 265                          struct mlxsw_sp_prefix_usage *prefix_usage,
 266                          enum mlxsw_sp_l3proto proto)
 267 {
 268         struct mlxsw_sp_lpm_tree *lpm_tree;
 269         int err;
 270
 271         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 272         if (!lpm_tree)
 273                 return ERR_PTR(-EBUSY);
 274         lpm_tree->proto = proto;
 275         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 276         if (err)
 277                 return ERR_PTR(err);
 278
 279         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 280                                                 lpm_tree);
 281         if (err)
 282                 goto err_left_struct_set;
 283         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 284                sizeof(lpm_tree->prefix_usage));
 285         return lpm_tree;
 286
 287 err_left_struct_set:
 288         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 289         return ERR_PTR(err);
 290 }
 291
 292 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 293                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 294 {
 295         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 296 }
 297
 298 static struct mlxsw_sp_lpm_tree *
 299 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 300                       struct mlxsw_sp_prefix_usage *prefix_usage,
 301                       enum mlxsw_sp_l3proto proto)
 302 {
 303         struct mlxsw_sp_lpm_tree *lpm_tree;
 304         int i;
 305
 306         for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
 307                 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
 308                 if (lpm_tree->ref_count != 0 &&
 309                     lpm_tree->proto == proto &&
 310                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 311                                              prefix_usage))
 312                         goto inc_ref_count;
 313         }
 314         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 315                                             proto);
 316         if (IS_ERR(lpm_tree))
 317                 return lpm_tree;
 318
 319 inc_ref_count:
 320         lpm_tree->ref_count++;
 321         return lpm_tree;
 322 }
 323
 324 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 325                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 326 {
 327         if (--lpm_tree->ref_count == 0)
 328                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 329         return 0;
 330 }
 331
 332 #define MLXSW_SP_LPM_TREE_MIN 2 /* trees 0 and 1 are reserved */
 333
 334 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 335 {
 336         struct mlxsw_sp_lpm_tree *lpm_tree;
 337         u64 max_trees;
 338         int i;
 339
 340         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 341                 return -EIO;
 342
 343         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 344         mlxsw_sp->router.lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 345         mlxsw_sp->router.lpm.trees = kcalloc(mlxsw_sp->router.lpm.tree_count,
 346                                              sizeof(struct mlxsw_sp_lpm_tree),
 347                                              GFP_KERNEL);
 348         if (!mlxsw_sp->router.lpm.trees)
 349                 return -ENOMEM;
 350
 351         for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
 352                 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
 353                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 354         }
 355
 356         return 0;
 357 }
 358
 359 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 360 {
 361         kfree(mlxsw_sp->router.lpm.trees);
 362 }
 363
 364 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 365 {
 366         return !!vr->fib4;
 367 }
 368
 369 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 370 {
 371         struct mlxsw_sp_vr *vr;
 372         int i;
 373
 374         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 375                 vr = &mlxsw_sp->router.vrs[i];
 376                 if (!mlxsw_sp_vr_is_used(vr))
 377                         return vr;
 378         }
 379         return NULL;
 380 }
 381
 382 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 383                                      const struct mlxsw_sp_fib *fib)
 384 {
 385         char raltb_pl[MLXSW_REG_RALTB_LEN];
 386
 387         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 388                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 389                              fib->lpm_tree->id);
 390         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 391 }
 392
 393 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 394                                        const struct mlxsw_sp_fib *fib)
 395 {
 396         char raltb_pl[MLXSW_REG_RALTB_LEN];
 397
 398         /* Bind to tree 0 which is default */
 399         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 400                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 401         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 402 }
 403
 404 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 405 {
 406         /* For our purpose, squash main and local table into one */
 407         if (tb_id == RT_TABLE_LOCAL)
 408                 tb_id = RT_TABLE_MAIN;
 409         return tb_id;
 410 }
 411
 412 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 413                                             u32 tb_id)
 414 {
 415         struct mlxsw_sp_vr *vr;
 416         int i;
 417
 418         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 419
 420         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 421                 vr = &mlxsw_sp->router.vrs[i];
 422                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 423                         return vr;
 424         }
 425         return NULL;
 426 }
 427
 428 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 429                                             enum mlxsw_sp_l3proto proto)
 430 {
 431         switch (proto) {
 432         case MLXSW_SP_L3_PROTO_IPV4:
 433                 return vr->fib4;
 434         case MLXSW_SP_L3_PROTO_IPV6:
 435                 BUG_ON(1);
 436         }
 437         return NULL;
 438 }
 439
 440 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 441                                               u32 tb_id)
 442 {
 443         struct mlxsw_sp_vr *vr;
 444
 445         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 446         if (!vr)
 447                 return ERR_PTR(-EBUSY);
 448         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 449         if (IS_ERR(vr->fib4))
 450                 return ERR_CAST(vr->fib4);
 451         vr->tb_id = tb_id;
 452         return vr;
 453 }
 454
 455 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 456 {
 457         mlxsw_sp_fib_destroy(vr->fib4);
 458         vr->fib4 = NULL;
 459 }
 460
 461 static int
 462 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 463                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 464 {
 465         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 466         struct mlxsw_sp_lpm_tree *new_tree;
 467         int err;
 468
 469         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 470                 return 0;
 471
 472         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 473                                          fib->proto);
 474         if (IS_ERR(new_tree)) {
 475                 /* We failed to get a tree according to the required
 476                  * prefix usage. However, the current tree might be still good
 477                  * for us if our requirement is subset of the prefixes used
 478                  * in the tree.
 479                  */
 480                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 481                                                  &lpm_tree->prefix_usage))
 482                         return 0;
 483                 return PTR_ERR(new_tree);
 484         }
 485
 486         /* Prevent packet loss by overwriting existing binding */
 487         fib->lpm_tree = new_tree;
 488         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 489         if (err)
 490                 goto err_tree_bind;
 491         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 492
 493         return 0;
 494
 495 err_tree_bind:
 496         fib->lpm_tree = lpm_tree;
 497         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 498         return err;
 499 }
 500
 501 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 502 {
 503         struct mlxsw_sp_vr *vr;
 504
 505         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 506         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 507         if (!vr)
 508                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 509         return vr;
 510 }
 511
 512 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 513 {
 514         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 515                 mlxsw_sp_vr_destroy(vr);
 516 }
 517
 518 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 519 {
 520         struct mlxsw_sp_vr *vr;
 521         u64 max_vrs;
 522         int i;
 523
 524         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 525                 return -EIO;
 526
 527         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 528         mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 529                                        GFP_KERNEL);
 530         if (!mlxsw_sp->router.vrs)
 531                 return -ENOMEM;
 532
 533         for (i = 0; i < max_vrs; i++) {
 534                 vr = &mlxsw_sp->router.vrs[i];
 535                 vr->id = i;
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 542
 543 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 544 {
 545         /* At this stage we're guaranteed not to have new incoming
 546          * FIB notifications and the work queue is free from FIBs
 547          * sitting on top of mlxsw netdevs. However, we can still
 548          * have other FIBs queued. Flush the queue before flushing
 549          * the device's tables. No need for locks, as we're the only
 550          * writer.
 551          */
 552         mlxsw_core_flush_owq();
 553         mlxsw_sp_router_fib_flush(mlxsw_sp);
 554         kfree(mlxsw_sp->router.vrs);
 555 }
 556
 557 struct mlxsw_sp_neigh_key {
 558         struct neighbour *n;
 559 };
 560
 561 struct mlxsw_sp_neigh_entry {
 562         struct list_head rif_list_node;
 563         struct rhash_head ht_node;
 564         struct mlxsw_sp_neigh_key key;
 565         u16 rif;
 566         bool connected;
 567         unsigned char ha[ETH_ALEN];
 568         struct list_head nexthop_list; /* list of nexthops using
 569                                         * this neigh entry
 570                                         */
 571         struct list_head nexthop_neighs_list_node;
 572 };
 573
 574 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 575         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 576         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 577         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 578 };
 579
 580 static struct mlxsw_sp_neigh_entry *
 581 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 582                            u16 rif)
 583 {
 584         struct mlxsw_sp_neigh_entry *neigh_entry;
 585
 586         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 587         if (!neigh_entry)
 588                 return NULL;
 589
 590         neigh_entry->key.n = n;
 591         neigh_entry->rif = rif;
 592         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 593
 594         return neigh_entry;
 595 }
 596
 597 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 598 {
 599         kfree(neigh_entry);
 600 }
 601
 602 static int
 603 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 604                             struct mlxsw_sp_neigh_entry *neigh_entry)
 605 {
 606         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 607                                       &neigh_entry->ht_node,
 608                                       mlxsw_sp_neigh_ht_params);
 609 }
 610
 611 static void
 612 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 613                             struct mlxsw_sp_neigh_entry *neigh_entry)
 614 {
 615         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 616                                &neigh_entry->ht_node,
 617                                mlxsw_sp_neigh_ht_params);
 618 }
 619
 620 static struct mlxsw_sp_neigh_entry *
 621 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 622 {
 623         struct mlxsw_sp_neigh_entry *neigh_entry;
 624         struct mlxsw_sp_rif *rif;
 625         int err;
 626
 627         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 628         if (!rif)
 629                 return ERR_PTR(-EINVAL);
 630
 631         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 632         if (!neigh_entry)
 633                 return ERR_PTR(-ENOMEM);
 634
 635         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 636         if (err)
 637                 goto err_neigh_entry_insert;
 638
 639         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 640
 641         return neigh_entry;
 642
 643 err_neigh_entry_insert:
 644         mlxsw_sp_neigh_entry_free(neigh_entry);
 645         return ERR_PTR(err);
 646 }
 647
 648 static void
 649 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 650                              struct mlxsw_sp_neigh_entry *neigh_entry)
 651 {
 652         list_del(&neigh_entry->rif_list_node);
 653         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 654         mlxsw_sp_neigh_entry_free(neigh_entry);
 655 }
 656
 657 static struct mlxsw_sp_neigh_entry *
 658 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 659 {
 660         struct mlxsw_sp_neigh_key key;
 661
 662         key.n = n;
 663         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 664                                       &key, mlxsw_sp_neigh_ht_params);
 665 }
 666
 667 static void
 668 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 669 {
 670         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 671
 672         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 673 }
 674
 675 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 676                                                    char *rauhtd_pl,
 677                                                    int ent_index)
 678 {
 679         struct net_device *dev;
 680         struct neighbour *n;
 681         __be32 dipn;
 682         u32 dip;
 683         u16 rif;
 684
 685         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 686
 687         if (!mlxsw_sp->rifs[rif]) {
 688                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 689                 return;
 690         }
 691
 692         dipn = htonl(dip);
 693         dev = mlxsw_sp->rifs[rif]->dev;
 694         n = neigh_lookup(&arp_tbl, &dipn, dev);
 695         if (!n) {
 696                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 697                            &dip);
 698                 return;
 699         }
 700
 701         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 702         neigh_event_send(n, NULL);
 703         neigh_release(n);
 704 }
 705
 706 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 707                                                    char *rauhtd_pl,
 708                                                    int rec_index)
 709 {
 710         u8 num_entries;
 711         int i;
 712
 713         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 714                                                                 rec_index);
 715         /* Hardware starts counting at 0, so add 1. */
 716         num_entries++;
 717
 718         /* Each record consists of several neighbour entries. */
 719         for (i = 0; i < num_entries; i++) {
 720                 int ent_index;
 721
 722                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 723                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 724                                                        ent_index);
 725         }
 726
 727 }
 728
 729 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 730                                               char *rauhtd_pl, int rec_index)
 731 {
 732         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 733         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 734                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 735                                                        rec_index);
 736                 break;
 737         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 738                 WARN_ON_ONCE(1);
 739                 break;
 740         }
 741 }
 742
 743 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 744 {
 745         u8 num_rec, last_rec_index, num_entries;
 746
 747         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 748         last_rec_index = num_rec - 1;
 749
 750         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 751                 return false;
 752         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 753             MLXSW_REG_RAUHTD_TYPE_IPV6)
 754                 return true;
 755
 756         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 757                                                                 last_rec_index);
 758         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 759                 return true;
 760         return false;
 761 }
 762
 763 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 764 {
 765         char *rauhtd_pl;
 766         u8 num_rec;
 767         int i, err;
 768
 769         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 770         if (!rauhtd_pl)
 771                 return -ENOMEM;
 772
 773         /* Make sure the neighbour's netdev isn't removed in the
 774          * process.
 775          */
 776         rtnl_lock();
 777         do {
 778                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 779                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 780                                       rauhtd_pl);
 781                 if (err) {
 782                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 783                         break;
 784                 }
 785                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 786                 for (i = 0; i < num_rec; i++)
 787                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 788                                                           i);
 789         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 790         rtnl_unlock();
 791
 792         kfree(rauhtd_pl);
 793         return err;
 794 }
 795
 796 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 797 {
 798         struct mlxsw_sp_neigh_entry *neigh_entry;
 799
 800         /* Take RTNL mutex here to prevent lists from changes */
 801         rtnl_lock();
 802         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 803                             nexthop_neighs_list_node)
 804                 /* If this neigh have nexthops, make the kernel think this neigh
 805                  * is active regardless of the traffic.
 806                  */
 807                 neigh_event_send(neigh_entry->key.n, NULL);
 808         rtnl_unlock();
 809 }
 810
 811 static void
 812 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 813 {
 814         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 815
 816         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 817                                msecs_to_jiffies(interval));
 818 }
 819
 820 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 821 {
 822         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 823                                                  router.neighs_update.dw.work);
 824         int err;
 825
 826         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 827         if (err)
 828                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 829
 830         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 831
 832         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 833 }
 834
 835 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 836 {
 837         struct mlxsw_sp_neigh_entry *neigh_entry;
 838         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 839                                                  router.nexthop_probe_dw.work);
 840
 841         /* Iterate over nexthop neighbours, find those who are unresolved and
 842          * send arp on them. This solves the chicken-egg problem when
 843          * the nexthop wouldn't get offloaded until the neighbor is resolved
 844          * but it wouldn't get resolved ever in case traffic is flowing in HW
 845          * using different nexthop.
 846          *
 847          * Take RTNL mutex here to prevent lists from changes.
 848          */
 849         rtnl_lock();
 850         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 851                             nexthop_neighs_list_node)
 852                 if (!neigh_entry->connected)
 853                         neigh_event_send(neigh_entry->key.n, NULL);
 854         rtnl_unlock();
 855
 856         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 857                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 858 }
 859
 860 static void
 861 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 862                               struct mlxsw_sp_neigh_entry *neigh_entry,
 863                               bool removing);
 864
 865 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
 866 {
 867         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
 868                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
 869 }
 870
 871 static void
 872 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 873                                 struct mlxsw_sp_neigh_entry *neigh_entry,
 874                                 enum mlxsw_reg_rauht_op op)
 875 {
 876         struct neighbour *n = neigh_entry->key.n;
 877         u32 dip = ntohl(*((__be32 *) n->primary_key));
 878         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 879
 880         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 881                               dip);
 882         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 883 }
 884
 885 static void
 886 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 887                             struct mlxsw_sp_neigh_entry *neigh_entry,
 888                             bool adding)
 889 {
 890         if (!adding && !neigh_entry->connected)
 891                 return;
 892         neigh_entry->connected = adding;
 893         if (neigh_entry->key.n->tbl == &arp_tbl)
 894                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 895                                                 mlxsw_sp_rauht_op(adding));
 896         else
 897                 WARN_ON_ONCE(1);
 898 }
 899
 900 struct mlxsw_sp_neigh_event_work {
 901         struct work_struct work;
 902         struct mlxsw_sp *mlxsw_sp;
 903         struct neighbour *n;
 904 };
 905
 906 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
 907 {
 908         struct mlxsw_sp_neigh_event_work *neigh_work =
 909                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
 910         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
 911         struct mlxsw_sp_neigh_entry *neigh_entry;
 912         struct neighbour *n = neigh_work->n;
 913         unsigned char ha[ETH_ALEN];
 914         bool entry_connected;
 915         u8 nud_state, dead;
 916
 917         /* If these parameters are changed after we release the lock,
 918          * then we are guaranteed to receive another event letting us
 919          * know about it.
 920          */
 921         read_lock_bh(&n->lock);
 922         memcpy(ha, n->ha, ETH_ALEN);
 923         nud_state = n->nud_state;
 924         dead = n->dead;
 925         read_unlock_bh(&n->lock);
 926
 927         rtnl_lock();
 928         entry_connected = nud_state & NUD_VALID && !dead;
 929         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 930         if (!entry_connected && !neigh_entry)
 931                 goto out;
 932         if (!neigh_entry) {
 933                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
 934                 if (IS_ERR(neigh_entry))
 935                         goto out;
 936         }
 937
 938         memcpy(neigh_entry->ha, ha, ETH_ALEN);
 939         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
 940         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
 941
 942         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 943                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
 944
 945 out:
 946         rtnl_unlock();
 947         neigh_release(n);
 948         kfree(neigh_work);
 949 }
 950
 951 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 952                                    unsigned long event, void *ptr)
 953 {
 954         struct mlxsw_sp_neigh_event_work *neigh_work;
 955         struct mlxsw_sp_port *mlxsw_sp_port;
 956         struct mlxsw_sp *mlxsw_sp;
 957         unsigned long interval;
 958         struct neigh_parms *p;
 959         struct neighbour *n;
 960
 961         switch (event) {
 962         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
 963                 p = ptr;
 964
 965                 /* We don't care about changes in the default table. */
 966                 if (!p->dev || p->tbl != &arp_tbl)
 967                         return NOTIFY_DONE;
 968
 969                 /* We are in atomic context and can't take RTNL mutex,
 970                  * so use RCU variant to walk the device chain.
 971                  */
 972                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
 973                 if (!mlxsw_sp_port)
 974                         return NOTIFY_DONE;
 975
 976                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 977                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
 978                 mlxsw_sp->router.neighs_update.interval = interval;
 979
 980                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 981                 break;
 982         case NETEVENT_NEIGH_UPDATE:
 983                 n = ptr;
 984
 985                 if (n->tbl != &arp_tbl)
 986                         return NOTIFY_DONE;
 987
 988                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
 989                 if (!mlxsw_sp_port)
 990                         return NOTIFY_DONE;
 991
 992                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
 993                 if (!neigh_work) {
 994                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 995                         return NOTIFY_BAD;
 996                 }
 997
 998                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
 999                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1000                 neigh_work->n = n;
1001
1002                 /* Take a reference to ensure the neighbour won't be
1003                  * destructed until we drop the reference in delayed
1004                  * work.
1005                  */
1006                 neigh_clone(n);
1007                 mlxsw_core_schedule_work(&neigh_work->work);
1008                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1009                 break;
1010         }
1011
1012         return NOTIFY_DONE;
1013 }
1014
1015 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1016 {
1017         int err;
1018
1019         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1020                               &mlxsw_sp_neigh_ht_params);
1021         if (err)
1022                 return err;
1023
1024         /* Initialize the polling interval according to the default
1025          * table.
1026          */
1027         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1028
1029         /* Create the delayed works for the activity_update */
1030         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1031                           mlxsw_sp_router_neighs_update_work);
1032         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1033                           mlxsw_sp_router_probe_unresolved_nexthops);
1034         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1035         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1036         return 0;
1037 }
1038
1039 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1040 {
1041         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1042         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1043         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1044 }
1045
1046 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1047                                     const struct mlxsw_sp_rif *rif)
1048 {
1049         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1050
1051         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1052                              rif->rif_index, rif->addr);
1053         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1054 }
1055
1056 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1057                                          struct mlxsw_sp_rif *rif)
1058 {
1059         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1060
1061         mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1062         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1063                                  rif_list_node)
1064                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1065 }
1066
1067 struct mlxsw_sp_nexthop_key {
1068         struct fib_nh *fib_nh;
1069 };
1070
1071 struct mlxsw_sp_nexthop {
1072         struct list_head neigh_list_node; /* member of neigh entry list */
1073         struct list_head rif_list_node;
1074         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1075                                                 * this belongs to
1076                                                 */
1077         struct rhash_head ht_node;
1078         struct mlxsw_sp_nexthop_key key;
1079         struct mlxsw_sp_rif *rif;
1080         u8 should_offload:1, /* set indicates this neigh is connected and
1081                               * should be put to KVD linear area of this group.
1082                               */
1083            offloaded:1, /* set in case the neigh is actually put into
1084                          * KVD linear area of this group.
1085                          */
1086            update:1; /* set indicates that MAC of this neigh should be
1087                       * updated in HW
1088                       */
1089         struct mlxsw_sp_neigh_entry *neigh_entry;
1090 };
1091
1092 struct mlxsw_sp_nexthop_group_key {
1093         struct fib_info *fi;
1094 };
1095
1096 struct mlxsw_sp_nexthop_group {
1097         struct rhash_head ht_node;
1098         struct list_head fib_list; /* list of fib entries that use this group */
1099         struct mlxsw_sp_nexthop_group_key key;
1100         u8 adj_index_valid:1,
1101            gateway:1; /* routes using the group use a gateway */
1102         u32 adj_index;
1103         u16 ecmp_size;
1104         u16 count;
1105         struct mlxsw_sp_nexthop nexthops[0];
1106 #define nh_rif  nexthops[0].rif
1107 };
1108
1109 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1110         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1111         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1112         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1113 };
1114
1115 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1116                                          struct mlxsw_sp_nexthop_group *nh_grp)
1117 {
1118         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1119                                       &nh_grp->ht_node,
1120                                       mlxsw_sp_nexthop_group_ht_params);
1121 }
1122
1123 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1124                                           struct mlxsw_sp_nexthop_group *nh_grp)
1125 {
1126         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1127                                &nh_grp->ht_node,
1128                                mlxsw_sp_nexthop_group_ht_params);
1129 }
1130
1131 static struct mlxsw_sp_nexthop_group *
1132 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1133                               struct mlxsw_sp_nexthop_group_key key)
1134 {
1135         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1136                                       mlxsw_sp_nexthop_group_ht_params);
1137 }
1138
1139 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1140         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1141         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1142         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1143 };
1144
1145 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1146                                    struct mlxsw_sp_nexthop *nh)
1147 {
1148         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1149                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1150 }
1151
1152 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1153                                     struct mlxsw_sp_nexthop *nh)
1154 {
1155         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1156                                mlxsw_sp_nexthop_ht_params);
1157 }
1158
1159 static struct mlxsw_sp_nexthop *
1160 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1161                         struct mlxsw_sp_nexthop_key key)
1162 {
1163         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1164                                       mlxsw_sp_nexthop_ht_params);
1165 }
1166
1167 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1168                                              const struct mlxsw_sp_fib *fib,
1169                                              u32 adj_index, u16 ecmp_size,
1170                                              u32 new_adj_index,
1171                                              u16 new_ecmp_size)
1172 {
1173         char raleu_pl[MLXSW_REG_RALEU_LEN];
1174
1175         mlxsw_reg_raleu_pack(raleu_pl,
1176                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1177                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1178                              new_ecmp_size);
1179         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1180 }
1181
1182 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1183                                           struct mlxsw_sp_nexthop_group *nh_grp,
1184                                           u32 old_adj_index, u16 old_ecmp_size)
1185 {
1186         struct mlxsw_sp_fib_entry *fib_entry;
1187         struct mlxsw_sp_fib *fib = NULL;
1188         int err;
1189
1190         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1191                 if (fib == fib_entry->fib_node->fib)
1192                         continue;
1193                 fib = fib_entry->fib_node->fib;
1194                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1195                                                         old_adj_index,
1196                                                         old_ecmp_size,
1197                                                         nh_grp->adj_index,
1198                                                         nh_grp->ecmp_size);
1199                 if (err)
1200                         return err;
1201         }
1202         return 0;
1203 }
1204
1205 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1206                                        struct mlxsw_sp_nexthop *nh)
1207 {
1208         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1209         char ratr_pl[MLXSW_REG_RATR_LEN];
1210
1211         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1212                             true, adj_index, neigh_entry->rif);
1213         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1214         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1215 }
1216
1217 static int
1218 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1219                                   struct mlxsw_sp_nexthop_group *nh_grp,
1220                                   bool reallocate)
1221 {
1222         u32 adj_index = nh_grp->adj_index; /* base */
1223         struct mlxsw_sp_nexthop *nh;
1224         int i;
1225         int err;
1226
1227         for (i = 0; i < nh_grp->count; i++) {
1228                 nh = &nh_grp->nexthops[i];
1229
1230                 if (!nh->should_offload) {
1231                         nh->offloaded = 0;
1232                         continue;
1233                 }
1234
1235                 if (nh->update || reallocate) {
1236                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1237                                                           adj_index, nh);
1238                         if (err)
1239                                 return err;
1240                         nh->update = 0;
1241                         nh->offloaded = 1;
1242                 }
1243                 adj_index++;
1244         }
1245         return 0;
1246 }
1247
1248 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1249                                      struct mlxsw_sp_fib_entry *fib_entry);
1250
1251 static int
1252 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1253                                     struct mlxsw_sp_nexthop_group *nh_grp)
1254 {
1255         struct mlxsw_sp_fib_entry *fib_entry;
1256         int err;
1257
1258         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1259                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1260                 if (err)
1261                         return err;
1262         }
1263         return 0;
1264 }
1265
1266 static void
1267 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1268                                struct mlxsw_sp_nexthop_group *nh_grp)
1269 {
1270         struct mlxsw_sp_nexthop *nh;
1271         bool offload_change = false;
1272         u32 adj_index;
1273         u16 ecmp_size = 0;
1274         bool old_adj_index_valid;
1275         u32 old_adj_index;
1276         u16 old_ecmp_size;
1277         int ret;
1278         int i;
1279         int err;
1280
1281         if (!nh_grp->gateway) {
1282                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1283                 return;
1284         }
1285
1286         for (i = 0; i < nh_grp->count; i++) {
1287                 nh = &nh_grp->nexthops[i];
1288
1289                 if (nh->should_offload ^ nh->offloaded) {
1290                         offload_change = true;
1291                         if (nh->should_offload)
1292                                 nh->update = 1;
1293                 }
1294                 if (nh->should_offload)
1295                         ecmp_size++;
1296         }
1297         if (!offload_change) {
1298                 /* Nothing was added or removed, so no need to reallocate. Just
1299                  * update MAC on existing adjacency indexes.
1300                  */
1301                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1302                                                         false);
1303                 if (err) {
1304                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1305                         goto set_trap;
1306                 }
1307                 return;
1308         }
1309         if (!ecmp_size)
1310                 /* No neigh of this group is connected so we just set
1311                  * the trap and let everthing flow through kernel.
1312                  */
1313                 goto set_trap;
1314
1315         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1316         if (ret < 0) {
1317                 /* We ran out of KVD linear space, just set the
1318                  * trap and let everything flow through kernel.
1319                  */
1320                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1321                 goto set_trap;
1322         }
1323         adj_index = ret;
1324         old_adj_index_valid = nh_grp->adj_index_valid;
1325         old_adj_index = nh_grp->adj_index;
1326         old_ecmp_size = nh_grp->ecmp_size;
1327         nh_grp->adj_index_valid = 1;
1328         nh_grp->adj_index = adj_index;
1329         nh_grp->ecmp_size = ecmp_size;
1330         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1331         if (err) {
1332                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1333                 goto set_trap;
1334         }
1335
1336         if (!old_adj_index_valid) {
1337                 /* The trap was set for fib entries, so we have to call
1338                  * fib entry update to unset it and use adjacency index.
1339                  */
1340                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1341                 if (err) {
1342                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1343                         goto set_trap;
1344                 }
1345                 return;
1346         }
1347
1348         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1349                                              old_adj_index, old_ecmp_size);
1350         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1351         if (err) {
1352                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1353                 goto set_trap;
1354         }
1355         return;
1356
1357 set_trap:
1358         old_adj_index_valid = nh_grp->adj_index_valid;
1359         nh_grp->adj_index_valid = 0;
1360         for (i = 0; i < nh_grp->count; i++) {
1361                 nh = &nh_grp->nexthops[i];
1362                 nh->offloaded = 0;
1363         }
1364         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1365         if (err)
1366                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1367         if (old_adj_index_valid)
1368                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1369 }
1370
1371 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1372                                             bool removing)
1373 {
1374         if (!removing && !nh->should_offload)
1375                 nh->should_offload = 1;
1376         else if (removing && nh->offloaded)
1377                 nh->should_offload = 0;
1378         nh->update = 1;
1379 }
1380
1381 static void
1382 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1383                               struct mlxsw_sp_neigh_entry *neigh_entry,
1384                               bool removing)
1385 {
1386         struct mlxsw_sp_nexthop *nh;
1387
1388         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1389                             neigh_list_node) {
1390                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1391                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1392         }
1393 }
1394
1395 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1396                                       struct mlxsw_sp_rif *rif)
1397 {
1398         if (nh->rif)
1399                 return;
1400
1401         nh->rif = rif;
1402         list_add(&nh->rif_list_node, &rif->nexthop_list);
1403 }
1404
1405 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1406 {
1407         if (!nh->rif)
1408                 return;
1409
1410         list_del(&nh->rif_list_node);
1411         nh->rif = NULL;
1412 }
1413
1414 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1415                                        struct mlxsw_sp_nexthop *nh)
1416 {
1417         struct mlxsw_sp_neigh_entry *neigh_entry;
1418         struct fib_nh *fib_nh = nh->key.fib_nh;
1419         struct neighbour *n;
1420         u8 nud_state, dead;
1421         int err;
1422
1423         if (!nh->nh_grp->gateway || nh->neigh_entry)
1424                 return 0;
1425
1426         /* Take a reference of neigh here ensuring that neigh would
1427          * not be detructed before the nexthop entry is finished.
1428          * The reference is taken either in neigh_lookup() or
1429          * in neigh_create() in case n is not found.
1430          */
1431         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1432         if (!n) {
1433                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1434                 if (IS_ERR(n))
1435                         return PTR_ERR(n);
1436                 neigh_event_send(n, NULL);
1437         }
1438         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1439         if (!neigh_entry) {
1440                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1441                 if (IS_ERR(neigh_entry)) {
1442                         err = -EINVAL;
1443                         goto err_neigh_entry_create;
1444                 }
1445         }
1446
1447         /* If that is the first nexthop connected to that neigh, add to
1448          * nexthop_neighs_list
1449          */
1450         if (list_empty(&neigh_entry->nexthop_list))
1451                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1452                               &mlxsw_sp->router.nexthop_neighs_list);
1453
1454         nh->neigh_entry = neigh_entry;
1455         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1456         read_lock_bh(&n->lock);
1457         nud_state = n->nud_state;
1458         dead = n->dead;
1459         read_unlock_bh(&n->lock);
1460         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1461
1462         return 0;
1463
1464 err_neigh_entry_create:
1465         neigh_release(n);
1466         return err;
1467 }
1468
1469 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1470                                         struct mlxsw_sp_nexthop *nh)
1471 {
1472         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473         struct neighbour *n;
1474
1475         if (!neigh_entry)
1476                 return;
1477         n = neigh_entry->key.n;
1478
1479         __mlxsw_sp_nexthop_neigh_update(nh, true);
1480         list_del(&nh->neigh_list_node);
1481         nh->neigh_entry = NULL;
1482
1483         /* If that is the last nexthop connected to that neigh, remove from
1484          * nexthop_neighs_list
1485          */
1486         if (list_empty(&neigh_entry->nexthop_list))
1487                 list_del(&neigh_entry->nexthop_neighs_list_node);
1488
1489         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1490                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1491
1492         neigh_release(n);
1493 }
1494
1495 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1496                                  struct mlxsw_sp_nexthop_group *nh_grp,
1497                                  struct mlxsw_sp_nexthop *nh,
1498                                  struct fib_nh *fib_nh)
1499 {
1500         struct net_device *dev = fib_nh->nh_dev;
1501         struct in_device *in_dev;
1502         struct mlxsw_sp_rif *rif;
1503         int err;
1504
1505         nh->nh_grp = nh_grp;
1506         nh->key.fib_nh = fib_nh;
1507         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1508         if (err)
1509                 return err;
1510
1511         if (!dev)
1512                 return 0;
1513
1514         in_dev = __in_dev_get_rtnl(dev);
1515         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1516             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1517                 return 0;
1518
1519         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1520         if (!rif)
1521                 return 0;
1522         mlxsw_sp_nexthop_rif_init(nh, rif);
1523
1524         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1525         if (err)
1526                 goto err_nexthop_neigh_init;
1527
1528         return 0;
1529
1530 err_nexthop_neigh_init:
1531         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1532         return err;
1533 }
1534
1535 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1536                                   struct mlxsw_sp_nexthop *nh)
1537 {
1538         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1539         mlxsw_sp_nexthop_rif_fini(nh);
1540         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1541 }
1542
1543 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1544                                    unsigned long event, struct fib_nh *fib_nh)
1545 {
1546         struct mlxsw_sp_nexthop_key key;
1547         struct mlxsw_sp_nexthop *nh;
1548         struct mlxsw_sp_rif *rif;
1549
1550         if (mlxsw_sp->router.aborted)
1551                 return;
1552
1553         key.fib_nh = fib_nh;
1554         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1555         if (WARN_ON_ONCE(!nh))
1556                 return;
1557
1558         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1559         if (!rif)
1560                 return;
1561
1562         switch (event) {
1563         case FIB_EVENT_NH_ADD:
1564                 mlxsw_sp_nexthop_rif_init(nh, rif);
1565                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1566                 break;
1567         case FIB_EVENT_NH_DEL:
1568                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1569                 mlxsw_sp_nexthop_rif_fini(nh);
1570                 break;
1571         }
1572
1573         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1574 }
1575
1576 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1577                                            struct mlxsw_sp_rif *rif)
1578 {
1579         struct mlxsw_sp_nexthop *nh, *tmp;
1580
1581         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1582                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1583                 mlxsw_sp_nexthop_rif_fini(nh);
1584                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1585         }
1586 }
1587
1588 static struct mlxsw_sp_nexthop_group *
1589 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1590 {
1591         struct mlxsw_sp_nexthop_group *nh_grp;
1592         struct mlxsw_sp_nexthop *nh;
1593         struct fib_nh *fib_nh;
1594         size_t alloc_size;
1595         int i;
1596         int err;
1597
1598         alloc_size = sizeof(*nh_grp) +
1599                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1600         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1601         if (!nh_grp)
1602                 return ERR_PTR(-ENOMEM);
1603         INIT_LIST_HEAD(&nh_grp->fib_list);
1604         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1605         nh_grp->count = fi->fib_nhs;
1606         nh_grp->key.fi = fi;
1607         for (i = 0; i < nh_grp->count; i++) {
1608                 nh = &nh_grp->nexthops[i];
1609                 fib_nh = &fi->fib_nh[i];
1610                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1611                 if (err)
1612                         goto err_nexthop_init;
1613         }
1614         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1615         if (err)
1616                 goto err_nexthop_group_insert;
1617         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1618         return nh_grp;
1619
1620 err_nexthop_group_insert:
1621 err_nexthop_init:
1622         for (i--; i >= 0; i--) {
1623                 nh = &nh_grp->nexthops[i];
1624                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1625         }
1626         kfree(nh_grp);
1627         return ERR_PTR(err);
1628 }
1629
1630 static void
1631 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1632                                struct mlxsw_sp_nexthop_group *nh_grp)
1633 {
1634         struct mlxsw_sp_nexthop *nh;
1635         int i;
1636
1637         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1638         for (i = 0; i < nh_grp->count; i++) {
1639                 nh = &nh_grp->nexthops[i];
1640                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1641         }
1642         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1643         WARN_ON_ONCE(nh_grp->adj_index_valid);
1644         kfree(nh_grp);
1645 }
1646
1647 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1648                                       struct mlxsw_sp_fib_entry *fib_entry,
1649                                       struct fib_info *fi)
1650 {
1651         struct mlxsw_sp_nexthop_group_key key;
1652         struct mlxsw_sp_nexthop_group *nh_grp;
1653
1654         key.fi = fi;
1655         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1656         if (!nh_grp) {
1657                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1658                 if (IS_ERR(nh_grp))
1659                         return PTR_ERR(nh_grp);
1660         }
1661         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1662         fib_entry->nh_group = nh_grp;
1663         return 0;
1664 }
1665
1666 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1667                                        struct mlxsw_sp_fib_entry *fib_entry)
1668 {
1669         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1670
1671         list_del(&fib_entry->nexthop_group_node);
1672         if (!list_empty(&nh_grp->fib_list))
1673                 return;
1674         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1675 }
1676
1677 static bool
1678 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1679 {
1680         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1681
1682         if (fib_entry->params.tos)
1683                 return false;
1684
1685         switch (fib_entry->type) {
1686         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1687                 return !!nh_group->adj_index_valid;
1688         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1689                 return !!nh_group->nh_rif;
1690         default:
1691                 return false;
1692         }
1693 }
1694
1695 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1696 {
1697         fib_entry->offloaded = true;
1698
1699         switch (fib_entry->fib_node->fib->proto) {
1700         case MLXSW_SP_L3_PROTO_IPV4:
1701                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1702                 break;
1703         case MLXSW_SP_L3_PROTO_IPV6:
1704                 WARN_ON_ONCE(1);
1705         }
1706 }
1707
1708 static void
1709 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1710 {
1711         switch (fib_entry->fib_node->fib->proto) {
1712         case MLXSW_SP_L3_PROTO_IPV4:
1713                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1714                 break;
1715         case MLXSW_SP_L3_PROTO_IPV6:
1716                 WARN_ON_ONCE(1);
1717         }
1718
1719         fib_entry->offloaded = false;
1720 }
1721
1722 static void
1723 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1724                                    enum mlxsw_reg_ralue_op op, int err)
1725 {
1726         switch (op) {
1727         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1728                 if (!fib_entry->offloaded)
1729                         return;
1730                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1731         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1732                 if (err)
1733                         return;
1734                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1735                     !fib_entry->offloaded)
1736                         mlxsw_sp_fib_entry_offload_set(fib_entry);
1737                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1738                          fib_entry->offloaded)
1739                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
1740                 return;
1741         default:
1742                 return;
1743         }
1744 }
1745
1746 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1747                                          struct mlxsw_sp_fib_entry *fib_entry,
1748                                          enum mlxsw_reg_ralue_op op)
1749 {
1750         char ralue_pl[MLXSW_REG_RALUE_LEN];
1751         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1752         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1753         enum mlxsw_reg_ralue_trap_action trap_action;
1754         u16 trap_id = 0;
1755         u32 adjacency_index = 0;
1756         u16 ecmp_size = 0;
1757
1758         /* In case the nexthop group adjacency index is valid, use it
1759          * with provided ECMP size. Otherwise, setup trap and pass
1760          * traffic to kernel.
1761          */
1762         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1763                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1764                 adjacency_index = fib_entry->nh_group->adj_index;
1765                 ecmp_size = fib_entry->nh_group->ecmp_size;
1766         } else {
1767                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1768                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1769         }
1770
1771         mlxsw_reg_ralue_pack4(ralue_pl,
1772                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1773                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1774                               *p_dip);
1775         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1776                                         adjacency_index, ecmp_size);
1777         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1778 }
1779
1780 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1781                                         struct mlxsw_sp_fib_entry *fib_entry,
1782                                         enum mlxsw_reg_ralue_op op)
1783 {
1784         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
1785         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1786         enum mlxsw_reg_ralue_trap_action trap_action;
1787         char ralue_pl[MLXSW_REG_RALUE_LEN];
1788         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1789         u16 trap_id = 0;
1790         u16 rif_index = 0;
1791
1792         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1793                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1794                 rif_index = rif->rif_index;
1795         } else {
1796                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1797                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1798         }
1799
1800         mlxsw_reg_ralue_pack4(ralue_pl,
1801                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1802                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1803                               *p_dip);
1804         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
1805                                        rif_index);
1806         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1807 }
1808
1809 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1810                                        struct mlxsw_sp_fib_entry *fib_entry,
1811                                        enum mlxsw_reg_ralue_op op)
1812 {
1813         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1814         char ralue_pl[MLXSW_REG_RALUE_LEN];
1815         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1816
1817         mlxsw_reg_ralue_pack4(ralue_pl,
1818                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1819                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1820                               *p_dip);
1821         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1822         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1823 }
1824
1825 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1826                                   struct mlxsw_sp_fib_entry *fib_entry,
1827                                   enum mlxsw_reg_ralue_op op)
1828 {
1829         switch (fib_entry->type) {
1830         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1831                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1832         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1833                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1834         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1835                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1836         }
1837         return -EINVAL;
1838 }
1839
1840 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1841                                  struct mlxsw_sp_fib_entry *fib_entry,
1842                                  enum mlxsw_reg_ralue_op op)
1843 {
1844         int err = -EINVAL;
1845
1846         switch (fib_entry->fib_node->fib->proto) {
1847         case MLXSW_SP_L3_PROTO_IPV4:
1848                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1849                 break;
1850         case MLXSW_SP_L3_PROTO_IPV6:
1851                 return err;
1852         }
1853         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1854         return err;
1855 }
1856
1857 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1858                                      struct mlxsw_sp_fib_entry *fib_entry)
1859 {
1860         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1861                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1862 }
1863
1864 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1865                                   struct mlxsw_sp_fib_entry *fib_entry)
1866 {
1867         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1868                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1869 }
1870
1871 static int
1872 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1873                              const struct fib_entry_notifier_info *fen_info,
1874                              struct mlxsw_sp_fib_entry *fib_entry)
1875 {
1876         struct fib_info *fi = fen_info->fi;
1877
1878         switch (fen_info->type) {
1879         case RTN_BROADCAST: /* fall through */
1880         case RTN_LOCAL:
1881                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1882                 return 0;
1883         case RTN_UNREACHABLE: /* fall through */
1884         case RTN_BLACKHOLE: /* fall through */
1885         case RTN_PROHIBIT:
1886                 /* Packets hitting these routes need to be trapped, but
1887                  * can do so with a lower priority than packets directed
1888                  * at the host, so use action type local instead of trap.
1889                  */
1890                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1891                 return 0;
1892         case RTN_UNICAST:
1893                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1894                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1895                 else
1896                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1897                 return 0;
1898         default:
1899                 return -EINVAL;
1900         }
1901 }
1902
1903 static struct mlxsw_sp_fib_entry *
1904 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1905                            struct mlxsw_sp_fib_node *fib_node,
1906                            const struct fib_entry_notifier_info *fen_info)
1907 {
1908         struct mlxsw_sp_fib_entry *fib_entry;
1909         int err;
1910
1911         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1912         if (!fib_entry) {
1913                 err = -ENOMEM;
1914                 goto err_fib_entry_alloc;
1915         }
1916
1917         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1918         if (err)
1919                 goto err_fib4_entry_type_set;
1920
1921         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1922         if (err)
1923                 goto err_nexthop_group_get;
1924
1925         fib_entry->params.prio = fen_info->fi->fib_priority;
1926         fib_entry->params.tb_id = fen_info->tb_id;
1927         fib_entry->params.type = fen_info->type;
1928         fib_entry->params.tos = fen_info->tos;
1929
1930         fib_entry->fib_node = fib_node;
1931
1932         return fib_entry;
1933
1934 err_nexthop_group_get:
1935 err_fib4_entry_type_set:
1936         kfree(fib_entry);
1937 err_fib_entry_alloc:
1938         return ERR_PTR(err);
1939 }
1940
1941 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1942                                         struct mlxsw_sp_fib_entry *fib_entry)
1943 {
1944         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1945         kfree(fib_entry);
1946 }
1947
1948 static struct mlxsw_sp_fib_node *
1949 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1950                        const struct fib_entry_notifier_info *fen_info);
1951
1952 static struct mlxsw_sp_fib_entry *
1953 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1954                            const struct fib_entry_notifier_info *fen_info)
1955 {
1956         struct mlxsw_sp_fib_entry *fib_entry;
1957         struct mlxsw_sp_fib_node *fib_node;
1958
1959         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1960         if (IS_ERR(fib_node))
1961                 return NULL;
1962
1963         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1964                 if (fib_entry->params.tb_id == fen_info->tb_id &&
1965                     fib_entry->params.tos == fen_info->tos &&
1966                     fib_entry->params.type == fen_info->type &&
1967                     fib_entry->nh_group->key.fi == fen_info->fi) {
1968                         return fib_entry;
1969                 }
1970         }
1971
1972         return NULL;
1973 }
1974
1975 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1976         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1977         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1978         .key_len = sizeof(struct mlxsw_sp_fib_key),
1979         .automatic_shrinking = true,
1980 };
1981
1982 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1983                                     struct mlxsw_sp_fib_node *fib_node)
1984 {
1985         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1986                                       mlxsw_sp_fib_ht_params);
1987 }
1988
1989 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1990                                      struct mlxsw_sp_fib_node *fib_node)
1991 {
1992         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1993                                mlxsw_sp_fib_ht_params);
1994 }
1995
1996 static struct mlxsw_sp_fib_node *
1997 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1998                          size_t addr_len, unsigned char prefix_len)
1999 {
2000         struct mlxsw_sp_fib_key key;
2001
2002         memset(&key, 0, sizeof(key));
2003         memcpy(key.addr, addr, addr_len);
2004         key.prefix_len = prefix_len;
2005         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2006 }
2007
2008 static struct mlxsw_sp_fib_node *
2009 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2010                          size_t addr_len, unsigned char prefix_len)
2011 {
2012         struct mlxsw_sp_fib_node *fib_node;
2013
2014         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2015         if (!fib_node)
2016                 return NULL;
2017
2018         INIT_LIST_HEAD(&fib_node->entry_list);
2019         list_add(&fib_node->list, &fib->node_list);
2020         memcpy(fib_node->key.addr, addr, addr_len);
2021         fib_node->key.prefix_len = prefix_len;
2022
2023         return fib_node;
2024 }
2025
2026 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2027 {
2028         list_del(&fib_node->list);
2029         WARN_ON(!list_empty(&fib_node->entry_list));
2030         kfree(fib_node);
2031 }
2032
2033 static bool
2034 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2035                                  const struct mlxsw_sp_fib_entry *fib_entry)
2036 {
2037         return list_first_entry(&fib_node->entry_list,
2038                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2039 }
2040
2041 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2042 {
2043         unsigned char prefix_len = fib_node->key.prefix_len;
2044         struct mlxsw_sp_fib *fib = fib_node->fib;
2045
2046         if (fib->prefix_ref_count[prefix_len]++ == 0)
2047                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2048 }
2049
2050 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2051 {
2052         unsigned char prefix_len = fib_node->key.prefix_len;
2053         struct mlxsw_sp_fib *fib = fib_node->fib;
2054
2055         if (--fib->prefix_ref_count[prefix_len] == 0)
2056                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2057 }
2058
2059 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2060                                   struct mlxsw_sp_fib_node *fib_node,
2061                                   struct mlxsw_sp_fib *fib)
2062 {
2063         struct mlxsw_sp_prefix_usage req_prefix_usage;
2064         struct mlxsw_sp_lpm_tree *lpm_tree;
2065         int err;
2066
2067         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2068         if (err)
2069                 return err;
2070         fib_node->fib = fib;
2071
2072         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2073         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2074
2075         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2076                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2077                                                  &req_prefix_usage);
2078                 if (err)
2079                         goto err_tree_check;
2080         } else {
2081                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2082                                                  fib->proto);
2083                 if (IS_ERR(lpm_tree))
2084                         return PTR_ERR(lpm_tree);
2085                 fib->lpm_tree = lpm_tree;
2086                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2087                 if (err)
2088                         goto err_tree_bind;
2089         }
2090
2091         mlxsw_sp_fib_node_prefix_inc(fib_node);
2092
2093         return 0;
2094
2095 err_tree_bind:
2096         fib->lpm_tree = NULL;
2097         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2098 err_tree_check:
2099         fib_node->fib = NULL;
2100         mlxsw_sp_fib_node_remove(fib, fib_node);
2101         return err;
2102 }
2103
2104 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2105                                    struct mlxsw_sp_fib_node *fib_node)
2106 {
2107         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2108         struct mlxsw_sp_fib *fib = fib_node->fib;
2109
2110         mlxsw_sp_fib_node_prefix_dec(fib_node);
2111
2112         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2113                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2114                 fib->lpm_tree = NULL;
2115                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2116         } else {
2117                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2118         }
2119
2120         fib_node->fib = NULL;
2121         mlxsw_sp_fib_node_remove(fib, fib_node);
2122 }
2123
2124 static struct mlxsw_sp_fib_node *
2125 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2126                        const struct fib_entry_notifier_info *fen_info)
2127 {
2128         struct mlxsw_sp_fib_node *fib_node;
2129         struct mlxsw_sp_fib *fib;
2130         struct mlxsw_sp_vr *vr;
2131         int err;
2132
2133         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2134         if (IS_ERR(vr))
2135                 return ERR_CAST(vr);
2136         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2137
2138         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2139                                             sizeof(fen_info->dst),
2140                                             fen_info->dst_len);
2141         if (fib_node)
2142                 return fib_node;
2143
2144         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2145                                             sizeof(fen_info->dst),
2146                                             fen_info->dst_len);
2147         if (!fib_node) {
2148                 err = -ENOMEM;
2149                 goto err_fib_node_create;
2150         }
2151
2152         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2153         if (err)
2154                 goto err_fib_node_init;
2155
2156         return fib_node;
2157
2158 err_fib_node_init:
2159         mlxsw_sp_fib_node_destroy(fib_node);
2160 err_fib_node_create:
2161         mlxsw_sp_vr_put(vr);
2162         return ERR_PTR(err);
2163 }
2164
2165 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2166                                    struct mlxsw_sp_fib_node *fib_node)
2167 {
2168         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2169
2170         if (!list_empty(&fib_node->entry_list))
2171                 return;
2172         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2173         mlxsw_sp_fib_node_destroy(fib_node);
2174         mlxsw_sp_vr_put(vr);
2175 }
2176
2177 static struct mlxsw_sp_fib_entry *
2178 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2179                               const struct mlxsw_sp_fib_entry_params *params)
2180 {
2181         struct mlxsw_sp_fib_entry *fib_entry;
2182
2183         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2184                 if (fib_entry->params.tb_id > params->tb_id)
2185                         continue;
2186                 if (fib_entry->params.tb_id != params->tb_id)
2187                         break;
2188                 if (fib_entry->params.tos > params->tos)
2189                         continue;
2190                 if (fib_entry->params.prio >= params->prio ||
2191                     fib_entry->params.tos < params->tos)
2192                         return fib_entry;
2193         }
2194
2195         return NULL;
2196 }
2197
2198 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2199                                           struct mlxsw_sp_fib_entry *new_entry)
2200 {
2201         struct mlxsw_sp_fib_node *fib_node;
2202
2203         if (WARN_ON(!fib_entry))
2204                 return -EINVAL;
2205
2206         fib_node = fib_entry->fib_node;
2207         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2208                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2209                     fib_entry->params.tos != new_entry->params.tos ||
2210                     fib_entry->params.prio != new_entry->params.prio)
2211                         break;
2212         }
2213
2214         list_add_tail(&new_entry->list, &fib_entry->list);
2215         return 0;
2216 }
2217
2218 static int
2219 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2220                                struct mlxsw_sp_fib_entry *new_entry,
2221                                bool replace, bool append)
2222 {
2223         struct mlxsw_sp_fib_entry *fib_entry;
2224
2225         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2226
2227         if (append)
2228                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2229         if (replace && WARN_ON(!fib_entry))
2230                 return -EINVAL;
2231
2232         /* Insert new entry before replaced one, so that we can later
2233          * remove the second.
2234          */
2235         if (fib_entry) {
2236                 list_add_tail(&new_entry->list, &fib_entry->list);
2237         } else {
2238                 struct mlxsw_sp_fib_entry *last;
2239
2240                 list_for_each_entry(last, &fib_node->entry_list, list) {
2241                         if (new_entry->params.tb_id > last->params.tb_id)
2242                                 break;
2243                         fib_entry = last;
2244                 }
2245
2246                 if (fib_entry)
2247                         list_add(&new_entry->list, &fib_entry->list);
2248                 else
2249                         list_add(&new_entry->list, &fib_node->entry_list);
2250         }
2251
2252         return 0;
2253 }
2254
2255 static void
2256 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2257 {
2258         list_del(&fib_entry->list);
2259 }
2260
2261 static int
2262 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2263                              const struct mlxsw_sp_fib_node *fib_node,
2264                              struct mlxsw_sp_fib_entry *fib_entry)
2265 {
2266         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2267                 return 0;
2268
2269         /* To prevent packet loss, overwrite the previously offloaded
2270          * entry.
2271          */
2272         if (!list_is_singular(&fib_node->entry_list)) {
2273                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2274                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2275
2276                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2277         }
2278
2279         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2280 }
2281
2282 static void
2283 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2284                              const struct mlxsw_sp_fib_node *fib_node,
2285                              struct mlxsw_sp_fib_entry *fib_entry)
2286 {
2287         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2288                 return;
2289
2290         /* Promote the next entry by overwriting the deleted entry */
2291         if (!list_is_singular(&fib_node->entry_list)) {
2292                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2293                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2294
2295                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2296                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2297                 return;
2298         }
2299
2300         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2301 }
2302
2303 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2304                                          struct mlxsw_sp_fib_entry *fib_entry,
2305                                          bool replace, bool append)
2306 {
2307         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2308         int err;
2309
2310         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2311                                              append);
2312         if (err)
2313                 return err;
2314
2315         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2316         if (err)
2317                 goto err_fib4_node_entry_add;
2318
2319         return 0;
2320
2321 err_fib4_node_entry_add:
2322         mlxsw_sp_fib4_node_list_remove(fib_entry);
2323         return err;
2324 }
2325
2326 static void
2327 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2328                                 struct mlxsw_sp_fib_entry *fib_entry)
2329 {
2330         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2331
2332         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2333         mlxsw_sp_fib4_node_list_remove(fib_entry);
2334 }
2335
2336 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2337                                         struct mlxsw_sp_fib_entry *fib_entry,
2338                                         bool replace)
2339 {
2340         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2341         struct mlxsw_sp_fib_entry *replaced;
2342
2343         if (!replace)
2344                 return;
2345
2346         /* We inserted the new entry before replaced one */
2347         replaced = list_next_entry(fib_entry, list);
2348
2349         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2350         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2351         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2352 }
2353
2354 static int
2355 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2356                          const struct fib_entry_notifier_info *fen_info,
2357                          bool replace, bool append)
2358 {
2359         struct mlxsw_sp_fib_entry *fib_entry;
2360         struct mlxsw_sp_fib_node *fib_node;
2361         int err;
2362
2363         if (mlxsw_sp->router.aborted)
2364                 return 0;
2365
2366         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2367         if (IS_ERR(fib_node)) {
2368                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2369                 return PTR_ERR(fib_node);
2370         }
2371
2372         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2373         if (IS_ERR(fib_entry)) {
2374                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2375                 err = PTR_ERR(fib_entry);
2376                 goto err_fib4_entry_create;
2377         }
2378
2379         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2380                                             append);
2381         if (err) {
2382                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2383                 goto err_fib4_node_entry_link;
2384         }
2385
2386         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2387
2388         return 0;
2389
2390 err_fib4_node_entry_link:
2391         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2392 err_fib4_entry_create:
2393         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2394         return err;
2395 }
2396
2397 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2398                                      struct fib_entry_notifier_info *fen_info)
2399 {
2400         struct mlxsw_sp_fib_entry *fib_entry;
2401         struct mlxsw_sp_fib_node *fib_node;
2402
2403         if (mlxsw_sp->router.aborted)
2404                 return;
2405
2406         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2407         if (WARN_ON(!fib_entry))
2408                 return;
2409         fib_node = fib_entry->fib_node;
2410
2411         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2412         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2413         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2414 }
2415
2416 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2417 {
2418         char ralta_pl[MLXSW_REG_RALTA_LEN];
2419         char ralst_pl[MLXSW_REG_RALST_LEN];
2420         int i, err;
2421
2422         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2423                              MLXSW_SP_LPM_TREE_MIN);
2424         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2425         if (err)
2426                 return err;
2427
2428         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2429         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2430         if (err)
2431                 return err;
2432
2433         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2434                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2435                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2436                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2437
2438                 if (!mlxsw_sp_vr_is_used(vr))
2439                         continue;
2440
2441                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2442                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2443                                      MLXSW_SP_LPM_TREE_MIN);
2444                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2445                                       raltb_pl);
2446                 if (err)
2447                         return err;
2448
2449                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2450                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2451                                       0);
2452                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2453                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2454                                       ralue_pl);
2455                 if (err)
2456                         return err;
2457         }
2458
2459         return 0;
2460 }
2461
2462 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2463                                      struct mlxsw_sp_fib_node *fib_node)
2464 {
2465         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2466
2467         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2468                 bool do_break = &tmp->list == &fib_node->entry_list;
2469
2470                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2471                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2472                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2473                 /* Break when entry list is empty and node was freed.
2474                  * Otherwise, we'll access freed memory in the next
2475                  * iteration.
2476                  */
2477                 if (do_break)
2478                         break;
2479         }
2480 }
2481
2482 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2483                                     struct mlxsw_sp_fib_node *fib_node)
2484 {
2485         switch (fib_node->fib->proto) {
2486         case MLXSW_SP_L3_PROTO_IPV4:
2487                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2488                 break;
2489         case MLXSW_SP_L3_PROTO_IPV6:
2490                 WARN_ON_ONCE(1);
2491                 break;
2492         }
2493 }
2494
2495 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2496                                   struct mlxsw_sp_vr *vr,
2497                                   enum mlxsw_sp_l3proto proto)
2498 {
2499         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2500         struct mlxsw_sp_fib_node *fib_node, *tmp;
2501
2502         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2503                 bool do_break = &tmp->list == &fib->node_list;
2504
2505                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2506                 if (do_break)
2507                         break;
2508         }
2509 }
2510
2511 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2512 {
2513         int i;
2514
2515         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2516                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2517
2518                 if (!mlxsw_sp_vr_is_used(vr))
2519                         continue;
2520                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2521         }
2522 }
2523
2524 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2525 {
2526         int err;
2527
2528         if (mlxsw_sp->router.aborted)
2529                 return;
2530         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2531         mlxsw_sp_router_fib_flush(mlxsw_sp);
2532         mlxsw_sp->router.aborted = true;
2533         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2534         if (err)
2535                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2536 }
2537
2538 struct mlxsw_sp_fib_event_work {
2539         struct work_struct work;
2540         union {
2541                 struct fib_entry_notifier_info fen_info;
2542                 struct fib_rule_notifier_info fr_info;
2543                 struct fib_nh_notifier_info fnh_info;
2544         };
2545         struct mlxsw_sp *mlxsw_sp;
2546         unsigned long event;
2547 };
2548
2549 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2550 {
2551         struct mlxsw_sp_fib_event_work *fib_work =
2552                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2553         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2554         struct fib_rule *rule;
2555         bool replace, append;
2556         int err;
2557
2558         /* Protect internal structures from changes */
2559         rtnl_lock();
2560         switch (fib_work->event) {
2561         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2562         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2563         case FIB_EVENT_ENTRY_ADD:
2564                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2565                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2566                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2567                                                replace, append);
2568                 if (err)
2569                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2570                 fib_info_put(fib_work->fen_info.fi);
2571                 break;
2572         case FIB_EVENT_ENTRY_DEL:
2573                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2574                 fib_info_put(fib_work->fen_info.fi);
2575                 break;
2576         case FIB_EVENT_RULE_ADD: /* fall through */
2577         case FIB_EVENT_RULE_DEL:
2578                 rule = fib_work->fr_info.rule;
2579                 if (!fib4_rule_default(rule) && !rule->l3mdev)
2580                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2581                 fib_rule_put(rule);
2582                 break;
2583         case FIB_EVENT_NH_ADD: /* fall through */
2584         case FIB_EVENT_NH_DEL:
2585                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2586                                        fib_work->fnh_info.fib_nh);
2587                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2588                 break;
2589         }
2590         rtnl_unlock();
2591         kfree(fib_work);
2592 }
2593
2594 /* Called with rcu_read_lock() */
2595 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2596                                      unsigned long event, void *ptr)
2597 {
2598         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2599         struct mlxsw_sp_fib_event_work *fib_work;
2600         struct fib_notifier_info *info = ptr;
2601
2602         if (!net_eq(info->net, &init_net))
2603                 return NOTIFY_DONE;
2604
2605         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2606         if (WARN_ON(!fib_work))
2607                 return NOTIFY_BAD;
2608
2609         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2610         fib_work->mlxsw_sp = mlxsw_sp;
2611         fib_work->event = event;
2612
2613         switch (event) {
2614         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2615         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2616         case FIB_EVENT_ENTRY_ADD: /* fall through */
2617         case FIB_EVENT_ENTRY_DEL:
2618                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2619                 /* Take referece on fib_info to prevent it from being
2620                  * freed while work is queued. Release it afterwards.
2621                  */
2622                 fib_info_hold(fib_work->fen_info.fi);
2623                 break;
2624         case FIB_EVENT_RULE_ADD: /* fall through */
2625         case FIB_EVENT_RULE_DEL:
2626                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2627                 fib_rule_get(fib_work->fr_info.rule);
2628                 break;
2629         case FIB_EVENT_NH_ADD: /* fall through */
2630         case FIB_EVENT_NH_DEL:
2631                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2632                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2633                 break;
2634         }
2635
2636         mlxsw_core_schedule_work(&fib_work->work);
2637
2638         return NOTIFY_DONE;
2639 }
2640
2641 static struct mlxsw_sp_rif *
2642 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2643                          const struct net_device *dev)
2644 {
2645         int i;
2646
2647         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2648                 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2649                         return mlxsw_sp->rifs[i];
2650
2651         return NULL;
2652 }
2653
2654 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2655 {
2656         char ritr_pl[MLXSW_REG_RITR_LEN];
2657         int err;
2658
2659         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2660         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2661         if (WARN_ON_ONCE(err))
2662                 return err;
2663
2664         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2665         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2666 }
2667
2668 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2669                                           struct mlxsw_sp_rif *rif)
2670 {
2671         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2672         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2673         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2674 }
2675
2676 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2677                                        const struct in_device *in_dev,
2678                                        unsigned long event)
2679 {
2680         switch (event) {
2681         case NETDEV_UP:
2682                 if (!rif)
2683                         return true;
2684                 return false;
2685         case NETDEV_DOWN:
2686                 if (rif && !in_dev->ifa_list &&
2687                     !netif_is_l3_slave(rif->dev))
2688                         return true;
2689                 /* It is possible we already removed the RIF ourselves
2690                  * if it was assigned to a netdev that is now a bridge
2691                  * or LAG slave.
2692                  */
2693                 return false;
2694         }
2695
2696         return false;
2697 }
2698
2699 #define MLXSW_SP_INVALID_INDEX_RIF 0xffff
2700 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2701 {
2702         int i;
2703
2704         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2705                 if (!mlxsw_sp->rifs[i])
2706                         return i;
2707
2708         return MLXSW_SP_INVALID_INDEX_RIF;
2709 }
2710
2711 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2712                                            bool *p_lagged, u16 *p_system_port)
2713 {
2714         u8 local_port = mlxsw_sp_vport->local_port;
2715
2716         *p_lagged = mlxsw_sp_vport->lagged;
2717         *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2718 }
2719
2720 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2721                                     u16 vr_id, struct net_device *l3_dev,
2722                                     u16 rif_index, bool create)
2723 {
2724         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2725         bool lagged = mlxsw_sp_vport->lagged;
2726         char ritr_pl[MLXSW_REG_RITR_LEN];
2727         u16 system_port;
2728
2729         mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif_index,
2730                             vr_id, l3_dev->mtu, l3_dev->dev_addr);
2731
2732         mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2733         mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2734                                   mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2735
2736         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2737 }
2738
2739 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2740
2741 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif_index)
2742 {
2743         return MLXSW_SP_RFID_BASE + rif_index;
2744 }
2745
2746 static struct mlxsw_sp_fid *
2747 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2748 {
2749         struct mlxsw_sp_fid *f;
2750
2751         f = kzalloc(sizeof(*f), GFP_KERNEL);
2752         if (!f)
2753                 return NULL;
2754
2755         f->leave = mlxsw_sp_vport_rif_sp_leave;
2756         f->ref_count = 0;
2757         f->dev = l3_dev;
2758         f->fid = fid;
2759
2760         return f;
2761 }
2762
2763 static struct mlxsw_sp_rif *
2764 mlxsw_sp_rif_alloc(u16 rif_index, u16 vr_id, struct net_device *l3_dev,
2765                    struct mlxsw_sp_fid *f)
2766 {
2767         struct mlxsw_sp_rif *rif;
2768
2769         rif = kzalloc(sizeof(*rif), GFP_KERNEL);
2770         if (!rif)
2771                 return NULL;
2772
2773         INIT_LIST_HEAD(&rif->nexthop_list);
2774         INIT_LIST_HEAD(&rif->neigh_list);
2775         ether_addr_copy(rif->addr, l3_dev->dev_addr);
2776         rif->mtu = l3_dev->mtu;
2777         rif->vr_id = vr_id;
2778         rif->dev = l3_dev;
2779         rif->rif_index = rif_index;
2780         rif->f = f;
2781
2782         return rif;
2783 }
2784
2785 static struct mlxsw_sp_rif *
2786 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2787                              struct net_device *l3_dev)
2788 {
2789         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2790         u32 tb_id = l3mdev_fib_table(l3_dev);
2791         struct mlxsw_sp_vr *vr;
2792         struct mlxsw_sp_fid *f;
2793         struct mlxsw_sp_rif *rif;
2794         u16 fid, rif_index;
2795         int err;
2796
2797         rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
2798         if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
2799                 return ERR_PTR(-ERANGE);
2800
2801         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
2802         if (IS_ERR(vr))
2803                 return ERR_CAST(vr);
2804
2805         err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev,
2806                                        rif_index, true);
2807         if (err)
2808                 goto err_vport_rif_sp_op;
2809
2810         fid = mlxsw_sp_rif_sp_to_fid(rif_index);
2811         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2812         if (err)
2813                 goto err_rif_fdb_op;
2814
2815         f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2816         if (!f) {
2817                 err = -ENOMEM;
2818                 goto err_rfid_alloc;
2819         }
2820
2821         rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
2822         if (!rif) {
2823                 err = -ENOMEM;
2824                 goto err_rif_alloc;
2825         }
2826
2827         f->rif = rif;
2828         mlxsw_sp->rifs[rif_index] = rif;
2829         vr->rif_count++;
2830
2831         return rif;
2832
2833 err_rif_alloc:
2834         kfree(f);
2835 err_rfid_alloc:
2836         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2837 err_rif_fdb_op:
2838         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2839                                  false);
2840 err_vport_rif_sp_op:
2841         mlxsw_sp_vr_put(vr);
2842         return ERR_PTR(err);
2843 }
2844
2845 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2846                                           struct mlxsw_sp_rif *rif)
2847 {
2848         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2849         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
2850         struct net_device *l3_dev = rif->dev;
2851         struct mlxsw_sp_fid *f = rif->f;
2852         u16 rif_index = rif->rif_index;
2853         u16 fid = f->fid;
2854
2855         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
2856
2857         vr->rif_count--;
2858         mlxsw_sp->rifs[rif_index] = NULL;
2859         f->rif = NULL;
2860
2861         kfree(rif);
2862
2863         kfree(f);
2864
2865         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2866
2867         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2868                                  false);
2869         mlxsw_sp_vr_put(vr);
2870 }
2871
2872 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2873                                       struct net_device *l3_dev)
2874 {
2875         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2876         struct mlxsw_sp_rif *rif;
2877
2878         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2879         if (!rif) {
2880                 rif = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2881                 if (IS_ERR(rif))
2882                         return PTR_ERR(rif);
2883         }
2884
2885         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, rif->f);
2886         rif->f->ref_count++;
2887
2888         netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", rif->f->fid);
2889
2890         return 0;
2891 }
2892
2893 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2894 {
2895         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2896
2897         netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2898
2899         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2900         if (--f->ref_count == 0)
2901                 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->rif);
2902 }
2903
2904 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2905                                          struct net_device *port_dev,
2906                                          unsigned long event, u16 vid)
2907 {
2908         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2909         struct mlxsw_sp_port *mlxsw_sp_vport;
2910
2911         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2912         if (WARN_ON(!mlxsw_sp_vport))
2913                 return -EINVAL;
2914
2915         switch (event) {
2916         case NETDEV_UP:
2917                 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2918         case NETDEV_DOWN:
2919                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2920                 break;
2921         }
2922
2923         return 0;
2924 }
2925
2926 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2927                                         unsigned long event)
2928 {
2929         if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2930                 return 0;
2931
2932         return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2933 }
2934
2935 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2936                                          struct net_device *lag_dev,
2937                                          unsigned long event, u16 vid)
2938 {
2939         struct net_device *port_dev;
2940         struct list_head *iter;
2941         int err;
2942
2943         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2944                 if (mlxsw_sp_port_dev_check(port_dev)) {
2945                         err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2946                                                             event, vid);
2947                         if (err)
2948                                 return err;
2949                 }
2950         }
2951
2952         return 0;
2953 }
2954
2955 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2956                                        unsigned long event)
2957 {
2958         if (netif_is_bridge_port(lag_dev))
2959                 return 0;
2960
2961         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2962 }
2963
2964 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2965                                                     struct net_device *l3_dev)
2966 {
2967         u16 fid;
2968
2969         if (is_vlan_dev(l3_dev))
2970                 fid = vlan_dev_vlan_id(l3_dev);
2971         else if (mlxsw_sp->master_bridge.dev == l3_dev)
2972                 fid = 1;
2973         else
2974                 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2975
2976         return mlxsw_sp_fid_find(mlxsw_sp, fid);
2977 }
2978
2979 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
2980 {
2981         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
2982 }
2983
2984 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2985 {
2986         return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2987                MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2988 }
2989
2990 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2991 {
2992         return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2993 }
2994
2995 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2996                                           bool set)
2997 {
2998         u8 router_port = mlxsw_sp_router_port(mlxsw_sp);
2999         enum mlxsw_flood_table_type table_type;
3000         char *sftr_pl;
3001         u16 index;
3002         int err;
3003
3004         sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
3005         if (!sftr_pl)
3006                 return -ENOMEM;
3007
3008         table_type = mlxsw_sp_flood_table_type_get(fid);
3009         index = mlxsw_sp_flood_table_index_get(fid);
3010         mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
3011                             1, router_port, set);
3012         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
3013
3014         kfree(sftr_pl);
3015         return err;
3016 }
3017
3018 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
3019 {
3020         if (mlxsw_sp_fid_is_vfid(fid))
3021                 return MLXSW_REG_RITR_FID_IF;
3022         else
3023                 return MLXSW_REG_RITR_VLAN_IF;
3024 }
3025
3026 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
3027                                   struct net_device *l3_dev,
3028                                   u16 fid, u16 rif,
3029                                   bool create)
3030 {
3031         enum mlxsw_reg_ritr_if_type rif_type;
3032         char ritr_pl[MLXSW_REG_RITR_LEN];
3033
3034         rif_type = mlxsw_sp_rif_type_get(fid);
3035         mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3036                             l3_dev->dev_addr);
3037         mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3038
3039         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3040 }
3041
3042 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3043                                       struct net_device *l3_dev,
3044                                       struct mlxsw_sp_fid *f)
3045 {
3046         u32 tb_id = l3mdev_fib_table(l3_dev);
3047         struct mlxsw_sp_rif *rif;
3048         struct mlxsw_sp_vr *vr;
3049         u16 rif_index;
3050         int err;
3051
3052         rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
3053         if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
3054                 return -ERANGE;
3055
3056         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3057         if (IS_ERR(vr))
3058                 return PTR_ERR(vr);
3059
3060         err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3061         if (err)
3062                 goto err_port_flood_set;
3063
3064         err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid,
3065                                      rif_index, true);
3066         if (err)
3067                 goto err_rif_bridge_op;
3068
3069         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3070         if (err)
3071                 goto err_rif_fdb_op;
3072
3073         rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
3074         if (!rif) {
3075                 err = -ENOMEM;
3076                 goto err_rif_alloc;
3077         }
3078
3079         f->rif = rif;
3080         mlxsw_sp->rifs[rif_index] = rif;
3081         vr->rif_count++;
3082
3083         netdev_dbg(l3_dev, "RIF=%d created\n", rif_index);
3084
3085         return 0;
3086
3087 err_rif_alloc:
3088         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3089 err_rif_fdb_op:
3090         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3091                                false);
3092 err_rif_bridge_op:
3093         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3094 err_port_flood_set:
3095         mlxsw_sp_vr_put(vr);
3096         return err;
3097 }
3098
3099 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3100                                  struct mlxsw_sp_rif *rif)
3101 {
3102         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
3103         struct net_device *l3_dev = rif->dev;
3104         struct mlxsw_sp_fid *f = rif->f;
3105         u16 rif_index = rif->rif_index;
3106
3107         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3108
3109         vr->rif_count--;
3110         mlxsw_sp->rifs[rif_index] = NULL;
3111         f->rif = NULL;
3112
3113         kfree(rif);
3114
3115         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3116
3117         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3118                                false);
3119
3120         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3121
3122         mlxsw_sp_vr_put(vr);
3123
3124         netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif_index);
3125 }
3126
3127 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3128                                           struct net_device *br_dev,
3129                                           unsigned long event)
3130 {
3131         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3132         struct mlxsw_sp_fid *f;
3133
3134         /* FID can either be an actual FID if the L3 device is the
3135          * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3136          * L3 device is a VLAN-unaware bridge and we get a vFID.
3137          */
3138         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3139         if (WARN_ON(!f))
3140                 return -EINVAL;
3141
3142         switch (event) {
3143         case NETDEV_UP:
3144                 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3145         case NETDEV_DOWN:
3146                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3147                 break;
3148         }
3149
3150         return 0;
3151 }
3152
3153 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3154                                         unsigned long event)
3155 {
3156         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3157         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3158         u16 vid = vlan_dev_vlan_id(vlan_dev);
3159
3160         if (mlxsw_sp_port_dev_check(real_dev))
3161                 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3162                                                      vid);
3163         else if (netif_is_lag_master(real_dev))
3164                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3165                                                      vid);
3166         else if (netif_is_bridge_master(real_dev) &&
3167                  mlxsw_sp->master_bridge.dev == real_dev)
3168                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3169                                                       event);
3170
3171         return 0;
3172 }
3173
3174 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3175                             unsigned long event, void *ptr)
3176 {
3177         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3178         struct net_device *dev = ifa->ifa_dev->dev;
3179         struct mlxsw_sp *mlxsw_sp;
3180         struct mlxsw_sp_rif *rif;
3181         int err = 0;
3182
3183         mlxsw_sp = mlxsw_sp_lower_get(dev);
3184         if (!mlxsw_sp)
3185                 goto out;
3186
3187         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3188         if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3189                 goto out;
3190
3191         if (mlxsw_sp_port_dev_check(dev))
3192                 err = mlxsw_sp_inetaddr_port_event(dev, event);
3193         else if (netif_is_lag_master(dev))
3194                 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3195         else if (netif_is_bridge_master(dev))
3196                 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3197         else if (is_vlan_dev(dev))
3198                 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3199
3200 out:
3201         return notifier_from_errno(err);
3202 }
3203
3204 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3205                              const char *mac, int mtu)
3206 {
3207         char ritr_pl[MLXSW_REG_RITR_LEN];
3208         int err;
3209
3210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3212         if (err)
3213                 return err;
3214
3215         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3216         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3217         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3218         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3219 }
3220
3221 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3222 {
3223         struct mlxsw_sp *mlxsw_sp;
3224         struct mlxsw_sp_rif *rif;
3225         int err;
3226
3227         mlxsw_sp = mlxsw_sp_lower_get(dev);
3228         if (!mlxsw_sp)
3229                 return 0;
3230
3231         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3232         if (!rif)
3233                 return 0;
3234
3235         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, false);
3236         if (err)
3237                 return err;
3238
3239         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3240                                 dev->mtu);
3241         if (err)
3242                 goto err_rif_edit;
3243
3244         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, rif->f->fid, true);
3245         if (err)
3246                 goto err_rif_fdb_op;
3247
3248         ether_addr_copy(rif->addr, dev->dev_addr);
3249         rif->mtu = dev->mtu;
3250
3251         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3252
3253         return 0;
3254
3255 err_rif_fdb_op:
3256         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3257 err_rif_edit:
3258         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, true);
3259         return err;
3260 }
3261
3262 int mlxsw_sp_vport_vrf_join(struct mlxsw_sp_port *mlxsw_sp_vport)
3263 {
3264         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
3265         struct net_device *dev = mlxsw_sp_vport->dev;
3266
3267         /* In case vPort already has a RIF, then we need to drop it.
3268          * A new one will be created using the VRF's VR.
3269          */
3270         if (f && f->rif)
3271                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3272
3273         return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, dev);
3274 }
3275
3276 void mlxsw_sp_vport_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
3277 {
3278         mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3279 }
3280
3281 int mlxsw_sp_port_vrf_join(struct mlxsw_sp_port *mlxsw_sp_port)
3282 {
3283         struct mlxsw_sp_port *mlxsw_sp_vport;
3284
3285         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3286         if (WARN_ON(!mlxsw_sp_vport))
3287                 return -EINVAL;
3288
3289         return mlxsw_sp_vport_vrf_join(mlxsw_sp_vport);
3290 }
3291
3292 void mlxsw_sp_port_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_port)
3293 {
3294         struct mlxsw_sp_port *mlxsw_sp_vport;
3295
3296         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3297         if (WARN_ON(!mlxsw_sp_vport))
3298                 return;
3299
3300         mlxsw_sp_vport_vrf_leave(mlxsw_sp_vport);
3301 }
3302
3303 int mlxsw_sp_bridge_vrf_join(struct mlxsw_sp *mlxsw_sp,
3304                              struct net_device *l3_dev)
3305 {
3306         struct mlxsw_sp_fid *f;
3307
3308         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3309         if (WARN_ON(!f))
3310                 return -EINVAL;
3311
3312         if (f->rif)
3313                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3314
3315         return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3316 }
3317
3318 void mlxsw_sp_bridge_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3319                                struct net_device *l3_dev)
3320 {
3321         struct mlxsw_sp_fid *f;
3322
3323         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3324         if (WARN_ON(!f))
3325                 return;
3326         mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3327 }
3328
3329 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3330 {
3331         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3332
3333         /* Flush pending FIB notifications and then flush the device's
3334          * table before requesting another dump. The FIB notification
3335          * block is unregistered, so no need to take RTNL.
3336          */
3337         mlxsw_core_flush_owq();
3338         mlxsw_sp_router_fib_flush(mlxsw_sp);
3339 }
3340
3341 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3342 {
3343         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3344         u64 max_rifs;
3345         int err;
3346
3347         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3348                 return -EIO;
3349
3350         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3351         mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3352                                  GFP_KERNEL);
3353         if (!mlxsw_sp->rifs)
3354                 return -ENOMEM;
3355
3356         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3357         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3358         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3359         if (err)
3360                 goto err_rgcr_fail;
3361
3362         return 0;
3363
3364 err_rgcr_fail:
3365         kfree(mlxsw_sp->rifs);
3366         return err;
3367 }
3368
3369 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3370 {
3371         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3372         int i;
3373
3374         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3375         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3376
3377         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3378                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3379
3380         kfree(mlxsw_sp->rifs);
3381 }
3382
3383 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3384 {
3385         int err;
3386
3387         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3388         err = __mlxsw_sp_router_init(mlxsw_sp);
3389         if (err)
3390                 return err;
3391
3392         err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3393                               &mlxsw_sp_nexthop_ht_params);
3394         if (err)
3395                 goto err_nexthop_ht_init;
3396
3397         err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3398                               &mlxsw_sp_nexthop_group_ht_params);
3399         if (err)
3400                 goto err_nexthop_group_ht_init;
3401
3402         err = mlxsw_sp_lpm_init(mlxsw_sp);
3403         if (err)
3404                 goto err_lpm_init;
3405
3406         err = mlxsw_sp_vrs_init(mlxsw_sp);
3407         if (err)
3408                 goto err_vrs_init;
3409
3410         err = mlxsw_sp_neigh_init(mlxsw_sp);
3411         if (err)
3412                 goto err_neigh_init;
3413
3414         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3415         err = register_fib_notifier(&mlxsw_sp->fib_nb,
3416                                     mlxsw_sp_router_fib_dump_flush);
3417         if (err)
3418                 goto err_register_fib_notifier;
3419
3420         return 0;
3421
3422 err_register_fib_notifier:
3423         mlxsw_sp_neigh_fini(mlxsw_sp);
3424 err_neigh_init:
3425         mlxsw_sp_vrs_fini(mlxsw_sp);
3426 err_vrs_init:
3427         mlxsw_sp_lpm_fini(mlxsw_sp);
3428 err_lpm_init:
3429         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3430 err_nexthop_group_ht_init:
3431         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3432 err_nexthop_ht_init:
3433         __mlxsw_sp_router_fini(mlxsw_sp);
3434         return err;
3435 }
3436
3437 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3438 {
3439         unregister_fib_notifier(&mlxsw_sp->fib_nb);
3440         mlxsw_sp_neigh_fini(mlxsw_sp);
3441         mlxsw_sp_vrs_fini(mlxsw_sp);
3442         mlxsw_sp_lpm_fini(mlxsw_sp);
3443         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3444         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3445         __mlxsw_sp_router_fini(mlxsw_sp);
3446 }