drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <linux/inetdevice.h>
  44 #include <linux/netdevice.h>
  45 #include <net/netevent.h>
  46 #include <net/neighbour.h>
  47 #include <net/arp.h>
  48 #include <net/ip_fib.h>
  49 #include <net/fib_rules.h>
  50 #include <net/l3mdev.h>
  51
  52 #include "spectrum.h"
  53 #include "core.h"
  54 #include "reg.h"
  55
  56 struct mlxsw_sp_rif {
  57         struct list_head nexthop_list;
  58         struct list_head neigh_list;
  59         struct net_device *dev;
  60         struct mlxsw_sp_fid *f;
  61         unsigned char addr[ETH_ALEN];
  62         int mtu;
  63         u16 rif_index;
  64         u16 vr_id;
  65 };
  66
  67 static struct mlxsw_sp_rif *
  68 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
  69                          const struct net_device *dev);
  70
  71 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  72         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  73
  74 static bool
  75 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  76                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  77 {
  78         unsigned char prefix;
  79
  80         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  81                 if (!test_bit(prefix, prefix_usage2->b))
  82                         return false;
  83         }
  84         return true;
  85 }
  86
  87 static bool
  88 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  89                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  90 {
  91         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  92 }
  93
  94 static bool
  95 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  96 {
  97         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  98
  99         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 100 }
 101
 102 static void
 103 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 104                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 105 {
 106         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 107 }
 108
 109 static void
 110 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 111                           unsigned char prefix_len)
 112 {
 113         set_bit(prefix_len, prefix_usage->b);
 114 }
 115
 116 static void
 117 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 118                             unsigned char prefix_len)
 119 {
 120         clear_bit(prefix_len, prefix_usage->b);
 121 }
 122
 123 struct mlxsw_sp_fib_key {
 124         unsigned char addr[sizeof(struct in6_addr)];
 125         unsigned char prefix_len;
 126 };
 127
 128 enum mlxsw_sp_fib_entry_type {
 129         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 130         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 131         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 132 };
 133
 134 struct mlxsw_sp_nexthop_group;
 135
 136 struct mlxsw_sp_fib_node {
 137         struct list_head entry_list;
 138         struct list_head list;
 139         struct rhash_head ht_node;
 140         struct mlxsw_sp_fib *fib;
 141         struct mlxsw_sp_fib_key key;
 142 };
 143
 144 struct mlxsw_sp_fib_entry_params {
 145         u32 tb_id;
 146         u32 prio;
 147         u8 tos;
 148         u8 type;
 149 };
 150
 151 struct mlxsw_sp_fib_entry {
 152         struct list_head list;
 153         struct mlxsw_sp_fib_node *fib_node;
 154         enum mlxsw_sp_fib_entry_type type;
 155         struct list_head nexthop_group_node;
 156         struct mlxsw_sp_nexthop_group *nh_group;
 157         struct mlxsw_sp_fib_entry_params params;
 158         bool offloaded;
 159 };
 160
 161 struct mlxsw_sp_fib {
 162         struct rhashtable ht;
 163         struct list_head node_list;
 164         struct mlxsw_sp_vr *vr;
 165         struct mlxsw_sp_lpm_tree *lpm_tree;
 166         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 167         struct mlxsw_sp_prefix_usage prefix_usage;
 168         enum mlxsw_sp_l3proto proto;
 169 };
 170
 171 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 172
 173 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 174                                                 enum mlxsw_sp_l3proto proto)
 175 {
 176         struct mlxsw_sp_fib *fib;
 177         int err;
 178
 179         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 180         if (!fib)
 181                 return ERR_PTR(-ENOMEM);
 182         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 183         if (err)
 184                 goto err_rhashtable_init;
 185         INIT_LIST_HEAD(&fib->node_list);
 186         fib->proto = proto;
 187         fib->vr = vr;
 188         return fib;
 189
 190 err_rhashtable_init:
 191         kfree(fib);
 192         return ERR_PTR(err);
 193 }
 194
 195 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 196 {
 197         WARN_ON(!list_empty(&fib->node_list));
 198         WARN_ON(fib->lpm_tree);
 199         rhashtable_destroy(&fib->ht);
 200         kfree(fib);
 201 }
 202
 203 static struct mlxsw_sp_lpm_tree *
 204 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 205 {
 206         static struct mlxsw_sp_lpm_tree *lpm_tree;
 207         int i;
 208
 209         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 210                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 211                 if (lpm_tree->ref_count == 0)
 212                         return lpm_tree;
 213         }
 214         return NULL;
 215 }
 216
 217 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 218                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 219 {
 220         char ralta_pl[MLXSW_REG_RALTA_LEN];
 221
 222         mlxsw_reg_ralta_pack(ralta_pl, true,
 223                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 224                              lpm_tree->id);
 225         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 226 }
 227
 228 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 229                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 230 {
 231         char ralta_pl[MLXSW_REG_RALTA_LEN];
 232
 233         mlxsw_reg_ralta_pack(ralta_pl, false,
 234                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 235                              lpm_tree->id);
 236         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 237 }
 238
 239 static int
 240 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 241                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 242                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 243 {
 244         char ralst_pl[MLXSW_REG_RALST_LEN];
 245         u8 root_bin = 0;
 246         u8 prefix;
 247         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 248
 249         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 250                 root_bin = prefix;
 251
 252         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 253         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 254                 if (prefix == 0)
 255                         continue;
 256                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 257                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 258                 last_prefix = prefix;
 259         }
 260         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 261 }
 262
 263 static struct mlxsw_sp_lpm_tree *
 264 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 265                          struct mlxsw_sp_prefix_usage *prefix_usage,
 266                          enum mlxsw_sp_l3proto proto)
 267 {
 268         struct mlxsw_sp_lpm_tree *lpm_tree;
 269         int err;
 270
 271         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 272         if (!lpm_tree)
 273                 return ERR_PTR(-EBUSY);
 274         lpm_tree->proto = proto;
 275         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 276         if (err)
 277                 return ERR_PTR(err);
 278
 279         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 280                                                 lpm_tree);
 281         if (err)
 282                 goto err_left_struct_set;
 283         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 284                sizeof(lpm_tree->prefix_usage));
 285         return lpm_tree;
 286
 287 err_left_struct_set:
 288         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 289         return ERR_PTR(err);
 290 }
 291
 292 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 293                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 294 {
 295         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 296 }
 297
 298 static struct mlxsw_sp_lpm_tree *
 299 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 300                       struct mlxsw_sp_prefix_usage *prefix_usage,
 301                       enum mlxsw_sp_l3proto proto)
 302 {
 303         struct mlxsw_sp_lpm_tree *lpm_tree;
 304         int i;
 305
 306         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 307                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 308                 if (lpm_tree->ref_count != 0 &&
 309                     lpm_tree->proto == proto &&
 310                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 311                                              prefix_usage))
 312                         goto inc_ref_count;
 313         }
 314         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 315                                             proto);
 316         if (IS_ERR(lpm_tree))
 317                 return lpm_tree;
 318
 319 inc_ref_count:
 320         lpm_tree->ref_count++;
 321         return lpm_tree;
 322 }
 323
 324 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 325                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 326 {
 327         if (--lpm_tree->ref_count == 0)
 328                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 329         return 0;
 330 }
 331
 332 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 333 {
 334         struct mlxsw_sp_lpm_tree *lpm_tree;
 335         int i;
 336
 337         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 338                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 339                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 340         }
 341 }
 342
 343 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 344 {
 345         return !!vr->fib4;
 346 }
 347
 348 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 349 {
 350         struct mlxsw_sp_vr *vr;
 351         int i;
 352
 353         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 354                 vr = &mlxsw_sp->router.vrs[i];
 355                 if (!mlxsw_sp_vr_is_used(vr))
 356                         return vr;
 357         }
 358         return NULL;
 359 }
 360
 361 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 362                                      const struct mlxsw_sp_fib *fib)
 363 {
 364         char raltb_pl[MLXSW_REG_RALTB_LEN];
 365
 366         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 367                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 368                              fib->lpm_tree->id);
 369         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 370 }
 371
 372 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 373                                        const struct mlxsw_sp_fib *fib)
 374 {
 375         char raltb_pl[MLXSW_REG_RALTB_LEN];
 376
 377         /* Bind to tree 0 which is default */
 378         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 379                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 380         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 381 }
 382
 383 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 384 {
 385         /* For our purpose, squash main and local table into one */
 386         if (tb_id == RT_TABLE_LOCAL)
 387                 tb_id = RT_TABLE_MAIN;
 388         return tb_id;
 389 }
 390
 391 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 392                                             u32 tb_id)
 393 {
 394         struct mlxsw_sp_vr *vr;
 395         int i;
 396
 397         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 398
 399         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 400                 vr = &mlxsw_sp->router.vrs[i];
 401                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 402                         return vr;
 403         }
 404         return NULL;
 405 }
 406
 407 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 408                                             enum mlxsw_sp_l3proto proto)
 409 {
 410         switch (proto) {
 411         case MLXSW_SP_L3_PROTO_IPV4:
 412                 return vr->fib4;
 413         case MLXSW_SP_L3_PROTO_IPV6:
 414                 BUG_ON(1);
 415         }
 416         return NULL;
 417 }
 418
 419 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 420                                               u32 tb_id)
 421 {
 422         struct mlxsw_sp_vr *vr;
 423
 424         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 425         if (!vr)
 426                 return ERR_PTR(-EBUSY);
 427         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 428         if (IS_ERR(vr->fib4))
 429                 return ERR_CAST(vr->fib4);
 430         vr->tb_id = tb_id;
 431         return vr;
 432 }
 433
 434 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 435 {
 436         mlxsw_sp_fib_destroy(vr->fib4);
 437         vr->fib4 = NULL;
 438 }
 439
 440 static int
 441 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
 442                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 443 {
 444         struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
 445         struct mlxsw_sp_lpm_tree *new_tree;
 446         int err;
 447
 448         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 449                 return 0;
 450
 451         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 452                                          fib->proto);
 453         if (IS_ERR(new_tree)) {
 454                 /* We failed to get a tree according to the required
 455                  * prefix usage. However, the current tree might be still good
 456                  * for us if our requirement is subset of the prefixes used
 457                  * in the tree.
 458                  */
 459                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 460                                                  &lpm_tree->prefix_usage))
 461                         return 0;
 462                 return PTR_ERR(new_tree);
 463         }
 464
 465         /* Prevent packet loss by overwriting existing binding */
 466         fib->lpm_tree = new_tree;
 467         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
 468         if (err)
 469                 goto err_tree_bind;
 470         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 471
 472         return 0;
 473
 474 err_tree_bind:
 475         fib->lpm_tree = lpm_tree;
 476         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 477         return err;
 478 }
 479
 480 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 481 {
 482         struct mlxsw_sp_vr *vr;
 483
 484         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 485         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 486         if (!vr)
 487                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 488         return vr;
 489 }
 490
 491 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 492 {
 493         if (!vr->rif_count && list_empty(&vr->fib4->node_list))
 494                 mlxsw_sp_vr_destroy(vr);
 495 }
 496
 497 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 498 {
 499         struct mlxsw_sp_vr *vr;
 500         u64 max_vrs;
 501         int i;
 502
 503         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 504                 return -EIO;
 505
 506         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 507         mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 508                                        GFP_KERNEL);
 509         if (!mlxsw_sp->router.vrs)
 510                 return -ENOMEM;
 511
 512         for (i = 0; i < max_vrs; i++) {
 513                 vr = &mlxsw_sp->router.vrs[i];
 514                 vr->id = i;
 515         }
 516
 517         return 0;
 518 }
 519
 520 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 521
 522 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 523 {
 524         /* At this stage we're guaranteed not to have new incoming
 525          * FIB notifications and the work queue is free from FIBs
 526          * sitting on top of mlxsw netdevs. However, we can still
 527          * have other FIBs queued. Flush the queue before flushing
 528          * the device's tables. No need for locks, as we're the only
 529          * writer.
 530          */
 531         mlxsw_core_flush_owq();
 532         mlxsw_sp_router_fib_flush(mlxsw_sp);
 533         kfree(mlxsw_sp->router.vrs);
 534 }
 535
 536 struct mlxsw_sp_neigh_key {
 537         struct neighbour *n;
 538 };
 539
 540 struct mlxsw_sp_neigh_entry {
 541         struct list_head rif_list_node;
 542         struct rhash_head ht_node;
 543         struct mlxsw_sp_neigh_key key;
 544         u16 rif;
 545         bool connected;
 546         unsigned char ha[ETH_ALEN];
 547         struct list_head nexthop_list; /* list of nexthops using
 548                                         * this neigh entry
 549                                         */
 550         struct list_head nexthop_neighs_list_node;
 551 };
 552
 553 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 554         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 555         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 556         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 557 };
 558
 559 static struct mlxsw_sp_neigh_entry *
 560 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
 561                            u16 rif)
 562 {
 563         struct mlxsw_sp_neigh_entry *neigh_entry;
 564
 565         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
 566         if (!neigh_entry)
 567                 return NULL;
 568
 569         neigh_entry->key.n = n;
 570         neigh_entry->rif = rif;
 571         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 572
 573         return neigh_entry;
 574 }
 575
 576 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
 577 {
 578         kfree(neigh_entry);
 579 }
 580
 581 static int
 582 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 583                             struct mlxsw_sp_neigh_entry *neigh_entry)
 584 {
 585         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 586                                       &neigh_entry->ht_node,
 587                                       mlxsw_sp_neigh_ht_params);
 588 }
 589
 590 static void
 591 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 592                             struct mlxsw_sp_neigh_entry *neigh_entry)
 593 {
 594         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 595                                &neigh_entry->ht_node,
 596                                mlxsw_sp_neigh_ht_params);
 597 }
 598
 599 static struct mlxsw_sp_neigh_entry *
 600 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 601 {
 602         struct mlxsw_sp_neigh_entry *neigh_entry;
 603         struct mlxsw_sp_rif *rif;
 604         int err;
 605
 606         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 607         if (!rif)
 608                 return ERR_PTR(-EINVAL);
 609
 610         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
 611         if (!neigh_entry)
 612                 return ERR_PTR(-ENOMEM);
 613
 614         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 615         if (err)
 616                 goto err_neigh_entry_insert;
 617
 618         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
 619
 620         return neigh_entry;
 621
 622 err_neigh_entry_insert:
 623         mlxsw_sp_neigh_entry_free(neigh_entry);
 624         return ERR_PTR(err);
 625 }
 626
 627 static void
 628 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 629                              struct mlxsw_sp_neigh_entry *neigh_entry)
 630 {
 631         list_del(&neigh_entry->rif_list_node);
 632         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 633         mlxsw_sp_neigh_entry_free(neigh_entry);
 634 }
 635
 636 static struct mlxsw_sp_neigh_entry *
 637 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 638 {
 639         struct mlxsw_sp_neigh_key key;
 640
 641         key.n = n;
 642         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 643                                       &key, mlxsw_sp_neigh_ht_params);
 644 }
 645
 646 static void
 647 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 648 {
 649         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 650
 651         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 652 }
 653
 654 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 655                                                    char *rauhtd_pl,
 656                                                    int ent_index)
 657 {
 658         struct net_device *dev;
 659         struct neighbour *n;
 660         __be32 dipn;
 661         u32 dip;
 662         u16 rif;
 663
 664         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 665
 666         if (!mlxsw_sp->rifs[rif]) {
 667                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 668                 return;
 669         }
 670
 671         dipn = htonl(dip);
 672         dev = mlxsw_sp->rifs[rif]->dev;
 673         n = neigh_lookup(&arp_tbl, &dipn, dev);
 674         if (!n) {
 675                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
 676                            &dip);
 677                 return;
 678         }
 679
 680         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 681         neigh_event_send(n, NULL);
 682         neigh_release(n);
 683 }
 684
 685 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 686                                                    char *rauhtd_pl,
 687                                                    int rec_index)
 688 {
 689         u8 num_entries;
 690         int i;
 691
 692         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 693                                                                 rec_index);
 694         /* Hardware starts counting at 0, so add 1. */
 695         num_entries++;
 696
 697         /* Each record consists of several neighbour entries. */
 698         for (i = 0; i < num_entries; i++) {
 699                 int ent_index;
 700
 701                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 702                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 703                                                        ent_index);
 704         }
 705
 706 }
 707
 708 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 709                                               char *rauhtd_pl, int rec_index)
 710 {
 711         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 712         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 713                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 714                                                        rec_index);
 715                 break;
 716         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 717                 WARN_ON_ONCE(1);
 718                 break;
 719         }
 720 }
 721
 722 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 723 {
 724         u8 num_rec, last_rec_index, num_entries;
 725
 726         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 727         last_rec_index = num_rec - 1;
 728
 729         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 730                 return false;
 731         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 732             MLXSW_REG_RAUHTD_TYPE_IPV6)
 733                 return true;
 734
 735         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 736                                                                 last_rec_index);
 737         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 738                 return true;
 739         return false;
 740 }
 741
 742 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 743 {
 744         char *rauhtd_pl;
 745         u8 num_rec;
 746         int i, err;
 747
 748         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 749         if (!rauhtd_pl)
 750                 return -ENOMEM;
 751
 752         /* Make sure the neighbour's netdev isn't removed in the
 753          * process.
 754          */
 755         rtnl_lock();
 756         do {
 757                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 758                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 759                                       rauhtd_pl);
 760                 if (err) {
 761                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 762                         break;
 763                 }
 764                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 765                 for (i = 0; i < num_rec; i++)
 766                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 767                                                           i);
 768         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 769         rtnl_unlock();
 770
 771         kfree(rauhtd_pl);
 772         return err;
 773 }
 774
 775 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 776 {
 777         struct mlxsw_sp_neigh_entry *neigh_entry;
 778
 779         /* Take RTNL mutex here to prevent lists from changes */
 780         rtnl_lock();
 781         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 782                             nexthop_neighs_list_node)
 783                 /* If this neigh have nexthops, make the kernel think this neigh
 784                  * is active regardless of the traffic.
 785                  */
 786                 neigh_event_send(neigh_entry->key.n, NULL);
 787         rtnl_unlock();
 788 }
 789
 790 static void
 791 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 792 {
 793         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 794
 795         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 796                                msecs_to_jiffies(interval));
 797 }
 798
 799 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 800 {
 801         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 802                                                  router.neighs_update.dw.work);
 803         int err;
 804
 805         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 806         if (err)
 807                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 808
 809         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 810
 811         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 812 }
 813
 814 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 815 {
 816         struct mlxsw_sp_neigh_entry *neigh_entry;
 817         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 818                                                  router.nexthop_probe_dw.work);
 819
 820         /* Iterate over nexthop neighbours, find those who are unresolved and
 821          * send arp on them. This solves the chicken-egg problem when
 822          * the nexthop wouldn't get offloaded until the neighbor is resolved
 823          * but it wouldn't get resolved ever in case traffic is flowing in HW
 824          * using different nexthop.
 825          *
 826          * Take RTNL mutex here to prevent lists from changes.
 827          */
 828         rtnl_lock();
 829         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 830                             nexthop_neighs_list_node)
 831                 if (!neigh_entry->connected)
 832                         neigh_event_send(neigh_entry->key.n, NULL);
 833         rtnl_unlock();
 834
 835         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 836                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 837 }
 838
 839 static void
 840 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 841                               struct mlxsw_sp_neigh_entry *neigh_entry,
 842                               bool removing);
 843
 844 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
 845 {
 846         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
 847                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
 848 }
 849
 850 static void
 851 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
 852                                 struct mlxsw_sp_neigh_entry *neigh_entry,
 853                                 enum mlxsw_reg_rauht_op op)
 854 {
 855         struct neighbour *n = neigh_entry->key.n;
 856         u32 dip = ntohl(*((__be32 *) n->primary_key));
 857         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 858
 859         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
 860                               dip);
 861         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
 862 }
 863
 864 static void
 865 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
 866                             struct mlxsw_sp_neigh_entry *neigh_entry,
 867                             bool adding)
 868 {
 869         if (!adding && !neigh_entry->connected)
 870                 return;
 871         neigh_entry->connected = adding;
 872         if (neigh_entry->key.n->tbl == &arp_tbl)
 873                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
 874                                                 mlxsw_sp_rauht_op(adding));
 875         else
 876                 WARN_ON_ONCE(1);
 877 }
 878
 879 struct mlxsw_sp_neigh_event_work {
 880         struct work_struct work;
 881         struct mlxsw_sp *mlxsw_sp;
 882         struct neighbour *n;
 883 };
 884
 885 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
 886 {
 887         struct mlxsw_sp_neigh_event_work *neigh_work =
 888                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
 889         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
 890         struct mlxsw_sp_neigh_entry *neigh_entry;
 891         struct neighbour *n = neigh_work->n;
 892         unsigned char ha[ETH_ALEN];
 893         bool entry_connected;
 894         u8 nud_state, dead;
 895
 896         /* If these parameters are changed after we release the lock,
 897          * then we are guaranteed to receive another event letting us
 898          * know about it.
 899          */
 900         read_lock_bh(&n->lock);
 901         memcpy(ha, n->ha, ETH_ALEN);
 902         nud_state = n->nud_state;
 903         dead = n->dead;
 904         read_unlock_bh(&n->lock);
 905
 906         rtnl_lock();
 907         entry_connected = nud_state & NUD_VALID && !dead;
 908         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 909         if (!entry_connected && !neigh_entry)
 910                 goto out;
 911         if (!neigh_entry) {
 912                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
 913                 if (IS_ERR(neigh_entry))
 914                         goto out;
 915         }
 916
 917         memcpy(neigh_entry->ha, ha, ETH_ALEN);
 918         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
 919         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
 920
 921         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
 922                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
 923
 924 out:
 925         rtnl_unlock();
 926         neigh_release(n);
 927         kfree(neigh_work);
 928 }
 929
 930 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
 931                                    unsigned long event, void *ptr)
 932 {
 933         struct mlxsw_sp_neigh_event_work *neigh_work;
 934         struct mlxsw_sp_port *mlxsw_sp_port;
 935         struct mlxsw_sp *mlxsw_sp;
 936         unsigned long interval;
 937         struct neigh_parms *p;
 938         struct neighbour *n;
 939
 940         switch (event) {
 941         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
 942                 p = ptr;
 943
 944                 /* We don't care about changes in the default table. */
 945                 if (!p->dev || p->tbl != &arp_tbl)
 946                         return NOTIFY_DONE;
 947
 948                 /* We are in atomic context and can't take RTNL mutex,
 949                  * so use RCU variant to walk the device chain.
 950                  */
 951                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
 952                 if (!mlxsw_sp_port)
 953                         return NOTIFY_DONE;
 954
 955                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 956                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
 957                 mlxsw_sp->router.neighs_update.interval = interval;
 958
 959                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 960                 break;
 961         case NETEVENT_NEIGH_UPDATE:
 962                 n = ptr;
 963
 964                 if (n->tbl != &arp_tbl)
 965                         return NOTIFY_DONE;
 966
 967                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
 968                 if (!mlxsw_sp_port)
 969                         return NOTIFY_DONE;
 970
 971                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
 972                 if (!neigh_work) {
 973                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 974                         return NOTIFY_BAD;
 975                 }
 976
 977                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
 978                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 979                 neigh_work->n = n;
 980
 981                 /* Take a reference to ensure the neighbour won't be
 982                  * destructed until we drop the reference in delayed
 983                  * work.
 984                  */
 985                 neigh_clone(n);
 986                 mlxsw_core_schedule_work(&neigh_work->work);
 987                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
 988                 break;
 989         }
 990
 991         return NOTIFY_DONE;
 992 }
 993
 994 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
 995 {
 996         int err;
 997
 998         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
 999                               &mlxsw_sp_neigh_ht_params);
1000         if (err)
1001                 return err;
1002
1003         /* Initialize the polling interval according to the default
1004          * table.
1005          */
1006         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1007
1008         /* Create the delayed works for the activity_update */
1009         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1010                           mlxsw_sp_router_neighs_update_work);
1011         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1012                           mlxsw_sp_router_probe_unresolved_nexthops);
1013         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1014         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1015         return 0;
1016 }
1017
1018 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1019 {
1020         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1021         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1022         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1023 }
1024
1025 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1026                                     const struct mlxsw_sp_rif *rif)
1027 {
1028         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1029
1030         mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1031                              rif->rif_index, rif->addr);
1032         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1033 }
1034
1035 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1036                                          struct mlxsw_sp_rif *rif)
1037 {
1038         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1039
1040         mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1041         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1042                                  rif_list_node)
1043                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1044 }
1045
1046 struct mlxsw_sp_nexthop_key {
1047         struct fib_nh *fib_nh;
1048 };
1049
1050 struct mlxsw_sp_nexthop {
1051         struct list_head neigh_list_node; /* member of neigh entry list */
1052         struct list_head rif_list_node;
1053         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1054                                                 * this belongs to
1055                                                 */
1056         struct rhash_head ht_node;
1057         struct mlxsw_sp_nexthop_key key;
1058         struct mlxsw_sp_rif *rif;
1059         u8 should_offload:1, /* set indicates this neigh is connected and
1060                               * should be put to KVD linear area of this group.
1061                               */
1062            offloaded:1, /* set in case the neigh is actually put into
1063                          * KVD linear area of this group.
1064                          */
1065            update:1; /* set indicates that MAC of this neigh should be
1066                       * updated in HW
1067                       */
1068         struct mlxsw_sp_neigh_entry *neigh_entry;
1069 };
1070
1071 struct mlxsw_sp_nexthop_group_key {
1072         struct fib_info *fi;
1073 };
1074
1075 struct mlxsw_sp_nexthop_group {
1076         struct rhash_head ht_node;
1077         struct list_head fib_list; /* list of fib entries that use this group */
1078         struct mlxsw_sp_nexthop_group_key key;
1079         u8 adj_index_valid:1,
1080            gateway:1; /* routes using the group use a gateway */
1081         u32 adj_index;
1082         u16 ecmp_size;
1083         u16 count;
1084         struct mlxsw_sp_nexthop nexthops[0];
1085 #define nh_rif  nexthops[0].rif
1086 };
1087
1088 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1089         .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1090         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1091         .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1092 };
1093
1094 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1095                                          struct mlxsw_sp_nexthop_group *nh_grp)
1096 {
1097         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1098                                       &nh_grp->ht_node,
1099                                       mlxsw_sp_nexthop_group_ht_params);
1100 }
1101
1102 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1103                                           struct mlxsw_sp_nexthop_group *nh_grp)
1104 {
1105         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1106                                &nh_grp->ht_node,
1107                                mlxsw_sp_nexthop_group_ht_params);
1108 }
1109
1110 static struct mlxsw_sp_nexthop_group *
1111 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1112                               struct mlxsw_sp_nexthop_group_key key)
1113 {
1114         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1115                                       mlxsw_sp_nexthop_group_ht_params);
1116 }
1117
1118 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1119         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1120         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1121         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1122 };
1123
1124 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1125                                    struct mlxsw_sp_nexthop *nh)
1126 {
1127         return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1128                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1129 }
1130
1131 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1132                                     struct mlxsw_sp_nexthop *nh)
1133 {
1134         rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1135                                mlxsw_sp_nexthop_ht_params);
1136 }
1137
1138 static struct mlxsw_sp_nexthop *
1139 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1140                         struct mlxsw_sp_nexthop_key key)
1141 {
1142         return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1143                                       mlxsw_sp_nexthop_ht_params);
1144 }
1145
1146 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1147                                              const struct mlxsw_sp_fib *fib,
1148                                              u32 adj_index, u16 ecmp_size,
1149                                              u32 new_adj_index,
1150                                              u16 new_ecmp_size)
1151 {
1152         char raleu_pl[MLXSW_REG_RALEU_LEN];
1153
1154         mlxsw_reg_raleu_pack(raleu_pl,
1155                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
1156                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
1157                              new_ecmp_size);
1158         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1159 }
1160
1161 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1162                                           struct mlxsw_sp_nexthop_group *nh_grp,
1163                                           u32 old_adj_index, u16 old_ecmp_size)
1164 {
1165         struct mlxsw_sp_fib_entry *fib_entry;
1166         struct mlxsw_sp_fib *fib = NULL;
1167         int err;
1168
1169         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1170                 if (fib == fib_entry->fib_node->fib)
1171                         continue;
1172                 fib = fib_entry->fib_node->fib;
1173                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1174                                                         old_adj_index,
1175                                                         old_ecmp_size,
1176                                                         nh_grp->adj_index,
1177                                                         nh_grp->ecmp_size);
1178                 if (err)
1179                         return err;
1180         }
1181         return 0;
1182 }
1183
1184 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1185                                        struct mlxsw_sp_nexthop *nh)
1186 {
1187         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1188         char ratr_pl[MLXSW_REG_RATR_LEN];
1189
1190         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1191                             true, adj_index, neigh_entry->rif);
1192         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1193         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1194 }
1195
1196 static int
1197 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1198                                   struct mlxsw_sp_nexthop_group *nh_grp,
1199                                   bool reallocate)
1200 {
1201         u32 adj_index = nh_grp->adj_index; /* base */
1202         struct mlxsw_sp_nexthop *nh;
1203         int i;
1204         int err;
1205
1206         for (i = 0; i < nh_grp->count; i++) {
1207                 nh = &nh_grp->nexthops[i];
1208
1209                 if (!nh->should_offload) {
1210                         nh->offloaded = 0;
1211                         continue;
1212                 }
1213
1214                 if (nh->update || reallocate) {
1215                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1216                                                           adj_index, nh);
1217                         if (err)
1218                                 return err;
1219                         nh->update = 0;
1220                         nh->offloaded = 1;
1221                 }
1222                 adj_index++;
1223         }
1224         return 0;
1225 }
1226
1227 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1228                                      struct mlxsw_sp_fib_entry *fib_entry);
1229
1230 static int
1231 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1232                                     struct mlxsw_sp_nexthop_group *nh_grp)
1233 {
1234         struct mlxsw_sp_fib_entry *fib_entry;
1235         int err;
1236
1237         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1238                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1239                 if (err)
1240                         return err;
1241         }
1242         return 0;
1243 }
1244
1245 static void
1246 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1247                                struct mlxsw_sp_nexthop_group *nh_grp)
1248 {
1249         struct mlxsw_sp_nexthop *nh;
1250         bool offload_change = false;
1251         u32 adj_index;
1252         u16 ecmp_size = 0;
1253         bool old_adj_index_valid;
1254         u32 old_adj_index;
1255         u16 old_ecmp_size;
1256         int ret;
1257         int i;
1258         int err;
1259
1260         if (!nh_grp->gateway) {
1261                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1262                 return;
1263         }
1264
1265         for (i = 0; i < nh_grp->count; i++) {
1266                 nh = &nh_grp->nexthops[i];
1267
1268                 if (nh->should_offload ^ nh->offloaded) {
1269                         offload_change = true;
1270                         if (nh->should_offload)
1271                                 nh->update = 1;
1272                 }
1273                 if (nh->should_offload)
1274                         ecmp_size++;
1275         }
1276         if (!offload_change) {
1277                 /* Nothing was added or removed, so no need to reallocate. Just
1278                  * update MAC on existing adjacency indexes.
1279                  */
1280                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1281                                                         false);
1282                 if (err) {
1283                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1284                         goto set_trap;
1285                 }
1286                 return;
1287         }
1288         if (!ecmp_size)
1289                 /* No neigh of this group is connected so we just set
1290                  * the trap and let everthing flow through kernel.
1291                  */
1292                 goto set_trap;
1293
1294         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1295         if (ret < 0) {
1296                 /* We ran out of KVD linear space, just set the
1297                  * trap and let everything flow through kernel.
1298                  */
1299                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1300                 goto set_trap;
1301         }
1302         adj_index = ret;
1303         old_adj_index_valid = nh_grp->adj_index_valid;
1304         old_adj_index = nh_grp->adj_index;
1305         old_ecmp_size = nh_grp->ecmp_size;
1306         nh_grp->adj_index_valid = 1;
1307         nh_grp->adj_index = adj_index;
1308         nh_grp->ecmp_size = ecmp_size;
1309         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1310         if (err) {
1311                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1312                 goto set_trap;
1313         }
1314
1315         if (!old_adj_index_valid) {
1316                 /* The trap was set for fib entries, so we have to call
1317                  * fib entry update to unset it and use adjacency index.
1318                  */
1319                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1320                 if (err) {
1321                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1322                         goto set_trap;
1323                 }
1324                 return;
1325         }
1326
1327         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1328                                              old_adj_index, old_ecmp_size);
1329         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1330         if (err) {
1331                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1332                 goto set_trap;
1333         }
1334         return;
1335
1336 set_trap:
1337         old_adj_index_valid = nh_grp->adj_index_valid;
1338         nh_grp->adj_index_valid = 0;
1339         for (i = 0; i < nh_grp->count; i++) {
1340                 nh = &nh_grp->nexthops[i];
1341                 nh->offloaded = 0;
1342         }
1343         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1344         if (err)
1345                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1346         if (old_adj_index_valid)
1347                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1348 }
1349
1350 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1351                                             bool removing)
1352 {
1353         if (!removing && !nh->should_offload)
1354                 nh->should_offload = 1;
1355         else if (removing && nh->offloaded)
1356                 nh->should_offload = 0;
1357         nh->update = 1;
1358 }
1359
1360 static void
1361 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1362                               struct mlxsw_sp_neigh_entry *neigh_entry,
1363                               bool removing)
1364 {
1365         struct mlxsw_sp_nexthop *nh;
1366
1367         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1368                             neigh_list_node) {
1369                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1370                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1371         }
1372 }
1373
1374 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1375                                       struct mlxsw_sp_rif *rif)
1376 {
1377         if (nh->rif)
1378                 return;
1379
1380         nh->rif = rif;
1381         list_add(&nh->rif_list_node, &rif->nexthop_list);
1382 }
1383
1384 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1385 {
1386         if (!nh->rif)
1387                 return;
1388
1389         list_del(&nh->rif_list_node);
1390         nh->rif = NULL;
1391 }
1392
1393 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1394                                        struct mlxsw_sp_nexthop *nh)
1395 {
1396         struct mlxsw_sp_neigh_entry *neigh_entry;
1397         struct fib_nh *fib_nh = nh->key.fib_nh;
1398         struct neighbour *n;
1399         u8 nud_state, dead;
1400         int err;
1401
1402         if (!nh->nh_grp->gateway || nh->neigh_entry)
1403                 return 0;
1404
1405         /* Take a reference of neigh here ensuring that neigh would
1406          * not be detructed before the nexthop entry is finished.
1407          * The reference is taken either in neigh_lookup() or
1408          * in neigh_create() in case n is not found.
1409          */
1410         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1411         if (!n) {
1412                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1413                 if (IS_ERR(n))
1414                         return PTR_ERR(n);
1415                 neigh_event_send(n, NULL);
1416         }
1417         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1418         if (!neigh_entry) {
1419                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1420                 if (IS_ERR(neigh_entry)) {
1421                         err = -EINVAL;
1422                         goto err_neigh_entry_create;
1423                 }
1424         }
1425
1426         /* If that is the first nexthop connected to that neigh, add to
1427          * nexthop_neighs_list
1428          */
1429         if (list_empty(&neigh_entry->nexthop_list))
1430                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1431                               &mlxsw_sp->router.nexthop_neighs_list);
1432
1433         nh->neigh_entry = neigh_entry;
1434         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1435         read_lock_bh(&n->lock);
1436         nud_state = n->nud_state;
1437         dead = n->dead;
1438         read_unlock_bh(&n->lock);
1439         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1440
1441         return 0;
1442
1443 err_neigh_entry_create:
1444         neigh_release(n);
1445         return err;
1446 }
1447
1448 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1449                                         struct mlxsw_sp_nexthop *nh)
1450 {
1451         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1452         struct neighbour *n;
1453
1454         if (!neigh_entry)
1455                 return;
1456         n = neigh_entry->key.n;
1457
1458         __mlxsw_sp_nexthop_neigh_update(nh, true);
1459         list_del(&nh->neigh_list_node);
1460         nh->neigh_entry = NULL;
1461
1462         /* If that is the last nexthop connected to that neigh, remove from
1463          * nexthop_neighs_list
1464          */
1465         if (list_empty(&neigh_entry->nexthop_list))
1466                 list_del(&neigh_entry->nexthop_neighs_list_node);
1467
1468         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1469                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1470
1471         neigh_release(n);
1472 }
1473
1474 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1475                                  struct mlxsw_sp_nexthop_group *nh_grp,
1476                                  struct mlxsw_sp_nexthop *nh,
1477                                  struct fib_nh *fib_nh)
1478 {
1479         struct net_device *dev = fib_nh->nh_dev;
1480         struct in_device *in_dev;
1481         struct mlxsw_sp_rif *rif;
1482         int err;
1483
1484         nh->nh_grp = nh_grp;
1485         nh->key.fib_nh = fib_nh;
1486         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1487         if (err)
1488                 return err;
1489
1490         if (!dev)
1491                 return 0;
1492
1493         in_dev = __in_dev_get_rtnl(dev);
1494         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1495             fib_nh->nh_flags & RTNH_F_LINKDOWN)
1496                 return 0;
1497
1498         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1499         if (!rif)
1500                 return 0;
1501         mlxsw_sp_nexthop_rif_init(nh, rif);
1502
1503         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1504         if (err)
1505                 goto err_nexthop_neigh_init;
1506
1507         return 0;
1508
1509 err_nexthop_neigh_init:
1510         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1511         return err;
1512 }
1513
1514 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1515                                   struct mlxsw_sp_nexthop *nh)
1516 {
1517         mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1518         mlxsw_sp_nexthop_rif_fini(nh);
1519         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1520 }
1521
1522 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1523                                    unsigned long event, struct fib_nh *fib_nh)
1524 {
1525         struct mlxsw_sp_nexthop_key key;
1526         struct mlxsw_sp_nexthop *nh;
1527         struct mlxsw_sp_rif *rif;
1528
1529         if (mlxsw_sp->router.aborted)
1530                 return;
1531
1532         key.fib_nh = fib_nh;
1533         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1534         if (WARN_ON_ONCE(!nh))
1535                 return;
1536
1537         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1538         if (!rif)
1539                 return;
1540
1541         switch (event) {
1542         case FIB_EVENT_NH_ADD:
1543                 mlxsw_sp_nexthop_rif_init(nh, rif);
1544                 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1545                 break;
1546         case FIB_EVENT_NH_DEL:
1547                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1548                 mlxsw_sp_nexthop_rif_fini(nh);
1549                 break;
1550         }
1551
1552         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1553 }
1554
1555 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1556                                            struct mlxsw_sp_rif *rif)
1557 {
1558         struct mlxsw_sp_nexthop *nh, *tmp;
1559
1560         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1561                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1562                 mlxsw_sp_nexthop_rif_fini(nh);
1563                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1564         }
1565 }
1566
1567 static struct mlxsw_sp_nexthop_group *
1568 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1569 {
1570         struct mlxsw_sp_nexthop_group *nh_grp;
1571         struct mlxsw_sp_nexthop *nh;
1572         struct fib_nh *fib_nh;
1573         size_t alloc_size;
1574         int i;
1575         int err;
1576
1577         alloc_size = sizeof(*nh_grp) +
1578                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1579         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1580         if (!nh_grp)
1581                 return ERR_PTR(-ENOMEM);
1582         INIT_LIST_HEAD(&nh_grp->fib_list);
1583         nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1584         nh_grp->count = fi->fib_nhs;
1585         nh_grp->key.fi = fi;
1586         for (i = 0; i < nh_grp->count; i++) {
1587                 nh = &nh_grp->nexthops[i];
1588                 fib_nh = &fi->fib_nh[i];
1589                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1590                 if (err)
1591                         goto err_nexthop_init;
1592         }
1593         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1594         if (err)
1595                 goto err_nexthop_group_insert;
1596         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1597         return nh_grp;
1598
1599 err_nexthop_group_insert:
1600 err_nexthop_init:
1601         for (i--; i >= 0; i--) {
1602                 nh = &nh_grp->nexthops[i];
1603                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1604         }
1605         kfree(nh_grp);
1606         return ERR_PTR(err);
1607 }
1608
1609 static void
1610 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1611                                struct mlxsw_sp_nexthop_group *nh_grp)
1612 {
1613         struct mlxsw_sp_nexthop *nh;
1614         int i;
1615
1616         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1617         for (i = 0; i < nh_grp->count; i++) {
1618                 nh = &nh_grp->nexthops[i];
1619                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1620         }
1621         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1622         WARN_ON_ONCE(nh_grp->adj_index_valid);
1623         kfree(nh_grp);
1624 }
1625
1626 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1627                                       struct mlxsw_sp_fib_entry *fib_entry,
1628                                       struct fib_info *fi)
1629 {
1630         struct mlxsw_sp_nexthop_group_key key;
1631         struct mlxsw_sp_nexthop_group *nh_grp;
1632
1633         key.fi = fi;
1634         nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1635         if (!nh_grp) {
1636                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1637                 if (IS_ERR(nh_grp))
1638                         return PTR_ERR(nh_grp);
1639         }
1640         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1641         fib_entry->nh_group = nh_grp;
1642         return 0;
1643 }
1644
1645 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1646                                        struct mlxsw_sp_fib_entry *fib_entry)
1647 {
1648         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1649
1650         list_del(&fib_entry->nexthop_group_node);
1651         if (!list_empty(&nh_grp->fib_list))
1652                 return;
1653         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1654 }
1655
1656 static bool
1657 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1658 {
1659         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1660
1661         if (fib_entry->params.tos)
1662                 return false;
1663
1664         switch (fib_entry->type) {
1665         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1666                 return !!nh_group->adj_index_valid;
1667         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1668                 return !!nh_group->nh_rif;
1669         default:
1670                 return false;
1671         }
1672 }
1673
1674 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1675 {
1676         fib_entry->offloaded = true;
1677
1678         switch (fib_entry->fib_node->fib->proto) {
1679         case MLXSW_SP_L3_PROTO_IPV4:
1680                 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1681                 break;
1682         case MLXSW_SP_L3_PROTO_IPV6:
1683                 WARN_ON_ONCE(1);
1684         }
1685 }
1686
1687 static void
1688 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1689 {
1690         switch (fib_entry->fib_node->fib->proto) {
1691         case MLXSW_SP_L3_PROTO_IPV4:
1692                 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1693                 break;
1694         case MLXSW_SP_L3_PROTO_IPV6:
1695                 WARN_ON_ONCE(1);
1696         }
1697
1698         fib_entry->offloaded = false;
1699 }
1700
1701 static void
1702 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1703                                    enum mlxsw_reg_ralue_op op, int err)
1704 {
1705         switch (op) {
1706         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1707                 if (!fib_entry->offloaded)
1708                         return;
1709                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1710         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1711                 if (err)
1712                         return;
1713                 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1714                     !fib_entry->offloaded)
1715                         mlxsw_sp_fib_entry_offload_set(fib_entry);
1716                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1717                          fib_entry->offloaded)
1718                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
1719                 return;
1720         default:
1721                 return;
1722         }
1723 }
1724
1725 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1726                                          struct mlxsw_sp_fib_entry *fib_entry,
1727                                          enum mlxsw_reg_ralue_op op)
1728 {
1729         char ralue_pl[MLXSW_REG_RALUE_LEN];
1730         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1731         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1732         enum mlxsw_reg_ralue_trap_action trap_action;
1733         u16 trap_id = 0;
1734         u32 adjacency_index = 0;
1735         u16 ecmp_size = 0;
1736
1737         /* In case the nexthop group adjacency index is valid, use it
1738          * with provided ECMP size. Otherwise, setup trap and pass
1739          * traffic to kernel.
1740          */
1741         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1742                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1743                 adjacency_index = fib_entry->nh_group->adj_index;
1744                 ecmp_size = fib_entry->nh_group->ecmp_size;
1745         } else {
1746                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1747                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1748         }
1749
1750         mlxsw_reg_ralue_pack4(ralue_pl,
1751                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1752                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1753                               *p_dip);
1754         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1755                                         adjacency_index, ecmp_size);
1756         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1757 }
1758
1759 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1760                                         struct mlxsw_sp_fib_entry *fib_entry,
1761                                         enum mlxsw_reg_ralue_op op)
1762 {
1763         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
1764         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1765         enum mlxsw_reg_ralue_trap_action trap_action;
1766         char ralue_pl[MLXSW_REG_RALUE_LEN];
1767         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1768         u16 trap_id = 0;
1769         u16 rif_index = 0;
1770
1771         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1772                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1773                 rif_index = rif->rif_index;
1774         } else {
1775                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1776                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1777         }
1778
1779         mlxsw_reg_ralue_pack4(ralue_pl,
1780                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1781                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1782                               *p_dip);
1783         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
1784                                        rif_index);
1785         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1786 }
1787
1788 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1789                                        struct mlxsw_sp_fib_entry *fib_entry,
1790                                        enum mlxsw_reg_ralue_op op)
1791 {
1792         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1793         char ralue_pl[MLXSW_REG_RALUE_LEN];
1794         u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1795
1796         mlxsw_reg_ralue_pack4(ralue_pl,
1797                               (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1798                               fib->vr->id, fib_entry->fib_node->key.prefix_len,
1799                               *p_dip);
1800         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1801         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1802 }
1803
1804 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1805                                   struct mlxsw_sp_fib_entry *fib_entry,
1806                                   enum mlxsw_reg_ralue_op op)
1807 {
1808         switch (fib_entry->type) {
1809         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1810                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1811         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1812                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1813         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1814                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1815         }
1816         return -EINVAL;
1817 }
1818
1819 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1820                                  struct mlxsw_sp_fib_entry *fib_entry,
1821                                  enum mlxsw_reg_ralue_op op)
1822 {
1823         int err = -EINVAL;
1824
1825         switch (fib_entry->fib_node->fib->proto) {
1826         case MLXSW_SP_L3_PROTO_IPV4:
1827                 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1828                 break;
1829         case MLXSW_SP_L3_PROTO_IPV6:
1830                 return err;
1831         }
1832         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1833         return err;
1834 }
1835
1836 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1837                                      struct mlxsw_sp_fib_entry *fib_entry)
1838 {
1839         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1840                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1841 }
1842
1843 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1844                                   struct mlxsw_sp_fib_entry *fib_entry)
1845 {
1846         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1847                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1848 }
1849
1850 static int
1851 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1852                              const struct fib_entry_notifier_info *fen_info,
1853                              struct mlxsw_sp_fib_entry *fib_entry)
1854 {
1855         struct fib_info *fi = fen_info->fi;
1856
1857         switch (fen_info->type) {
1858         case RTN_BROADCAST: /* fall through */
1859         case RTN_LOCAL:
1860                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1861                 return 0;
1862         case RTN_UNREACHABLE: /* fall through */
1863         case RTN_BLACKHOLE: /* fall through */
1864         case RTN_PROHIBIT:
1865                 /* Packets hitting these routes need to be trapped, but
1866                  * can do so with a lower priority than packets directed
1867                  * at the host, so use action type local instead of trap.
1868                  */
1869                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1870                 return 0;
1871         case RTN_UNICAST:
1872                 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1873                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1874                 else
1875                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1876                 return 0;
1877         default:
1878                 return -EINVAL;
1879         }
1880 }
1881
1882 static struct mlxsw_sp_fib_entry *
1883 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1884                            struct mlxsw_sp_fib_node *fib_node,
1885                            const struct fib_entry_notifier_info *fen_info)
1886 {
1887         struct mlxsw_sp_fib_entry *fib_entry;
1888         int err;
1889
1890         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1891         if (!fib_entry) {
1892                 err = -ENOMEM;
1893                 goto err_fib_entry_alloc;
1894         }
1895
1896         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1897         if (err)
1898                 goto err_fib4_entry_type_set;
1899
1900         err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1901         if (err)
1902                 goto err_nexthop_group_get;
1903
1904         fib_entry->params.prio = fen_info->fi->fib_priority;
1905         fib_entry->params.tb_id = fen_info->tb_id;
1906         fib_entry->params.type = fen_info->type;
1907         fib_entry->params.tos = fen_info->tos;
1908
1909         fib_entry->fib_node = fib_node;
1910
1911         return fib_entry;
1912
1913 err_nexthop_group_get:
1914 err_fib4_entry_type_set:
1915         kfree(fib_entry);
1916 err_fib_entry_alloc:
1917         return ERR_PTR(err);
1918 }
1919
1920 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1921                                         struct mlxsw_sp_fib_entry *fib_entry)
1922 {
1923         mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1924         kfree(fib_entry);
1925 }
1926
1927 static struct mlxsw_sp_fib_node *
1928 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1929                        const struct fib_entry_notifier_info *fen_info);
1930
1931 static struct mlxsw_sp_fib_entry *
1932 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1933                            const struct fib_entry_notifier_info *fen_info)
1934 {
1935         struct mlxsw_sp_fib_entry *fib_entry;
1936         struct mlxsw_sp_fib_node *fib_node;
1937
1938         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1939         if (IS_ERR(fib_node))
1940                 return NULL;
1941
1942         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1943                 if (fib_entry->params.tb_id == fen_info->tb_id &&
1944                     fib_entry->params.tos == fen_info->tos &&
1945                     fib_entry->params.type == fen_info->type &&
1946                     fib_entry->nh_group->key.fi == fen_info->fi) {
1947                         return fib_entry;
1948                 }
1949         }
1950
1951         return NULL;
1952 }
1953
1954 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1955         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1956         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1957         .key_len = sizeof(struct mlxsw_sp_fib_key),
1958         .automatic_shrinking = true,
1959 };
1960
1961 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1962                                     struct mlxsw_sp_fib_node *fib_node)
1963 {
1964         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1965                                       mlxsw_sp_fib_ht_params);
1966 }
1967
1968 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1969                                      struct mlxsw_sp_fib_node *fib_node)
1970 {
1971         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1972                                mlxsw_sp_fib_ht_params);
1973 }
1974
1975 static struct mlxsw_sp_fib_node *
1976 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1977                          size_t addr_len, unsigned char prefix_len)
1978 {
1979         struct mlxsw_sp_fib_key key;
1980
1981         memset(&key, 0, sizeof(key));
1982         memcpy(key.addr, addr, addr_len);
1983         key.prefix_len = prefix_len;
1984         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
1985 }
1986
1987 static struct mlxsw_sp_fib_node *
1988 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
1989                          size_t addr_len, unsigned char prefix_len)
1990 {
1991         struct mlxsw_sp_fib_node *fib_node;
1992
1993         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
1994         if (!fib_node)
1995                 return NULL;
1996
1997         INIT_LIST_HEAD(&fib_node->entry_list);
1998         list_add(&fib_node->list, &fib->node_list);
1999         memcpy(fib_node->key.addr, addr, addr_len);
2000         fib_node->key.prefix_len = prefix_len;
2001
2002         return fib_node;
2003 }
2004
2005 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2006 {
2007         list_del(&fib_node->list);
2008         WARN_ON(!list_empty(&fib_node->entry_list));
2009         kfree(fib_node);
2010 }
2011
2012 static bool
2013 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2014                                  const struct mlxsw_sp_fib_entry *fib_entry)
2015 {
2016         return list_first_entry(&fib_node->entry_list,
2017                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
2018 }
2019
2020 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2021 {
2022         unsigned char prefix_len = fib_node->key.prefix_len;
2023         struct mlxsw_sp_fib *fib = fib_node->fib;
2024
2025         if (fib->prefix_ref_count[prefix_len]++ == 0)
2026                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2027 }
2028
2029 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2030 {
2031         unsigned char prefix_len = fib_node->key.prefix_len;
2032         struct mlxsw_sp_fib *fib = fib_node->fib;
2033
2034         if (--fib->prefix_ref_count[prefix_len] == 0)
2035                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2036 }
2037
2038 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2039                                   struct mlxsw_sp_fib_node *fib_node,
2040                                   struct mlxsw_sp_fib *fib)
2041 {
2042         struct mlxsw_sp_prefix_usage req_prefix_usage;
2043         struct mlxsw_sp_lpm_tree *lpm_tree;
2044         int err;
2045
2046         err = mlxsw_sp_fib_node_insert(fib, fib_node);
2047         if (err)
2048                 return err;
2049         fib_node->fib = fib;
2050
2051         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2052         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2053
2054         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2055                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2056                                                  &req_prefix_usage);
2057                 if (err)
2058                         goto err_tree_check;
2059         } else {
2060                 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2061                                                  fib->proto);
2062                 if (IS_ERR(lpm_tree))
2063                         return PTR_ERR(lpm_tree);
2064                 fib->lpm_tree = lpm_tree;
2065                 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2066                 if (err)
2067                         goto err_tree_bind;
2068         }
2069
2070         mlxsw_sp_fib_node_prefix_inc(fib_node);
2071
2072         return 0;
2073
2074 err_tree_bind:
2075         fib->lpm_tree = NULL;
2076         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2077 err_tree_check:
2078         fib_node->fib = NULL;
2079         mlxsw_sp_fib_node_remove(fib, fib_node);
2080         return err;
2081 }
2082
2083 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2084                                    struct mlxsw_sp_fib_node *fib_node)
2085 {
2086         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2087         struct mlxsw_sp_fib *fib = fib_node->fib;
2088
2089         mlxsw_sp_fib_node_prefix_dec(fib_node);
2090
2091         if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2092                 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2093                 fib->lpm_tree = NULL;
2094                 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2095         } else {
2096                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2097         }
2098
2099         fib_node->fib = NULL;
2100         mlxsw_sp_fib_node_remove(fib, fib_node);
2101 }
2102
2103 static struct mlxsw_sp_fib_node *
2104 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2105                        const struct fib_entry_notifier_info *fen_info)
2106 {
2107         struct mlxsw_sp_fib_node *fib_node;
2108         struct mlxsw_sp_fib *fib;
2109         struct mlxsw_sp_vr *vr;
2110         int err;
2111
2112         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2113         if (IS_ERR(vr))
2114                 return ERR_CAST(vr);
2115         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2116
2117         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2118                                             sizeof(fen_info->dst),
2119                                             fen_info->dst_len);
2120         if (fib_node)
2121                 return fib_node;
2122
2123         fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2124                                             sizeof(fen_info->dst),
2125                                             fen_info->dst_len);
2126         if (!fib_node) {
2127                 err = -ENOMEM;
2128                 goto err_fib_node_create;
2129         }
2130
2131         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2132         if (err)
2133                 goto err_fib_node_init;
2134
2135         return fib_node;
2136
2137 err_fib_node_init:
2138         mlxsw_sp_fib_node_destroy(fib_node);
2139 err_fib_node_create:
2140         mlxsw_sp_vr_put(vr);
2141         return ERR_PTR(err);
2142 }
2143
2144 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2145                                    struct mlxsw_sp_fib_node *fib_node)
2146 {
2147         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2148
2149         if (!list_empty(&fib_node->entry_list))
2150                 return;
2151         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2152         mlxsw_sp_fib_node_destroy(fib_node);
2153         mlxsw_sp_vr_put(vr);
2154 }
2155
2156 static struct mlxsw_sp_fib_entry *
2157 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2158                               const struct mlxsw_sp_fib_entry_params *params)
2159 {
2160         struct mlxsw_sp_fib_entry *fib_entry;
2161
2162         list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2163                 if (fib_entry->params.tb_id > params->tb_id)
2164                         continue;
2165                 if (fib_entry->params.tb_id != params->tb_id)
2166                         break;
2167                 if (fib_entry->params.tos > params->tos)
2168                         continue;
2169                 if (fib_entry->params.prio >= params->prio ||
2170                     fib_entry->params.tos < params->tos)
2171                         return fib_entry;
2172         }
2173
2174         return NULL;
2175 }
2176
2177 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2178                                           struct mlxsw_sp_fib_entry *new_entry)
2179 {
2180         struct mlxsw_sp_fib_node *fib_node;
2181
2182         if (WARN_ON(!fib_entry))
2183                 return -EINVAL;
2184
2185         fib_node = fib_entry->fib_node;
2186         list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2187                 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2188                     fib_entry->params.tos != new_entry->params.tos ||
2189                     fib_entry->params.prio != new_entry->params.prio)
2190                         break;
2191         }
2192
2193         list_add_tail(&new_entry->list, &fib_entry->list);
2194         return 0;
2195 }
2196
2197 static int
2198 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2199                                struct mlxsw_sp_fib_entry *new_entry,
2200                                bool replace, bool append)
2201 {
2202         struct mlxsw_sp_fib_entry *fib_entry;
2203
2204         fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2205
2206         if (append)
2207                 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2208         if (replace && WARN_ON(!fib_entry))
2209                 return -EINVAL;
2210
2211         /* Insert new entry before replaced one, so that we can later
2212          * remove the second.
2213          */
2214         if (fib_entry) {
2215                 list_add_tail(&new_entry->list, &fib_entry->list);
2216         } else {
2217                 struct mlxsw_sp_fib_entry *last;
2218
2219                 list_for_each_entry(last, &fib_node->entry_list, list) {
2220                         if (new_entry->params.tb_id > last->params.tb_id)
2221                                 break;
2222                         fib_entry = last;
2223                 }
2224
2225                 if (fib_entry)
2226                         list_add(&new_entry->list, &fib_entry->list);
2227                 else
2228                         list_add(&new_entry->list, &fib_node->entry_list);
2229         }
2230
2231         return 0;
2232 }
2233
2234 static void
2235 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2236 {
2237         list_del(&fib_entry->list);
2238 }
2239
2240 static int
2241 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2242                              const struct mlxsw_sp_fib_node *fib_node,
2243                              struct mlxsw_sp_fib_entry *fib_entry)
2244 {
2245         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2246                 return 0;
2247
2248         /* To prevent packet loss, overwrite the previously offloaded
2249          * entry.
2250          */
2251         if (!list_is_singular(&fib_node->entry_list)) {
2252                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2253                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2254
2255                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2256         }
2257
2258         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2259 }
2260
2261 static void
2262 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2263                              const struct mlxsw_sp_fib_node *fib_node,
2264                              struct mlxsw_sp_fib_entry *fib_entry)
2265 {
2266         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2267                 return;
2268
2269         /* Promote the next entry by overwriting the deleted entry */
2270         if (!list_is_singular(&fib_node->entry_list)) {
2271                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2272                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2273
2274                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2275                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2276                 return;
2277         }
2278
2279         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2280 }
2281
2282 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2283                                          struct mlxsw_sp_fib_entry *fib_entry,
2284                                          bool replace, bool append)
2285 {
2286         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2287         int err;
2288
2289         err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2290                                              append);
2291         if (err)
2292                 return err;
2293
2294         err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2295         if (err)
2296                 goto err_fib4_node_entry_add;
2297
2298         return 0;
2299
2300 err_fib4_node_entry_add:
2301         mlxsw_sp_fib4_node_list_remove(fib_entry);
2302         return err;
2303 }
2304
2305 static void
2306 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2307                                 struct mlxsw_sp_fib_entry *fib_entry)
2308 {
2309         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2310
2311         mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2312         mlxsw_sp_fib4_node_list_remove(fib_entry);
2313 }
2314
2315 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2316                                         struct mlxsw_sp_fib_entry *fib_entry,
2317                                         bool replace)
2318 {
2319         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2320         struct mlxsw_sp_fib_entry *replaced;
2321
2322         if (!replace)
2323                 return;
2324
2325         /* We inserted the new entry before replaced one */
2326         replaced = list_next_entry(fib_entry, list);
2327
2328         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2329         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2330         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2331 }
2332
2333 static int
2334 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2335                          const struct fib_entry_notifier_info *fen_info,
2336                          bool replace, bool append)
2337 {
2338         struct mlxsw_sp_fib_entry *fib_entry;
2339         struct mlxsw_sp_fib_node *fib_node;
2340         int err;
2341
2342         if (mlxsw_sp->router.aborted)
2343                 return 0;
2344
2345         fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2346         if (IS_ERR(fib_node)) {
2347                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2348                 return PTR_ERR(fib_node);
2349         }
2350
2351         fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2352         if (IS_ERR(fib_entry)) {
2353                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2354                 err = PTR_ERR(fib_entry);
2355                 goto err_fib4_entry_create;
2356         }
2357
2358         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2359                                             append);
2360         if (err) {
2361                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2362                 goto err_fib4_node_entry_link;
2363         }
2364
2365         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2366
2367         return 0;
2368
2369 err_fib4_node_entry_link:
2370         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2371 err_fib4_entry_create:
2372         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2373         return err;
2374 }
2375
2376 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2377                                      struct fib_entry_notifier_info *fen_info)
2378 {
2379         struct mlxsw_sp_fib_entry *fib_entry;
2380         struct mlxsw_sp_fib_node *fib_node;
2381
2382         if (mlxsw_sp->router.aborted)
2383                 return;
2384
2385         fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2386         if (WARN_ON(!fib_entry))
2387                 return;
2388         fib_node = fib_entry->fib_node;
2389
2390         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2391         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2392         mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2393 }
2394
2395 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2396 {
2397         char ralta_pl[MLXSW_REG_RALTA_LEN];
2398         char ralst_pl[MLXSW_REG_RALST_LEN];
2399         int i, err;
2400
2401         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2402                              MLXSW_SP_LPM_TREE_MIN);
2403         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2404         if (err)
2405                 return err;
2406
2407         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2408         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2409         if (err)
2410                 return err;
2411
2412         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2413                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2414                 char raltb_pl[MLXSW_REG_RALTB_LEN];
2415                 char ralue_pl[MLXSW_REG_RALUE_LEN];
2416
2417                 if (!mlxsw_sp_vr_is_used(vr))
2418                         continue;
2419
2420                 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2421                                      MLXSW_REG_RALXX_PROTOCOL_IPV4,
2422                                      MLXSW_SP_LPM_TREE_MIN);
2423                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2424                                       raltb_pl);
2425                 if (err)
2426                         return err;
2427
2428                 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2429                                       MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2430                                       0);
2431                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2432                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2433                                       ralue_pl);
2434                 if (err)
2435                         return err;
2436         }
2437
2438         return 0;
2439 }
2440
2441 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2442                                      struct mlxsw_sp_fib_node *fib_node)
2443 {
2444         struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2445
2446         list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2447                 bool do_break = &tmp->list == &fib_node->entry_list;
2448
2449                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2450                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2451                 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2452                 /* Break when entry list is empty and node was freed.
2453                  * Otherwise, we'll access freed memory in the next
2454                  * iteration.
2455                  */
2456                 if (do_break)
2457                         break;
2458         }
2459 }
2460
2461 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2462                                     struct mlxsw_sp_fib_node *fib_node)
2463 {
2464         switch (fib_node->fib->proto) {
2465         case MLXSW_SP_L3_PROTO_IPV4:
2466                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2467                 break;
2468         case MLXSW_SP_L3_PROTO_IPV6:
2469                 WARN_ON_ONCE(1);
2470                 break;
2471         }
2472 }
2473
2474 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2475                                   struct mlxsw_sp_vr *vr,
2476                                   enum mlxsw_sp_l3proto proto)
2477 {
2478         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2479         struct mlxsw_sp_fib_node *fib_node, *tmp;
2480
2481         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2482                 bool do_break = &tmp->list == &fib->node_list;
2483
2484                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2485                 if (do_break)
2486                         break;
2487         }
2488 }
2489
2490 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2491 {
2492         int i;
2493
2494         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2495                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2496
2497                 if (!mlxsw_sp_vr_is_used(vr))
2498                         continue;
2499                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2500         }
2501 }
2502
2503 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2504 {
2505         int err;
2506
2507         if (mlxsw_sp->router.aborted)
2508                 return;
2509         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2510         mlxsw_sp_router_fib_flush(mlxsw_sp);
2511         mlxsw_sp->router.aborted = true;
2512         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2513         if (err)
2514                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2515 }
2516
2517 struct mlxsw_sp_fib_event_work {
2518         struct work_struct work;
2519         union {
2520                 struct fib_entry_notifier_info fen_info;
2521                 struct fib_rule_notifier_info fr_info;
2522                 struct fib_nh_notifier_info fnh_info;
2523         };
2524         struct mlxsw_sp *mlxsw_sp;
2525         unsigned long event;
2526 };
2527
2528 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2529 {
2530         struct mlxsw_sp_fib_event_work *fib_work =
2531                 container_of(work, struct mlxsw_sp_fib_event_work, work);
2532         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2533         struct fib_rule *rule;
2534         bool replace, append;
2535         int err;
2536
2537         /* Protect internal structures from changes */
2538         rtnl_lock();
2539         switch (fib_work->event) {
2540         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2541         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2542         case FIB_EVENT_ENTRY_ADD:
2543                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2544                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2545                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2546                                                replace, append);
2547                 if (err)
2548                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2549                 fib_info_put(fib_work->fen_info.fi);
2550                 break;
2551         case FIB_EVENT_ENTRY_DEL:
2552                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2553                 fib_info_put(fib_work->fen_info.fi);
2554                 break;
2555         case FIB_EVENT_RULE_ADD: /* fall through */
2556         case FIB_EVENT_RULE_DEL:
2557                 rule = fib_work->fr_info.rule;
2558                 if (!fib4_rule_default(rule) && !rule->l3mdev)
2559                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
2560                 fib_rule_put(rule);
2561                 break;
2562         case FIB_EVENT_NH_ADD: /* fall through */
2563         case FIB_EVENT_NH_DEL:
2564                 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2565                                        fib_work->fnh_info.fib_nh);
2566                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2567                 break;
2568         }
2569         rtnl_unlock();
2570         kfree(fib_work);
2571 }
2572
2573 /* Called with rcu_read_lock() */
2574 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2575                                      unsigned long event, void *ptr)
2576 {
2577         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2578         struct mlxsw_sp_fib_event_work *fib_work;
2579         struct fib_notifier_info *info = ptr;
2580
2581         if (!net_eq(info->net, &init_net))
2582                 return NOTIFY_DONE;
2583
2584         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2585         if (WARN_ON(!fib_work))
2586                 return NOTIFY_BAD;
2587
2588         INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2589         fib_work->mlxsw_sp = mlxsw_sp;
2590         fib_work->event = event;
2591
2592         switch (event) {
2593         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2594         case FIB_EVENT_ENTRY_APPEND: /* fall through */
2595         case FIB_EVENT_ENTRY_ADD: /* fall through */
2596         case FIB_EVENT_ENTRY_DEL:
2597                 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2598                 /* Take referece on fib_info to prevent it from being
2599                  * freed while work is queued. Release it afterwards.
2600                  */
2601                 fib_info_hold(fib_work->fen_info.fi);
2602                 break;
2603         case FIB_EVENT_RULE_ADD: /* fall through */
2604         case FIB_EVENT_RULE_DEL:
2605                 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2606                 fib_rule_get(fib_work->fr_info.rule);
2607                 break;
2608         case FIB_EVENT_NH_ADD: /* fall through */
2609         case FIB_EVENT_NH_DEL:
2610                 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2611                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2612                 break;
2613         }
2614
2615         mlxsw_core_schedule_work(&fib_work->work);
2616
2617         return NOTIFY_DONE;
2618 }
2619
2620 static struct mlxsw_sp_rif *
2621 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2622                          const struct net_device *dev)
2623 {
2624         int i;
2625
2626         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2627                 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2628                         return mlxsw_sp->rifs[i];
2629
2630         return NULL;
2631 }
2632
2633 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2634 {
2635         char ritr_pl[MLXSW_REG_RITR_LEN];
2636         int err;
2637
2638         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2639         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2640         if (WARN_ON_ONCE(err))
2641                 return err;
2642
2643         mlxsw_reg_ritr_enable_set(ritr_pl, false);
2644         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2645 }
2646
2647 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2648                                           struct mlxsw_sp_rif *rif)
2649 {
2650         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2651         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2652         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2653 }
2654
2655 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2656                                        const struct in_device *in_dev,
2657                                        unsigned long event)
2658 {
2659         switch (event) {
2660         case NETDEV_UP:
2661                 if (!rif)
2662                         return true;
2663                 return false;
2664         case NETDEV_DOWN:
2665                 if (rif && !in_dev->ifa_list &&
2666                     !netif_is_l3_slave(rif->dev))
2667                         return true;
2668                 /* It is possible we already removed the RIF ourselves
2669                  * if it was assigned to a netdev that is now a bridge
2670                  * or LAG slave.
2671                  */
2672                 return false;
2673         }
2674
2675         return false;
2676 }
2677
2678 #define MLXSW_SP_INVALID_INDEX_RIF 0xffff
2679 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2680 {
2681         int i;
2682
2683         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2684                 if (!mlxsw_sp->rifs[i])
2685                         return i;
2686
2687         return MLXSW_SP_INVALID_INDEX_RIF;
2688 }
2689
2690 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2691                                            bool *p_lagged, u16 *p_system_port)
2692 {
2693         u8 local_port = mlxsw_sp_vport->local_port;
2694
2695         *p_lagged = mlxsw_sp_vport->lagged;
2696         *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2697 }
2698
2699 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2700                                     u16 vr_id, struct net_device *l3_dev,
2701                                     u16 rif_index, bool create)
2702 {
2703         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2704         bool lagged = mlxsw_sp_vport->lagged;
2705         char ritr_pl[MLXSW_REG_RITR_LEN];
2706         u16 system_port;
2707
2708         mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif_index,
2709                             vr_id, l3_dev->mtu, l3_dev->dev_addr);
2710
2711         mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2712         mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2713                                   mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2714
2715         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2716 }
2717
2718 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2719
2720 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif_index)
2721 {
2722         return MLXSW_SP_RFID_BASE + rif_index;
2723 }
2724
2725 static struct mlxsw_sp_fid *
2726 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2727 {
2728         struct mlxsw_sp_fid *f;
2729
2730         f = kzalloc(sizeof(*f), GFP_KERNEL);
2731         if (!f)
2732                 return NULL;
2733
2734         f->leave = mlxsw_sp_vport_rif_sp_leave;
2735         f->ref_count = 0;
2736         f->dev = l3_dev;
2737         f->fid = fid;
2738
2739         return f;
2740 }
2741
2742 static struct mlxsw_sp_rif *
2743 mlxsw_sp_rif_alloc(u16 rif_index, u16 vr_id, struct net_device *l3_dev,
2744                    struct mlxsw_sp_fid *f)
2745 {
2746         struct mlxsw_sp_rif *rif;
2747
2748         rif = kzalloc(sizeof(*rif), GFP_KERNEL);
2749         if (!rif)
2750                 return NULL;
2751
2752         INIT_LIST_HEAD(&rif->nexthop_list);
2753         INIT_LIST_HEAD(&rif->neigh_list);
2754         ether_addr_copy(rif->addr, l3_dev->dev_addr);
2755         rif->mtu = l3_dev->mtu;
2756         rif->vr_id = vr_id;
2757         rif->dev = l3_dev;
2758         rif->rif_index = rif_index;
2759         rif->f = f;
2760
2761         return rif;
2762 }
2763
2764 static struct mlxsw_sp_rif *
2765 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2766                              struct net_device *l3_dev)
2767 {
2768         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2769         u32 tb_id = l3mdev_fib_table(l3_dev);
2770         struct mlxsw_sp_vr *vr;
2771         struct mlxsw_sp_fid *f;
2772         struct mlxsw_sp_rif *rif;
2773         u16 fid, rif_index;
2774         int err;
2775
2776         rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
2777         if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
2778                 return ERR_PTR(-ERANGE);
2779
2780         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
2781         if (IS_ERR(vr))
2782                 return ERR_CAST(vr);
2783
2784         err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev,
2785                                        rif_index, true);
2786         if (err)
2787                 goto err_vport_rif_sp_op;
2788
2789         fid = mlxsw_sp_rif_sp_to_fid(rif_index);
2790         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2791         if (err)
2792                 goto err_rif_fdb_op;
2793
2794         f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2795         if (!f) {
2796                 err = -ENOMEM;
2797                 goto err_rfid_alloc;
2798         }
2799
2800         rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
2801         if (!rif) {
2802                 err = -ENOMEM;
2803                 goto err_rif_alloc;
2804         }
2805
2806         f->rif = rif;
2807         mlxsw_sp->rifs[rif_index] = rif;
2808         vr->rif_count++;
2809
2810         return rif;
2811
2812 err_rif_alloc:
2813         kfree(f);
2814 err_rfid_alloc:
2815         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2816 err_rif_fdb_op:
2817         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2818                                  false);
2819 err_vport_rif_sp_op:
2820         mlxsw_sp_vr_put(vr);
2821         return ERR_PTR(err);
2822 }
2823
2824 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2825                                           struct mlxsw_sp_rif *rif)
2826 {
2827         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2828         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
2829         struct net_device *l3_dev = rif->dev;
2830         struct mlxsw_sp_fid *f = rif->f;
2831         u16 rif_index = rif->rif_index;
2832         u16 fid = f->fid;
2833
2834         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
2835
2836         vr->rif_count--;
2837         mlxsw_sp->rifs[rif_index] = NULL;
2838         f->rif = NULL;
2839
2840         kfree(rif);
2841
2842         kfree(f);
2843
2844         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2845
2846         mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2847                                  false);
2848         mlxsw_sp_vr_put(vr);
2849 }
2850
2851 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2852                                       struct net_device *l3_dev)
2853 {
2854         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2855         struct mlxsw_sp_rif *rif;
2856
2857         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2858         if (!rif) {
2859                 rif = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2860                 if (IS_ERR(rif))
2861                         return PTR_ERR(rif);
2862         }
2863
2864         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, rif->f);
2865         rif->f->ref_count++;
2866
2867         netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", rif->f->fid);
2868
2869         return 0;
2870 }
2871
2872 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2873 {
2874         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2875
2876         netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2877
2878         mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2879         if (--f->ref_count == 0)
2880                 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->rif);
2881 }
2882
2883 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2884                                          struct net_device *port_dev,
2885                                          unsigned long event, u16 vid)
2886 {
2887         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2888         struct mlxsw_sp_port *mlxsw_sp_vport;
2889
2890         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2891         if (WARN_ON(!mlxsw_sp_vport))
2892                 return -EINVAL;
2893
2894         switch (event) {
2895         case NETDEV_UP:
2896                 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2897         case NETDEV_DOWN:
2898                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2899                 break;
2900         }
2901
2902         return 0;
2903 }
2904
2905 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2906                                         unsigned long event)
2907 {
2908         if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2909                 return 0;
2910
2911         return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2912 }
2913
2914 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2915                                          struct net_device *lag_dev,
2916                                          unsigned long event, u16 vid)
2917 {
2918         struct net_device *port_dev;
2919         struct list_head *iter;
2920         int err;
2921
2922         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2923                 if (mlxsw_sp_port_dev_check(port_dev)) {
2924                         err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2925                                                             event, vid);
2926                         if (err)
2927                                 return err;
2928                 }
2929         }
2930
2931         return 0;
2932 }
2933
2934 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2935                                        unsigned long event)
2936 {
2937         if (netif_is_bridge_port(lag_dev))
2938                 return 0;
2939
2940         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2941 }
2942
2943 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2944                                                     struct net_device *l3_dev)
2945 {
2946         u16 fid;
2947
2948         if (is_vlan_dev(l3_dev))
2949                 fid = vlan_dev_vlan_id(l3_dev);
2950         else if (mlxsw_sp->master_bridge.dev == l3_dev)
2951                 fid = 1;
2952         else
2953                 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2954
2955         return mlxsw_sp_fid_find(mlxsw_sp, fid);
2956 }
2957
2958 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2959 {
2960         return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2961                MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2962 }
2963
2964 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2965 {
2966         return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2967 }
2968
2969 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2970                                           bool set)
2971 {
2972         enum mlxsw_flood_table_type table_type;
2973         char *sftr_pl;
2974         u16 index;
2975         int err;
2976
2977         sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
2978         if (!sftr_pl)
2979                 return -ENOMEM;
2980
2981         table_type = mlxsw_sp_flood_table_type_get(fid);
2982         index = mlxsw_sp_flood_table_index_get(fid);
2983         mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
2984                             1, MLXSW_PORT_ROUTER_PORT, set);
2985         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
2986
2987         kfree(sftr_pl);
2988         return err;
2989 }
2990
2991 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
2992 {
2993         if (mlxsw_sp_fid_is_vfid(fid))
2994                 return MLXSW_REG_RITR_FID_IF;
2995         else
2996                 return MLXSW_REG_RITR_VLAN_IF;
2997 }
2998
2999 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
3000                                   struct net_device *l3_dev,
3001                                   u16 fid, u16 rif,
3002                                   bool create)
3003 {
3004         enum mlxsw_reg_ritr_if_type rif_type;
3005         char ritr_pl[MLXSW_REG_RITR_LEN];
3006
3007         rif_type = mlxsw_sp_rif_type_get(fid);
3008         mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3009                             l3_dev->dev_addr);
3010         mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3011
3012         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3013 }
3014
3015 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3016                                       struct net_device *l3_dev,
3017                                       struct mlxsw_sp_fid *f)
3018 {
3019         u32 tb_id = l3mdev_fib_table(l3_dev);
3020         struct mlxsw_sp_rif *rif;
3021         struct mlxsw_sp_vr *vr;
3022         u16 rif_index;
3023         int err;
3024
3025         rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
3026         if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
3027                 return -ERANGE;
3028
3029         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3030         if (IS_ERR(vr))
3031                 return PTR_ERR(vr);
3032
3033         err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3034         if (err)
3035                 goto err_port_flood_set;
3036
3037         err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid,
3038                                      rif_index, true);
3039         if (err)
3040                 goto err_rif_bridge_op;
3041
3042         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3043         if (err)
3044                 goto err_rif_fdb_op;
3045
3046         rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
3047         if (!rif) {
3048                 err = -ENOMEM;
3049                 goto err_rif_alloc;
3050         }
3051
3052         f->rif = rif;
3053         mlxsw_sp->rifs[rif_index] = rif;
3054         vr->rif_count++;
3055
3056         netdev_dbg(l3_dev, "RIF=%d created\n", rif_index);
3057
3058         return 0;
3059
3060 err_rif_alloc:
3061         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3062 err_rif_fdb_op:
3063         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3064                                false);
3065 err_rif_bridge_op:
3066         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3067 err_port_flood_set:
3068         mlxsw_sp_vr_put(vr);
3069         return err;
3070 }
3071
3072 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3073                                  struct mlxsw_sp_rif *rif)
3074 {
3075         struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
3076         struct net_device *l3_dev = rif->dev;
3077         struct mlxsw_sp_fid *f = rif->f;
3078         u16 rif_index = rif->rif_index;
3079
3080         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3081
3082         vr->rif_count--;
3083         mlxsw_sp->rifs[rif_index] = NULL;
3084         f->rif = NULL;
3085
3086         kfree(rif);
3087
3088         mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3089
3090         mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3091                                false);
3092
3093         mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3094
3095         mlxsw_sp_vr_put(vr);
3096
3097         netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif_index);
3098 }
3099
3100 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3101                                           struct net_device *br_dev,
3102                                           unsigned long event)
3103 {
3104         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3105         struct mlxsw_sp_fid *f;
3106
3107         /* FID can either be an actual FID if the L3 device is the
3108          * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3109          * L3 device is a VLAN-unaware bridge and we get a vFID.
3110          */
3111         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3112         if (WARN_ON(!f))
3113                 return -EINVAL;
3114
3115         switch (event) {
3116         case NETDEV_UP:
3117                 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3118         case NETDEV_DOWN:
3119                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3120                 break;
3121         }
3122
3123         return 0;
3124 }
3125
3126 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3127                                         unsigned long event)
3128 {
3129         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3130         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3131         u16 vid = vlan_dev_vlan_id(vlan_dev);
3132
3133         if (mlxsw_sp_port_dev_check(real_dev))
3134                 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3135                                                      vid);
3136         else if (netif_is_lag_master(real_dev))
3137                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3138                                                      vid);
3139         else if (netif_is_bridge_master(real_dev) &&
3140                  mlxsw_sp->master_bridge.dev == real_dev)
3141                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3142                                                       event);
3143
3144         return 0;
3145 }
3146
3147 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3148                             unsigned long event, void *ptr)
3149 {
3150         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3151         struct net_device *dev = ifa->ifa_dev->dev;
3152         struct mlxsw_sp *mlxsw_sp;
3153         struct mlxsw_sp_rif *rif;
3154         int err = 0;
3155
3156         mlxsw_sp = mlxsw_sp_lower_get(dev);
3157         if (!mlxsw_sp)
3158                 goto out;
3159
3160         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3161         if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3162                 goto out;
3163
3164         if (mlxsw_sp_port_dev_check(dev))
3165                 err = mlxsw_sp_inetaddr_port_event(dev, event);
3166         else if (netif_is_lag_master(dev))
3167                 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3168         else if (netif_is_bridge_master(dev))
3169                 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3170         else if (is_vlan_dev(dev))
3171                 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3172
3173 out:
3174         return notifier_from_errno(err);
3175 }
3176
3177 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3178                              const char *mac, int mtu)
3179 {
3180         char ritr_pl[MLXSW_REG_RITR_LEN];
3181         int err;
3182
3183         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3184         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3185         if (err)
3186                 return err;
3187
3188         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3189         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3190         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3191         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3192 }
3193
3194 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3195 {
3196         struct mlxsw_sp *mlxsw_sp;
3197         struct mlxsw_sp_rif *rif;
3198         int err;
3199
3200         mlxsw_sp = mlxsw_sp_lower_get(dev);
3201         if (!mlxsw_sp)
3202                 return 0;
3203
3204         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3205         if (!rif)
3206                 return 0;
3207
3208         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, false);
3209         if (err)
3210                 return err;
3211
3212         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3213                                 dev->mtu);
3214         if (err)
3215                 goto err_rif_edit;
3216
3217         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, rif->f->fid, true);
3218         if (err)
3219                 goto err_rif_fdb_op;
3220
3221         ether_addr_copy(rif->addr, dev->dev_addr);
3222         rif->mtu = dev->mtu;
3223
3224         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3225
3226         return 0;
3227
3228 err_rif_fdb_op:
3229         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3230 err_rif_edit:
3231         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, true);
3232         return err;
3233 }
3234
3235 int mlxsw_sp_vport_vrf_join(struct mlxsw_sp_port *mlxsw_sp_vport)
3236 {
3237         struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
3238         struct net_device *dev = mlxsw_sp_vport->dev;
3239
3240         /* In case vPort already has a RIF, then we need to drop it.
3241          * A new one will be created using the VRF's VR.
3242          */
3243         if (f && f->rif)
3244                 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3245
3246         return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, dev);
3247 }
3248
3249 void mlxsw_sp_vport_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
3250 {
3251         mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3252 }
3253
3254 int mlxsw_sp_port_vrf_join(struct mlxsw_sp_port *mlxsw_sp_port)
3255 {
3256         struct mlxsw_sp_port *mlxsw_sp_vport;
3257
3258         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3259         if (WARN_ON(!mlxsw_sp_vport))
3260                 return -EINVAL;
3261
3262         return mlxsw_sp_vport_vrf_join(mlxsw_sp_vport);
3263 }
3264
3265 void mlxsw_sp_port_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_port)
3266 {
3267         struct mlxsw_sp_port *mlxsw_sp_vport;
3268
3269         mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3270         if (WARN_ON(!mlxsw_sp_vport))
3271                 return;
3272
3273         mlxsw_sp_vport_vrf_leave(mlxsw_sp_vport);
3274 }
3275
3276 int mlxsw_sp_bridge_vrf_join(struct mlxsw_sp *mlxsw_sp,
3277                              struct net_device *l3_dev)
3278 {
3279         struct mlxsw_sp_fid *f;
3280
3281         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3282         if (WARN_ON(!f))
3283                 return -EINVAL;
3284
3285         if (f->rif)
3286                 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3287
3288         return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3289 }
3290
3291 void mlxsw_sp_bridge_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3292                                struct net_device *l3_dev)
3293 {
3294         struct mlxsw_sp_fid *f;
3295
3296         f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3297         if (WARN_ON(!f))
3298                 return;
3299         mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3300 }
3301
3302 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3303 {
3304         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3305
3306         /* Flush pending FIB notifications and then flush the device's
3307          * table before requesting another dump. The FIB notification
3308          * block is unregistered, so no need to take RTNL.
3309          */
3310         mlxsw_core_flush_owq();
3311         mlxsw_sp_router_fib_flush(mlxsw_sp);
3312 }
3313
3314 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3315 {
3316         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3317         u64 max_rifs;
3318         int err;
3319
3320         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3321                 return -EIO;
3322
3323         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3324         mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3325                                  GFP_KERNEL);
3326         if (!mlxsw_sp->rifs)
3327                 return -ENOMEM;
3328
3329         mlxsw_reg_rgcr_pack(rgcr_pl, true);
3330         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3331         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3332         if (err)
3333                 goto err_rgcr_fail;
3334
3335         return 0;
3336
3337 err_rgcr_fail:
3338         kfree(mlxsw_sp->rifs);
3339         return err;
3340 }
3341
3342 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3343 {
3344         char rgcr_pl[MLXSW_REG_RGCR_LEN];
3345         int i;
3346
3347         mlxsw_reg_rgcr_pack(rgcr_pl, false);
3348         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3349
3350         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3351                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3352
3353         kfree(mlxsw_sp->rifs);
3354 }
3355
3356 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3357 {
3358         int err;
3359
3360         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3361         err = __mlxsw_sp_router_init(mlxsw_sp);
3362         if (err)
3363                 return err;
3364
3365         err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3366                               &mlxsw_sp_nexthop_ht_params);
3367         if (err)
3368                 goto err_nexthop_ht_init;
3369
3370         err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3371                               &mlxsw_sp_nexthop_group_ht_params);
3372         if (err)
3373                 goto err_nexthop_group_ht_init;
3374
3375         mlxsw_sp_lpm_init(mlxsw_sp);
3376         err = mlxsw_sp_vrs_init(mlxsw_sp);
3377         if (err)
3378                 goto err_vrs_init;
3379
3380         err = mlxsw_sp_neigh_init(mlxsw_sp);
3381         if (err)
3382                 goto err_neigh_init;
3383
3384         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3385         err = register_fib_notifier(&mlxsw_sp->fib_nb,
3386                                     mlxsw_sp_router_fib_dump_flush);
3387         if (err)
3388                 goto err_register_fib_notifier;
3389
3390         return 0;
3391
3392 err_register_fib_notifier:
3393         mlxsw_sp_neigh_fini(mlxsw_sp);
3394 err_neigh_init:
3395         mlxsw_sp_vrs_fini(mlxsw_sp);
3396 err_vrs_init:
3397         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3398 err_nexthop_group_ht_init:
3399         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3400 err_nexthop_ht_init:
3401         __mlxsw_sp_router_fini(mlxsw_sp);
3402         return err;
3403 }
3404
3405 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3406 {
3407         unregister_fib_notifier(&mlxsw_sp->fib_nb);
3408         mlxsw_sp_neigh_fini(mlxsw_sp);
3409         mlxsw_sp_vrs_fini(mlxsw_sp);
3410         rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3411         rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3412         __mlxsw_sp_router_fini(mlxsw_sp);
3413 }