2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <linux/inetdevice.h>
44 #include <linux/netdevice.h>
45 #include <net/netevent.h>
46 #include <net/neighbour.h>
48 #include <net/ip_fib.h>
49 #include <net/fib_rules.h>
50 #include <net/l3mdev.h>
57 struct list_head nexthop_list;
58 struct list_head neigh_list;
59 struct net_device *dev;
60 struct mlxsw_sp_fid *f;
61 unsigned char addr[ETH_ALEN];
67 static struct mlxsw_sp_rif *
68 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
69 const struct net_device *dev);
71 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
72 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
75 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
76 struct mlxsw_sp_prefix_usage *prefix_usage2)
80 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
81 if (!test_bit(prefix, prefix_usage2->b))
88 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
89 struct mlxsw_sp_prefix_usage *prefix_usage2)
91 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
95 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
97 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
99 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
103 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
104 struct mlxsw_sp_prefix_usage *prefix_usage2)
106 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
110 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
111 unsigned char prefix_len)
113 set_bit(prefix_len, prefix_usage->b);
117 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
118 unsigned char prefix_len)
120 clear_bit(prefix_len, prefix_usage->b);
123 struct mlxsw_sp_fib_key {
124 unsigned char addr[sizeof(struct in6_addr)];
125 unsigned char prefix_len;
128 enum mlxsw_sp_fib_entry_type {
129 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
130 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
131 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
134 struct mlxsw_sp_nexthop_group;
136 struct mlxsw_sp_fib_node {
137 struct list_head entry_list;
138 struct list_head list;
139 struct rhash_head ht_node;
140 struct mlxsw_sp_fib *fib;
141 struct mlxsw_sp_fib_key key;
144 struct mlxsw_sp_fib_entry_params {
151 struct mlxsw_sp_fib_entry {
152 struct list_head list;
153 struct mlxsw_sp_fib_node *fib_node;
154 enum mlxsw_sp_fib_entry_type type;
155 struct list_head nexthop_group_node;
156 struct mlxsw_sp_nexthop_group *nh_group;
157 struct mlxsw_sp_fib_entry_params params;
161 struct mlxsw_sp_fib {
162 struct rhashtable ht;
163 struct list_head node_list;
164 struct mlxsw_sp_vr *vr;
165 struct mlxsw_sp_lpm_tree *lpm_tree;
166 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
167 struct mlxsw_sp_prefix_usage prefix_usage;
168 enum mlxsw_sp_l3proto proto;
171 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
173 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
174 enum mlxsw_sp_l3proto proto)
176 struct mlxsw_sp_fib *fib;
179 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
181 return ERR_PTR(-ENOMEM);
182 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
184 goto err_rhashtable_init;
185 INIT_LIST_HEAD(&fib->node_list);
195 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
197 WARN_ON(!list_empty(&fib->node_list));
198 WARN_ON(fib->lpm_tree);
199 rhashtable_destroy(&fib->ht);
203 static struct mlxsw_sp_lpm_tree *
204 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
206 static struct mlxsw_sp_lpm_tree *lpm_tree;
209 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
210 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
211 if (lpm_tree->ref_count == 0)
217 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
218 struct mlxsw_sp_lpm_tree *lpm_tree)
220 char ralta_pl[MLXSW_REG_RALTA_LEN];
222 mlxsw_reg_ralta_pack(ralta_pl, true,
223 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
225 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
228 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
229 struct mlxsw_sp_lpm_tree *lpm_tree)
231 char ralta_pl[MLXSW_REG_RALTA_LEN];
233 mlxsw_reg_ralta_pack(ralta_pl, false,
234 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
236 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
240 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
241 struct mlxsw_sp_prefix_usage *prefix_usage,
242 struct mlxsw_sp_lpm_tree *lpm_tree)
244 char ralst_pl[MLXSW_REG_RALST_LEN];
247 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
249 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
252 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
253 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
256 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
257 MLXSW_REG_RALST_BIN_NO_CHILD);
258 last_prefix = prefix;
260 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
263 static struct mlxsw_sp_lpm_tree *
264 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
265 struct mlxsw_sp_prefix_usage *prefix_usage,
266 enum mlxsw_sp_l3proto proto)
268 struct mlxsw_sp_lpm_tree *lpm_tree;
271 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
273 return ERR_PTR(-EBUSY);
274 lpm_tree->proto = proto;
275 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
279 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
282 goto err_left_struct_set;
283 memcpy(&lpm_tree->prefix_usage, prefix_usage,
284 sizeof(lpm_tree->prefix_usage));
288 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
292 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
293 struct mlxsw_sp_lpm_tree *lpm_tree)
295 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
298 static struct mlxsw_sp_lpm_tree *
299 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
300 struct mlxsw_sp_prefix_usage *prefix_usage,
301 enum mlxsw_sp_l3proto proto)
303 struct mlxsw_sp_lpm_tree *lpm_tree;
306 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
307 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
308 if (lpm_tree->ref_count != 0 &&
309 lpm_tree->proto == proto &&
310 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
314 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
316 if (IS_ERR(lpm_tree))
320 lpm_tree->ref_count++;
324 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
325 struct mlxsw_sp_lpm_tree *lpm_tree)
327 if (--lpm_tree->ref_count == 0)
328 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
332 #define MLXSW_SP_LPM_TREE_MIN 2 /* trees 0 and 1 are reserved */
334 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
336 struct mlxsw_sp_lpm_tree *lpm_tree;
340 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
343 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
344 mlxsw_sp->router.lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
345 mlxsw_sp->router.lpm.trees = kcalloc(mlxsw_sp->router.lpm.tree_count,
346 sizeof(struct mlxsw_sp_lpm_tree),
348 if (!mlxsw_sp->router.lpm.trees)
351 for (i = 0; i < mlxsw_sp->router.lpm.tree_count; i++) {
352 lpm_tree = &mlxsw_sp->router.lpm.trees[i];
353 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
359 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
361 kfree(mlxsw_sp->router.lpm.trees);
364 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
369 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
371 struct mlxsw_sp_vr *vr;
374 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
375 vr = &mlxsw_sp->router.vrs[i];
376 if (!mlxsw_sp_vr_is_used(vr))
382 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
383 const struct mlxsw_sp_fib *fib)
385 char raltb_pl[MLXSW_REG_RALTB_LEN];
387 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
388 (enum mlxsw_reg_ralxx_protocol) fib->proto,
390 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
393 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
394 const struct mlxsw_sp_fib *fib)
396 char raltb_pl[MLXSW_REG_RALTB_LEN];
398 /* Bind to tree 0 which is default */
399 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
400 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
404 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
406 /* For our purpose, squash main and local table into one */
407 if (tb_id == RT_TABLE_LOCAL)
408 tb_id = RT_TABLE_MAIN;
412 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
415 struct mlxsw_sp_vr *vr;
418 tb_id = mlxsw_sp_fix_tb_id(tb_id);
420 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
421 vr = &mlxsw_sp->router.vrs[i];
422 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
428 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
429 enum mlxsw_sp_l3proto proto)
432 case MLXSW_SP_L3_PROTO_IPV4:
434 case MLXSW_SP_L3_PROTO_IPV6:
440 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
443 struct mlxsw_sp_vr *vr;
445 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
447 return ERR_PTR(-EBUSY);
448 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
449 if (IS_ERR(vr->fib4))
450 return ERR_CAST(vr->fib4);
455 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
457 mlxsw_sp_fib_destroy(vr->fib4);
462 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
463 struct mlxsw_sp_prefix_usage *req_prefix_usage)
465 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
466 struct mlxsw_sp_lpm_tree *new_tree;
469 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
472 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
474 if (IS_ERR(new_tree)) {
475 /* We failed to get a tree according to the required
476 * prefix usage. However, the current tree might be still good
477 * for us if our requirement is subset of the prefixes used
480 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
481 &lpm_tree->prefix_usage))
483 return PTR_ERR(new_tree);
486 /* Prevent packet loss by overwriting existing binding */
487 fib->lpm_tree = new_tree;
488 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
491 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
496 fib->lpm_tree = lpm_tree;
497 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
501 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
503 struct mlxsw_sp_vr *vr;
505 tb_id = mlxsw_sp_fix_tb_id(tb_id);
506 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
508 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
512 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
514 if (!vr->rif_count && list_empty(&vr->fib4->node_list))
515 mlxsw_sp_vr_destroy(vr);
518 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
520 struct mlxsw_sp_vr *vr;
524 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
527 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
528 mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
530 if (!mlxsw_sp->router.vrs)
533 for (i = 0; i < max_vrs; i++) {
534 vr = &mlxsw_sp->router.vrs[i];
541 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
543 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
545 /* At this stage we're guaranteed not to have new incoming
546 * FIB notifications and the work queue is free from FIBs
547 * sitting on top of mlxsw netdevs. However, we can still
548 * have other FIBs queued. Flush the queue before flushing
549 * the device's tables. No need for locks, as we're the only
552 mlxsw_core_flush_owq();
553 mlxsw_sp_router_fib_flush(mlxsw_sp);
554 kfree(mlxsw_sp->router.vrs);
557 struct mlxsw_sp_neigh_key {
561 struct mlxsw_sp_neigh_entry {
562 struct list_head rif_list_node;
563 struct rhash_head ht_node;
564 struct mlxsw_sp_neigh_key key;
567 unsigned char ha[ETH_ALEN];
568 struct list_head nexthop_list; /* list of nexthops using
571 struct list_head nexthop_neighs_list_node;
574 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
575 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
576 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
577 .key_len = sizeof(struct mlxsw_sp_neigh_key),
580 static struct mlxsw_sp_neigh_entry *
581 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
584 struct mlxsw_sp_neigh_entry *neigh_entry;
586 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
590 neigh_entry->key.n = n;
591 neigh_entry->rif = rif;
592 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
597 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
603 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
604 struct mlxsw_sp_neigh_entry *neigh_entry)
606 return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
607 &neigh_entry->ht_node,
608 mlxsw_sp_neigh_ht_params);
612 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
613 struct mlxsw_sp_neigh_entry *neigh_entry)
615 rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
616 &neigh_entry->ht_node,
617 mlxsw_sp_neigh_ht_params);
620 static struct mlxsw_sp_neigh_entry *
621 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
623 struct mlxsw_sp_neigh_entry *neigh_entry;
624 struct mlxsw_sp_rif *rif;
627 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
629 return ERR_PTR(-EINVAL);
631 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
633 return ERR_PTR(-ENOMEM);
635 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
637 goto err_neigh_entry_insert;
639 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
643 err_neigh_entry_insert:
644 mlxsw_sp_neigh_entry_free(neigh_entry);
649 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
650 struct mlxsw_sp_neigh_entry *neigh_entry)
652 list_del(&neigh_entry->rif_list_node);
653 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
654 mlxsw_sp_neigh_entry_free(neigh_entry);
657 static struct mlxsw_sp_neigh_entry *
658 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
660 struct mlxsw_sp_neigh_key key;
663 return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
664 &key, mlxsw_sp_neigh_ht_params);
668 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
670 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
672 mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
675 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
679 struct net_device *dev;
685 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
687 if (!mlxsw_sp->rifs[rif]) {
688 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
693 dev = mlxsw_sp->rifs[rif]->dev;
694 n = neigh_lookup(&arp_tbl, &dipn, dev);
696 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
701 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
702 neigh_event_send(n, NULL);
706 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
713 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
715 /* Hardware starts counting at 0, so add 1. */
718 /* Each record consists of several neighbour entries. */
719 for (i = 0; i < num_entries; i++) {
722 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
723 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
729 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
730 char *rauhtd_pl, int rec_index)
732 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
733 case MLXSW_REG_RAUHTD_TYPE_IPV4:
734 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
737 case MLXSW_REG_RAUHTD_TYPE_IPV6:
743 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
745 u8 num_rec, last_rec_index, num_entries;
747 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
748 last_rec_index = num_rec - 1;
750 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
752 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
753 MLXSW_REG_RAUHTD_TYPE_IPV6)
756 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
758 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
763 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
769 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
773 /* Make sure the neighbour's netdev isn't removed in the
778 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
779 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
782 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
785 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
786 for (i = 0; i < num_rec; i++)
787 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
789 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
796 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
798 struct mlxsw_sp_neigh_entry *neigh_entry;
800 /* Take RTNL mutex here to prevent lists from changes */
802 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
803 nexthop_neighs_list_node)
804 /* If this neigh have nexthops, make the kernel think this neigh
805 * is active regardless of the traffic.
807 neigh_event_send(neigh_entry->key.n, NULL);
812 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
814 unsigned long interval = mlxsw_sp->router.neighs_update.interval;
816 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
817 msecs_to_jiffies(interval));
820 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
822 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
823 router.neighs_update.dw.work);
826 err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
828 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
830 mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
832 mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
835 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
837 struct mlxsw_sp_neigh_entry *neigh_entry;
838 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
839 router.nexthop_probe_dw.work);
841 /* Iterate over nexthop neighbours, find those who are unresolved and
842 * send arp on them. This solves the chicken-egg problem when
843 * the nexthop wouldn't get offloaded until the neighbor is resolved
844 * but it wouldn't get resolved ever in case traffic is flowing in HW
845 * using different nexthop.
847 * Take RTNL mutex here to prevent lists from changes.
850 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
851 nexthop_neighs_list_node)
852 if (!neigh_entry->connected)
853 neigh_event_send(neigh_entry->key.n, NULL);
856 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
857 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
861 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
862 struct mlxsw_sp_neigh_entry *neigh_entry,
865 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
867 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
868 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
872 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
873 struct mlxsw_sp_neigh_entry *neigh_entry,
874 enum mlxsw_reg_rauht_op op)
876 struct neighbour *n = neigh_entry->key.n;
877 u32 dip = ntohl(*((__be32 *) n->primary_key));
878 char rauht_pl[MLXSW_REG_RAUHT_LEN];
880 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
882 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
886 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
887 struct mlxsw_sp_neigh_entry *neigh_entry,
890 if (!adding && !neigh_entry->connected)
892 neigh_entry->connected = adding;
893 if (neigh_entry->key.n->tbl == &arp_tbl)
894 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
895 mlxsw_sp_rauht_op(adding));
900 struct mlxsw_sp_neigh_event_work {
901 struct work_struct work;
902 struct mlxsw_sp *mlxsw_sp;
906 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
908 struct mlxsw_sp_neigh_event_work *neigh_work =
909 container_of(work, struct mlxsw_sp_neigh_event_work, work);
910 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
911 struct mlxsw_sp_neigh_entry *neigh_entry;
912 struct neighbour *n = neigh_work->n;
913 unsigned char ha[ETH_ALEN];
914 bool entry_connected;
917 /* If these parameters are changed after we release the lock,
918 * then we are guaranteed to receive another event letting us
921 read_lock_bh(&n->lock);
922 memcpy(ha, n->ha, ETH_ALEN);
923 nud_state = n->nud_state;
925 read_unlock_bh(&n->lock);
928 entry_connected = nud_state & NUD_VALID && !dead;
929 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
930 if (!entry_connected && !neigh_entry)
933 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
934 if (IS_ERR(neigh_entry))
938 memcpy(neigh_entry->ha, ha, ETH_ALEN);
939 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
940 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
942 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
943 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
951 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
952 unsigned long event, void *ptr)
954 struct mlxsw_sp_neigh_event_work *neigh_work;
955 struct mlxsw_sp_port *mlxsw_sp_port;
956 struct mlxsw_sp *mlxsw_sp;
957 unsigned long interval;
958 struct neigh_parms *p;
962 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
965 /* We don't care about changes in the default table. */
966 if (!p->dev || p->tbl != &arp_tbl)
969 /* We are in atomic context and can't take RTNL mutex,
970 * so use RCU variant to walk the device chain.
972 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
976 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
977 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
978 mlxsw_sp->router.neighs_update.interval = interval;
980 mlxsw_sp_port_dev_put(mlxsw_sp_port);
982 case NETEVENT_NEIGH_UPDATE:
985 if (n->tbl != &arp_tbl)
988 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
992 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
994 mlxsw_sp_port_dev_put(mlxsw_sp_port);
998 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
999 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1002 /* Take a reference to ensure the neighbour won't be
1003 * destructed until we drop the reference in delayed
1007 mlxsw_core_schedule_work(&neigh_work->work);
1008 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1015 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1019 err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1020 &mlxsw_sp_neigh_ht_params);
1024 /* Initialize the polling interval according to the default
1027 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1029 /* Create the delayed works for the activity_update */
1030 INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1031 mlxsw_sp_router_neighs_update_work);
1032 INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1033 mlxsw_sp_router_probe_unresolved_nexthops);
1034 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1035 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1039 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1041 cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1042 cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1043 rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1046 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1047 const struct mlxsw_sp_rif *rif)
1049 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1051 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1052 rif->rif_index, rif->addr);
1053 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1056 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1057 struct mlxsw_sp_rif *rif)
1059 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1061 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1062 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1064 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1067 struct mlxsw_sp_nexthop_key {
1068 struct fib_nh *fib_nh;
1071 struct mlxsw_sp_nexthop {
1072 struct list_head neigh_list_node; /* member of neigh entry list */
1073 struct list_head rif_list_node;
1074 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1077 struct rhash_head ht_node;
1078 struct mlxsw_sp_nexthop_key key;
1079 struct mlxsw_sp_rif *rif;
1080 u8 should_offload:1, /* set indicates this neigh is connected and
1081 * should be put to KVD linear area of this group.
1083 offloaded:1, /* set in case the neigh is actually put into
1084 * KVD linear area of this group.
1086 update:1; /* set indicates that MAC of this neigh should be
1089 struct mlxsw_sp_neigh_entry *neigh_entry;
1092 struct mlxsw_sp_nexthop_group_key {
1093 struct fib_info *fi;
1096 struct mlxsw_sp_nexthop_group {
1097 struct rhash_head ht_node;
1098 struct list_head fib_list; /* list of fib entries that use this group */
1099 struct mlxsw_sp_nexthop_group_key key;
1100 u8 adj_index_valid:1,
1101 gateway:1; /* routes using the group use a gateway */
1105 struct mlxsw_sp_nexthop nexthops[0];
1106 #define nh_rif nexthops[0].rif
1109 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1110 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1111 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1112 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1115 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1116 struct mlxsw_sp_nexthop_group *nh_grp)
1118 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1120 mlxsw_sp_nexthop_group_ht_params);
1123 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1124 struct mlxsw_sp_nexthop_group *nh_grp)
1126 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1128 mlxsw_sp_nexthop_group_ht_params);
1131 static struct mlxsw_sp_nexthop_group *
1132 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1133 struct mlxsw_sp_nexthop_group_key key)
1135 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1136 mlxsw_sp_nexthop_group_ht_params);
1139 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1140 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1141 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1142 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1145 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1146 struct mlxsw_sp_nexthop *nh)
1148 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1149 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1152 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1153 struct mlxsw_sp_nexthop *nh)
1155 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1156 mlxsw_sp_nexthop_ht_params);
1159 static struct mlxsw_sp_nexthop *
1160 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1161 struct mlxsw_sp_nexthop_key key)
1163 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1164 mlxsw_sp_nexthop_ht_params);
1167 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1168 const struct mlxsw_sp_fib *fib,
1169 u32 adj_index, u16 ecmp_size,
1173 char raleu_pl[MLXSW_REG_RALEU_LEN];
1175 mlxsw_reg_raleu_pack(raleu_pl,
1176 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1177 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1179 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1182 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1183 struct mlxsw_sp_nexthop_group *nh_grp,
1184 u32 old_adj_index, u16 old_ecmp_size)
1186 struct mlxsw_sp_fib_entry *fib_entry;
1187 struct mlxsw_sp_fib *fib = NULL;
1190 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1191 if (fib == fib_entry->fib_node->fib)
1193 fib = fib_entry->fib_node->fib;
1194 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1205 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1206 struct mlxsw_sp_nexthop *nh)
1208 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1209 char ratr_pl[MLXSW_REG_RATR_LEN];
1211 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1212 true, adj_index, neigh_entry->rif);
1213 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1214 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1218 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1219 struct mlxsw_sp_nexthop_group *nh_grp,
1222 u32 adj_index = nh_grp->adj_index; /* base */
1223 struct mlxsw_sp_nexthop *nh;
1227 for (i = 0; i < nh_grp->count; i++) {
1228 nh = &nh_grp->nexthops[i];
1230 if (!nh->should_offload) {
1235 if (nh->update || reallocate) {
1236 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1248 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1249 struct mlxsw_sp_fib_entry *fib_entry);
1252 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1253 struct mlxsw_sp_nexthop_group *nh_grp)
1255 struct mlxsw_sp_fib_entry *fib_entry;
1258 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1259 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1267 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1268 struct mlxsw_sp_nexthop_group *nh_grp)
1270 struct mlxsw_sp_nexthop *nh;
1271 bool offload_change = false;
1274 bool old_adj_index_valid;
1281 if (!nh_grp->gateway) {
1282 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1286 for (i = 0; i < nh_grp->count; i++) {
1287 nh = &nh_grp->nexthops[i];
1289 if (nh->should_offload ^ nh->offloaded) {
1290 offload_change = true;
1291 if (nh->should_offload)
1294 if (nh->should_offload)
1297 if (!offload_change) {
1298 /* Nothing was added or removed, so no need to reallocate. Just
1299 * update MAC on existing adjacency indexes.
1301 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1304 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1310 /* No neigh of this group is connected so we just set
1311 * the trap and let everthing flow through kernel.
1315 ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1317 /* We ran out of KVD linear space, just set the
1318 * trap and let everything flow through kernel.
1320 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1324 old_adj_index_valid = nh_grp->adj_index_valid;
1325 old_adj_index = nh_grp->adj_index;
1326 old_ecmp_size = nh_grp->ecmp_size;
1327 nh_grp->adj_index_valid = 1;
1328 nh_grp->adj_index = adj_index;
1329 nh_grp->ecmp_size = ecmp_size;
1330 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1332 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1336 if (!old_adj_index_valid) {
1337 /* The trap was set for fib entries, so we have to call
1338 * fib entry update to unset it and use adjacency index.
1340 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1342 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1348 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1349 old_adj_index, old_ecmp_size);
1350 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1352 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1358 old_adj_index_valid = nh_grp->adj_index_valid;
1359 nh_grp->adj_index_valid = 0;
1360 for (i = 0; i < nh_grp->count; i++) {
1361 nh = &nh_grp->nexthops[i];
1364 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1366 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1367 if (old_adj_index_valid)
1368 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1371 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1374 if (!removing && !nh->should_offload)
1375 nh->should_offload = 1;
1376 else if (removing && nh->offloaded)
1377 nh->should_offload = 0;
1382 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1383 struct mlxsw_sp_neigh_entry *neigh_entry,
1386 struct mlxsw_sp_nexthop *nh;
1388 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1390 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1391 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1395 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1396 struct mlxsw_sp_rif *rif)
1402 list_add(&nh->rif_list_node, &rif->nexthop_list);
1405 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1410 list_del(&nh->rif_list_node);
1414 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1415 struct mlxsw_sp_nexthop *nh)
1417 struct mlxsw_sp_neigh_entry *neigh_entry;
1418 struct fib_nh *fib_nh = nh->key.fib_nh;
1419 struct neighbour *n;
1423 if (!nh->nh_grp->gateway || nh->neigh_entry)
1426 /* Take a reference of neigh here ensuring that neigh would
1427 * not be detructed before the nexthop entry is finished.
1428 * The reference is taken either in neigh_lookup() or
1429 * in neigh_create() in case n is not found.
1431 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1433 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1436 neigh_event_send(n, NULL);
1438 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1440 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1441 if (IS_ERR(neigh_entry)) {
1443 goto err_neigh_entry_create;
1447 /* If that is the first nexthop connected to that neigh, add to
1448 * nexthop_neighs_list
1450 if (list_empty(&neigh_entry->nexthop_list))
1451 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1452 &mlxsw_sp->router.nexthop_neighs_list);
1454 nh->neigh_entry = neigh_entry;
1455 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1456 read_lock_bh(&n->lock);
1457 nud_state = n->nud_state;
1459 read_unlock_bh(&n->lock);
1460 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1464 err_neigh_entry_create:
1469 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1470 struct mlxsw_sp_nexthop *nh)
1472 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473 struct neighbour *n;
1477 n = neigh_entry->key.n;
1479 __mlxsw_sp_nexthop_neigh_update(nh, true);
1480 list_del(&nh->neigh_list_node);
1481 nh->neigh_entry = NULL;
1483 /* If that is the last nexthop connected to that neigh, remove from
1484 * nexthop_neighs_list
1486 if (list_empty(&neigh_entry->nexthop_list))
1487 list_del(&neigh_entry->nexthop_neighs_list_node);
1489 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1490 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1495 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1496 struct mlxsw_sp_nexthop_group *nh_grp,
1497 struct mlxsw_sp_nexthop *nh,
1498 struct fib_nh *fib_nh)
1500 struct net_device *dev = fib_nh->nh_dev;
1501 struct in_device *in_dev;
1502 struct mlxsw_sp_rif *rif;
1505 nh->nh_grp = nh_grp;
1506 nh->key.fib_nh = fib_nh;
1507 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1514 in_dev = __in_dev_get_rtnl(dev);
1515 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1516 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1519 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1522 mlxsw_sp_nexthop_rif_init(nh, rif);
1524 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1526 goto err_nexthop_neigh_init;
1530 err_nexthop_neigh_init:
1531 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1535 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1536 struct mlxsw_sp_nexthop *nh)
1538 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1539 mlxsw_sp_nexthop_rif_fini(nh);
1540 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1543 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1544 unsigned long event, struct fib_nh *fib_nh)
1546 struct mlxsw_sp_nexthop_key key;
1547 struct mlxsw_sp_nexthop *nh;
1548 struct mlxsw_sp_rif *rif;
1550 if (mlxsw_sp->router.aborted)
1553 key.fib_nh = fib_nh;
1554 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1555 if (WARN_ON_ONCE(!nh))
1558 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1563 case FIB_EVENT_NH_ADD:
1564 mlxsw_sp_nexthop_rif_init(nh, rif);
1565 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1567 case FIB_EVENT_NH_DEL:
1568 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1569 mlxsw_sp_nexthop_rif_fini(nh);
1573 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1576 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1577 struct mlxsw_sp_rif *rif)
1579 struct mlxsw_sp_nexthop *nh, *tmp;
1581 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1582 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1583 mlxsw_sp_nexthop_rif_fini(nh);
1584 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1588 static struct mlxsw_sp_nexthop_group *
1589 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1591 struct mlxsw_sp_nexthop_group *nh_grp;
1592 struct mlxsw_sp_nexthop *nh;
1593 struct fib_nh *fib_nh;
1598 alloc_size = sizeof(*nh_grp) +
1599 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1600 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1602 return ERR_PTR(-ENOMEM);
1603 INIT_LIST_HEAD(&nh_grp->fib_list);
1604 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1605 nh_grp->count = fi->fib_nhs;
1606 nh_grp->key.fi = fi;
1607 for (i = 0; i < nh_grp->count; i++) {
1608 nh = &nh_grp->nexthops[i];
1609 fib_nh = &fi->fib_nh[i];
1610 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1612 goto err_nexthop_init;
1614 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1616 goto err_nexthop_group_insert;
1617 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1620 err_nexthop_group_insert:
1622 for (i--; i >= 0; i--) {
1623 nh = &nh_grp->nexthops[i];
1624 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1627 return ERR_PTR(err);
1631 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1632 struct mlxsw_sp_nexthop_group *nh_grp)
1634 struct mlxsw_sp_nexthop *nh;
1637 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1638 for (i = 0; i < nh_grp->count; i++) {
1639 nh = &nh_grp->nexthops[i];
1640 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1642 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1643 WARN_ON_ONCE(nh_grp->adj_index_valid);
1647 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1648 struct mlxsw_sp_fib_entry *fib_entry,
1649 struct fib_info *fi)
1651 struct mlxsw_sp_nexthop_group_key key;
1652 struct mlxsw_sp_nexthop_group *nh_grp;
1655 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1657 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1659 return PTR_ERR(nh_grp);
1661 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1662 fib_entry->nh_group = nh_grp;
1666 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1667 struct mlxsw_sp_fib_entry *fib_entry)
1669 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1671 list_del(&fib_entry->nexthop_group_node);
1672 if (!list_empty(&nh_grp->fib_list))
1674 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1678 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1680 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1682 if (fib_entry->params.tos)
1685 switch (fib_entry->type) {
1686 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1687 return !!nh_group->adj_index_valid;
1688 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1689 return !!nh_group->nh_rif;
1695 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1697 fib_entry->offloaded = true;
1699 switch (fib_entry->fib_node->fib->proto) {
1700 case MLXSW_SP_L3_PROTO_IPV4:
1701 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1703 case MLXSW_SP_L3_PROTO_IPV6:
1709 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1711 switch (fib_entry->fib_node->fib->proto) {
1712 case MLXSW_SP_L3_PROTO_IPV4:
1713 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1715 case MLXSW_SP_L3_PROTO_IPV6:
1719 fib_entry->offloaded = false;
1723 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1724 enum mlxsw_reg_ralue_op op, int err)
1727 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1728 if (!fib_entry->offloaded)
1730 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1731 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1734 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1735 !fib_entry->offloaded)
1736 mlxsw_sp_fib_entry_offload_set(fib_entry);
1737 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1738 fib_entry->offloaded)
1739 mlxsw_sp_fib_entry_offload_unset(fib_entry);
1746 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1747 struct mlxsw_sp_fib_entry *fib_entry,
1748 enum mlxsw_reg_ralue_op op)
1750 char ralue_pl[MLXSW_REG_RALUE_LEN];
1751 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1752 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1753 enum mlxsw_reg_ralue_trap_action trap_action;
1755 u32 adjacency_index = 0;
1758 /* In case the nexthop group adjacency index is valid, use it
1759 * with provided ECMP size. Otherwise, setup trap and pass
1760 * traffic to kernel.
1762 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1763 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1764 adjacency_index = fib_entry->nh_group->adj_index;
1765 ecmp_size = fib_entry->nh_group->ecmp_size;
1767 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1768 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1771 mlxsw_reg_ralue_pack4(ralue_pl,
1772 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1773 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1775 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1776 adjacency_index, ecmp_size);
1777 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1780 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1781 struct mlxsw_sp_fib_entry *fib_entry,
1782 enum mlxsw_reg_ralue_op op)
1784 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
1785 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1786 enum mlxsw_reg_ralue_trap_action trap_action;
1787 char ralue_pl[MLXSW_REG_RALUE_LEN];
1788 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1792 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1793 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1794 rif_index = rif->rif_index;
1796 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1797 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1800 mlxsw_reg_ralue_pack4(ralue_pl,
1801 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1802 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1804 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
1806 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1809 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1810 struct mlxsw_sp_fib_entry *fib_entry,
1811 enum mlxsw_reg_ralue_op op)
1813 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
1814 char ralue_pl[MLXSW_REG_RALUE_LEN];
1815 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1817 mlxsw_reg_ralue_pack4(ralue_pl,
1818 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
1819 fib->vr->id, fib_entry->fib_node->key.prefix_len,
1821 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1822 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1825 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1826 struct mlxsw_sp_fib_entry *fib_entry,
1827 enum mlxsw_reg_ralue_op op)
1829 switch (fib_entry->type) {
1830 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1831 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1832 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1833 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1834 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1835 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1840 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1841 struct mlxsw_sp_fib_entry *fib_entry,
1842 enum mlxsw_reg_ralue_op op)
1846 switch (fib_entry->fib_node->fib->proto) {
1847 case MLXSW_SP_L3_PROTO_IPV4:
1848 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1850 case MLXSW_SP_L3_PROTO_IPV6:
1853 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1857 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1858 struct mlxsw_sp_fib_entry *fib_entry)
1860 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1861 MLXSW_REG_RALUE_OP_WRITE_WRITE);
1864 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1865 struct mlxsw_sp_fib_entry *fib_entry)
1867 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1868 MLXSW_REG_RALUE_OP_WRITE_DELETE);
1872 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1873 const struct fib_entry_notifier_info *fen_info,
1874 struct mlxsw_sp_fib_entry *fib_entry)
1876 struct fib_info *fi = fen_info->fi;
1878 switch (fen_info->type) {
1879 case RTN_BROADCAST: /* fall through */
1881 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1883 case RTN_UNREACHABLE: /* fall through */
1884 case RTN_BLACKHOLE: /* fall through */
1886 /* Packets hitting these routes need to be trapped, but
1887 * can do so with a lower priority than packets directed
1888 * at the host, so use action type local instead of trap.
1890 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1893 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1894 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1896 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1903 static struct mlxsw_sp_fib_entry *
1904 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1905 struct mlxsw_sp_fib_node *fib_node,
1906 const struct fib_entry_notifier_info *fen_info)
1908 struct mlxsw_sp_fib_entry *fib_entry;
1911 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1914 goto err_fib_entry_alloc;
1917 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1919 goto err_fib4_entry_type_set;
1921 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1923 goto err_nexthop_group_get;
1925 fib_entry->params.prio = fen_info->fi->fib_priority;
1926 fib_entry->params.tb_id = fen_info->tb_id;
1927 fib_entry->params.type = fen_info->type;
1928 fib_entry->params.tos = fen_info->tos;
1930 fib_entry->fib_node = fib_node;
1934 err_nexthop_group_get:
1935 err_fib4_entry_type_set:
1937 err_fib_entry_alloc:
1938 return ERR_PTR(err);
1941 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1942 struct mlxsw_sp_fib_entry *fib_entry)
1944 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1948 static struct mlxsw_sp_fib_node *
1949 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1950 const struct fib_entry_notifier_info *fen_info);
1952 static struct mlxsw_sp_fib_entry *
1953 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1954 const struct fib_entry_notifier_info *fen_info)
1956 struct mlxsw_sp_fib_entry *fib_entry;
1957 struct mlxsw_sp_fib_node *fib_node;
1959 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1960 if (IS_ERR(fib_node))
1963 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1964 if (fib_entry->params.tb_id == fen_info->tb_id &&
1965 fib_entry->params.tos == fen_info->tos &&
1966 fib_entry->params.type == fen_info->type &&
1967 fib_entry->nh_group->key.fi == fen_info->fi) {
1975 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1976 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1977 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1978 .key_len = sizeof(struct mlxsw_sp_fib_key),
1979 .automatic_shrinking = true,
1982 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1983 struct mlxsw_sp_fib_node *fib_node)
1985 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1986 mlxsw_sp_fib_ht_params);
1989 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1990 struct mlxsw_sp_fib_node *fib_node)
1992 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1993 mlxsw_sp_fib_ht_params);
1996 static struct mlxsw_sp_fib_node *
1997 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1998 size_t addr_len, unsigned char prefix_len)
2000 struct mlxsw_sp_fib_key key;
2002 memset(&key, 0, sizeof(key));
2003 memcpy(key.addr, addr, addr_len);
2004 key.prefix_len = prefix_len;
2005 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2008 static struct mlxsw_sp_fib_node *
2009 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2010 size_t addr_len, unsigned char prefix_len)
2012 struct mlxsw_sp_fib_node *fib_node;
2014 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2018 INIT_LIST_HEAD(&fib_node->entry_list);
2019 list_add(&fib_node->list, &fib->node_list);
2020 memcpy(fib_node->key.addr, addr, addr_len);
2021 fib_node->key.prefix_len = prefix_len;
2026 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2028 list_del(&fib_node->list);
2029 WARN_ON(!list_empty(&fib_node->entry_list));
2034 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2035 const struct mlxsw_sp_fib_entry *fib_entry)
2037 return list_first_entry(&fib_node->entry_list,
2038 struct mlxsw_sp_fib_entry, list) == fib_entry;
2041 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2043 unsigned char prefix_len = fib_node->key.prefix_len;
2044 struct mlxsw_sp_fib *fib = fib_node->fib;
2046 if (fib->prefix_ref_count[prefix_len]++ == 0)
2047 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2050 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2052 unsigned char prefix_len = fib_node->key.prefix_len;
2053 struct mlxsw_sp_fib *fib = fib_node->fib;
2055 if (--fib->prefix_ref_count[prefix_len] == 0)
2056 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2059 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2060 struct mlxsw_sp_fib_node *fib_node,
2061 struct mlxsw_sp_fib *fib)
2063 struct mlxsw_sp_prefix_usage req_prefix_usage;
2064 struct mlxsw_sp_lpm_tree *lpm_tree;
2067 err = mlxsw_sp_fib_node_insert(fib, fib_node);
2070 fib_node->fib = fib;
2072 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2073 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2075 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2076 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2079 goto err_tree_check;
2081 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2083 if (IS_ERR(lpm_tree))
2084 return PTR_ERR(lpm_tree);
2085 fib->lpm_tree = lpm_tree;
2086 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2091 mlxsw_sp_fib_node_prefix_inc(fib_node);
2096 fib->lpm_tree = NULL;
2097 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2099 fib_node->fib = NULL;
2100 mlxsw_sp_fib_node_remove(fib, fib_node);
2104 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2105 struct mlxsw_sp_fib_node *fib_node)
2107 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2108 struct mlxsw_sp_fib *fib = fib_node->fib;
2110 mlxsw_sp_fib_node_prefix_dec(fib_node);
2112 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2113 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2114 fib->lpm_tree = NULL;
2115 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2117 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2120 fib_node->fib = NULL;
2121 mlxsw_sp_fib_node_remove(fib, fib_node);
2124 static struct mlxsw_sp_fib_node *
2125 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2126 const struct fib_entry_notifier_info *fen_info)
2128 struct mlxsw_sp_fib_node *fib_node;
2129 struct mlxsw_sp_fib *fib;
2130 struct mlxsw_sp_vr *vr;
2133 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2135 return ERR_CAST(vr);
2136 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2138 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2139 sizeof(fen_info->dst),
2144 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2145 sizeof(fen_info->dst),
2149 goto err_fib_node_create;
2152 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2154 goto err_fib_node_init;
2159 mlxsw_sp_fib_node_destroy(fib_node);
2160 err_fib_node_create:
2161 mlxsw_sp_vr_put(vr);
2162 return ERR_PTR(err);
2165 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2166 struct mlxsw_sp_fib_node *fib_node)
2168 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2170 if (!list_empty(&fib_node->entry_list))
2172 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2173 mlxsw_sp_fib_node_destroy(fib_node);
2174 mlxsw_sp_vr_put(vr);
2177 static struct mlxsw_sp_fib_entry *
2178 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2179 const struct mlxsw_sp_fib_entry_params *params)
2181 struct mlxsw_sp_fib_entry *fib_entry;
2183 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2184 if (fib_entry->params.tb_id > params->tb_id)
2186 if (fib_entry->params.tb_id != params->tb_id)
2188 if (fib_entry->params.tos > params->tos)
2190 if (fib_entry->params.prio >= params->prio ||
2191 fib_entry->params.tos < params->tos)
2198 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2199 struct mlxsw_sp_fib_entry *new_entry)
2201 struct mlxsw_sp_fib_node *fib_node;
2203 if (WARN_ON(!fib_entry))
2206 fib_node = fib_entry->fib_node;
2207 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2208 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2209 fib_entry->params.tos != new_entry->params.tos ||
2210 fib_entry->params.prio != new_entry->params.prio)
2214 list_add_tail(&new_entry->list, &fib_entry->list);
2219 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2220 struct mlxsw_sp_fib_entry *new_entry,
2221 bool replace, bool append)
2223 struct mlxsw_sp_fib_entry *fib_entry;
2225 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2228 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2229 if (replace && WARN_ON(!fib_entry))
2232 /* Insert new entry before replaced one, so that we can later
2233 * remove the second.
2236 list_add_tail(&new_entry->list, &fib_entry->list);
2238 struct mlxsw_sp_fib_entry *last;
2240 list_for_each_entry(last, &fib_node->entry_list, list) {
2241 if (new_entry->params.tb_id > last->params.tb_id)
2247 list_add(&new_entry->list, &fib_entry->list);
2249 list_add(&new_entry->list, &fib_node->entry_list);
2256 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2258 list_del(&fib_entry->list);
2262 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2263 const struct mlxsw_sp_fib_node *fib_node,
2264 struct mlxsw_sp_fib_entry *fib_entry)
2266 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2269 /* To prevent packet loss, overwrite the previously offloaded
2272 if (!list_is_singular(&fib_node->entry_list)) {
2273 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2274 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2276 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2279 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2283 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2284 const struct mlxsw_sp_fib_node *fib_node,
2285 struct mlxsw_sp_fib_entry *fib_entry)
2287 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2290 /* Promote the next entry by overwriting the deleted entry */
2291 if (!list_is_singular(&fib_node->entry_list)) {
2292 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2293 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2295 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2296 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2300 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2303 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2304 struct mlxsw_sp_fib_entry *fib_entry,
2305 bool replace, bool append)
2307 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2310 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2315 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2317 goto err_fib4_node_entry_add;
2321 err_fib4_node_entry_add:
2322 mlxsw_sp_fib4_node_list_remove(fib_entry);
2327 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2328 struct mlxsw_sp_fib_entry *fib_entry)
2330 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2332 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2333 mlxsw_sp_fib4_node_list_remove(fib_entry);
2336 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2337 struct mlxsw_sp_fib_entry *fib_entry,
2340 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2341 struct mlxsw_sp_fib_entry *replaced;
2346 /* We inserted the new entry before replaced one */
2347 replaced = list_next_entry(fib_entry, list);
2349 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2350 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2351 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2355 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2356 const struct fib_entry_notifier_info *fen_info,
2357 bool replace, bool append)
2359 struct mlxsw_sp_fib_entry *fib_entry;
2360 struct mlxsw_sp_fib_node *fib_node;
2363 if (mlxsw_sp->router.aborted)
2366 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2367 if (IS_ERR(fib_node)) {
2368 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2369 return PTR_ERR(fib_node);
2372 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2373 if (IS_ERR(fib_entry)) {
2374 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2375 err = PTR_ERR(fib_entry);
2376 goto err_fib4_entry_create;
2379 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2382 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2383 goto err_fib4_node_entry_link;
2386 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2390 err_fib4_node_entry_link:
2391 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2392 err_fib4_entry_create:
2393 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2397 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2398 struct fib_entry_notifier_info *fen_info)
2400 struct mlxsw_sp_fib_entry *fib_entry;
2401 struct mlxsw_sp_fib_node *fib_node;
2403 if (mlxsw_sp->router.aborted)
2406 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2407 if (WARN_ON(!fib_entry))
2409 fib_node = fib_entry->fib_node;
2411 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2412 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2413 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2416 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2418 char ralta_pl[MLXSW_REG_RALTA_LEN];
2419 char ralst_pl[MLXSW_REG_RALST_LEN];
2422 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2423 MLXSW_SP_LPM_TREE_MIN);
2424 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2428 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2429 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2433 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2434 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2435 char raltb_pl[MLXSW_REG_RALTB_LEN];
2436 char ralue_pl[MLXSW_REG_RALUE_LEN];
2438 if (!mlxsw_sp_vr_is_used(vr))
2441 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2442 MLXSW_REG_RALXX_PROTOCOL_IPV4,
2443 MLXSW_SP_LPM_TREE_MIN);
2444 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2449 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2450 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2452 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2453 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2462 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2463 struct mlxsw_sp_fib_node *fib_node)
2465 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2467 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2468 bool do_break = &tmp->list == &fib_node->entry_list;
2470 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2471 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2472 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2473 /* Break when entry list is empty and node was freed.
2474 * Otherwise, we'll access freed memory in the next
2482 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2483 struct mlxsw_sp_fib_node *fib_node)
2485 switch (fib_node->fib->proto) {
2486 case MLXSW_SP_L3_PROTO_IPV4:
2487 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2489 case MLXSW_SP_L3_PROTO_IPV6:
2495 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2496 struct mlxsw_sp_vr *vr,
2497 enum mlxsw_sp_l3proto proto)
2499 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2500 struct mlxsw_sp_fib_node *fib_node, *tmp;
2502 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2503 bool do_break = &tmp->list == &fib->node_list;
2505 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2511 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2515 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2516 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[i];
2518 if (!mlxsw_sp_vr_is_used(vr))
2520 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2524 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2528 if (mlxsw_sp->router.aborted)
2530 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2531 mlxsw_sp_router_fib_flush(mlxsw_sp);
2532 mlxsw_sp->router.aborted = true;
2533 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2535 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2538 struct mlxsw_sp_fib_event_work {
2539 struct work_struct work;
2541 struct fib_entry_notifier_info fen_info;
2542 struct fib_rule_notifier_info fr_info;
2543 struct fib_nh_notifier_info fnh_info;
2545 struct mlxsw_sp *mlxsw_sp;
2546 unsigned long event;
2549 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2551 struct mlxsw_sp_fib_event_work *fib_work =
2552 container_of(work, struct mlxsw_sp_fib_event_work, work);
2553 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2554 struct fib_rule *rule;
2555 bool replace, append;
2558 /* Protect internal structures from changes */
2560 switch (fib_work->event) {
2561 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2562 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2563 case FIB_EVENT_ENTRY_ADD:
2564 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2565 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2566 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2569 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2570 fib_info_put(fib_work->fen_info.fi);
2572 case FIB_EVENT_ENTRY_DEL:
2573 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2574 fib_info_put(fib_work->fen_info.fi);
2576 case FIB_EVENT_RULE_ADD: /* fall through */
2577 case FIB_EVENT_RULE_DEL:
2578 rule = fib_work->fr_info.rule;
2579 if (!fib4_rule_default(rule) && !rule->l3mdev)
2580 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2583 case FIB_EVENT_NH_ADD: /* fall through */
2584 case FIB_EVENT_NH_DEL:
2585 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2586 fib_work->fnh_info.fib_nh);
2587 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2594 /* Called with rcu_read_lock() */
2595 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2596 unsigned long event, void *ptr)
2598 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2599 struct mlxsw_sp_fib_event_work *fib_work;
2600 struct fib_notifier_info *info = ptr;
2602 if (!net_eq(info->net, &init_net))
2605 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2606 if (WARN_ON(!fib_work))
2609 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2610 fib_work->mlxsw_sp = mlxsw_sp;
2611 fib_work->event = event;
2614 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2615 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2616 case FIB_EVENT_ENTRY_ADD: /* fall through */
2617 case FIB_EVENT_ENTRY_DEL:
2618 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2619 /* Take referece on fib_info to prevent it from being
2620 * freed while work is queued. Release it afterwards.
2622 fib_info_hold(fib_work->fen_info.fi);
2624 case FIB_EVENT_RULE_ADD: /* fall through */
2625 case FIB_EVENT_RULE_DEL:
2626 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2627 fib_rule_get(fib_work->fr_info.rule);
2629 case FIB_EVENT_NH_ADD: /* fall through */
2630 case FIB_EVENT_NH_DEL:
2631 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2632 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2636 mlxsw_core_schedule_work(&fib_work->work);
2641 static struct mlxsw_sp_rif *
2642 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2643 const struct net_device *dev)
2647 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2648 if (mlxsw_sp->rifs[i] && mlxsw_sp->rifs[i]->dev == dev)
2649 return mlxsw_sp->rifs[i];
2654 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2656 char ritr_pl[MLXSW_REG_RITR_LEN];
2659 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2660 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2661 if (WARN_ON_ONCE(err))
2664 mlxsw_reg_ritr_enable_set(ritr_pl, false);
2665 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2668 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2669 struct mlxsw_sp_rif *rif)
2671 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2672 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2673 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2676 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2677 const struct in_device *in_dev,
2678 unsigned long event)
2686 if (rif && !in_dev->ifa_list &&
2687 !netif_is_l3_slave(rif->dev))
2689 /* It is possible we already removed the RIF ourselves
2690 * if it was assigned to a netdev that is now a bridge
2699 #define MLXSW_SP_INVALID_INDEX_RIF 0xffff
2700 static int mlxsw_sp_avail_rif_get(struct mlxsw_sp *mlxsw_sp)
2704 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2705 if (!mlxsw_sp->rifs[i])
2708 return MLXSW_SP_INVALID_INDEX_RIF;
2711 static void mlxsw_sp_vport_rif_sp_attr_get(struct mlxsw_sp_port *mlxsw_sp_vport,
2712 bool *p_lagged, u16 *p_system_port)
2714 u8 local_port = mlxsw_sp_vport->local_port;
2716 *p_lagged = mlxsw_sp_vport->lagged;
2717 *p_system_port = *p_lagged ? mlxsw_sp_vport->lag_id : local_port;
2720 static int mlxsw_sp_vport_rif_sp_op(struct mlxsw_sp_port *mlxsw_sp_vport,
2721 u16 vr_id, struct net_device *l3_dev,
2722 u16 rif_index, bool create)
2724 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2725 bool lagged = mlxsw_sp_vport->lagged;
2726 char ritr_pl[MLXSW_REG_RITR_LEN];
2729 mlxsw_reg_ritr_pack(ritr_pl, create, MLXSW_REG_RITR_SP_IF, rif_index,
2730 vr_id, l3_dev->mtu, l3_dev->dev_addr);
2732 mlxsw_sp_vport_rif_sp_attr_get(mlxsw_sp_vport, &lagged, &system_port);
2733 mlxsw_reg_ritr_sp_if_pack(ritr_pl, lagged, system_port,
2734 mlxsw_sp_vport_vid_get(mlxsw_sp_vport));
2736 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2739 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport);
2741 static u16 mlxsw_sp_rif_sp_to_fid(u16 rif_index)
2743 return MLXSW_SP_RFID_BASE + rif_index;
2746 static struct mlxsw_sp_fid *
2747 mlxsw_sp_rfid_alloc(u16 fid, struct net_device *l3_dev)
2749 struct mlxsw_sp_fid *f;
2751 f = kzalloc(sizeof(*f), GFP_KERNEL);
2755 f->leave = mlxsw_sp_vport_rif_sp_leave;
2763 static struct mlxsw_sp_rif *
2764 mlxsw_sp_rif_alloc(u16 rif_index, u16 vr_id, struct net_device *l3_dev,
2765 struct mlxsw_sp_fid *f)
2767 struct mlxsw_sp_rif *rif;
2769 rif = kzalloc(sizeof(*rif), GFP_KERNEL);
2773 INIT_LIST_HEAD(&rif->nexthop_list);
2774 INIT_LIST_HEAD(&rif->neigh_list);
2775 ether_addr_copy(rif->addr, l3_dev->dev_addr);
2776 rif->mtu = l3_dev->mtu;
2779 rif->rif_index = rif_index;
2785 static struct mlxsw_sp_rif *
2786 mlxsw_sp_vport_rif_sp_create(struct mlxsw_sp_port *mlxsw_sp_vport,
2787 struct net_device *l3_dev)
2789 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2790 u32 tb_id = l3mdev_fib_table(l3_dev);
2791 struct mlxsw_sp_vr *vr;
2792 struct mlxsw_sp_fid *f;
2793 struct mlxsw_sp_rif *rif;
2797 rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
2798 if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
2799 return ERR_PTR(-ERANGE);
2801 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
2803 return ERR_CAST(vr);
2805 err = mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev,
2808 goto err_vport_rif_sp_op;
2810 fid = mlxsw_sp_rif_sp_to_fid(rif_index);
2811 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, true);
2813 goto err_rif_fdb_op;
2815 f = mlxsw_sp_rfid_alloc(fid, l3_dev);
2818 goto err_rfid_alloc;
2821 rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
2828 mlxsw_sp->rifs[rif_index] = rif;
2836 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2838 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2840 err_vport_rif_sp_op:
2841 mlxsw_sp_vr_put(vr);
2842 return ERR_PTR(err);
2845 static void mlxsw_sp_vport_rif_sp_destroy(struct mlxsw_sp_port *mlxsw_sp_vport,
2846 struct mlxsw_sp_rif *rif)
2848 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2849 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
2850 struct net_device *l3_dev = rif->dev;
2851 struct mlxsw_sp_fid *f = rif->f;
2852 u16 rif_index = rif->rif_index;
2855 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
2858 mlxsw_sp->rifs[rif_index] = NULL;
2865 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, fid, false);
2867 mlxsw_sp_vport_rif_sp_op(mlxsw_sp_vport, vr->id, l3_dev, rif_index,
2869 mlxsw_sp_vr_put(vr);
2872 static int mlxsw_sp_vport_rif_sp_join(struct mlxsw_sp_port *mlxsw_sp_vport,
2873 struct net_device *l3_dev)
2875 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_vport->mlxsw_sp;
2876 struct mlxsw_sp_rif *rif;
2878 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
2880 rif = mlxsw_sp_vport_rif_sp_create(mlxsw_sp_vport, l3_dev);
2882 return PTR_ERR(rif);
2885 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, rif->f);
2886 rif->f->ref_count++;
2888 netdev_dbg(mlxsw_sp_vport->dev, "Joined FID=%d\n", rif->f->fid);
2893 static void mlxsw_sp_vport_rif_sp_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
2895 struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
2897 netdev_dbg(mlxsw_sp_vport->dev, "Left FID=%d\n", f->fid);
2899 mlxsw_sp_vport_fid_set(mlxsw_sp_vport, NULL);
2900 if (--f->ref_count == 0)
2901 mlxsw_sp_vport_rif_sp_destroy(mlxsw_sp_vport, f->rif);
2904 static int mlxsw_sp_inetaddr_vport_event(struct net_device *l3_dev,
2905 struct net_device *port_dev,
2906 unsigned long event, u16 vid)
2908 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
2909 struct mlxsw_sp_port *mlxsw_sp_vport;
2911 mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, vid);
2912 if (WARN_ON(!mlxsw_sp_vport))
2917 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, l3_dev);
2919 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
2926 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
2927 unsigned long event)
2929 if (netif_is_bridge_port(port_dev) || netif_is_lag_port(port_dev))
2932 return mlxsw_sp_inetaddr_vport_event(port_dev, port_dev, event, 1);
2935 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
2936 struct net_device *lag_dev,
2937 unsigned long event, u16 vid)
2939 struct net_device *port_dev;
2940 struct list_head *iter;
2943 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
2944 if (mlxsw_sp_port_dev_check(port_dev)) {
2945 err = mlxsw_sp_inetaddr_vport_event(l3_dev, port_dev,
2955 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
2956 unsigned long event)
2958 if (netif_is_bridge_port(lag_dev))
2961 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
2964 static struct mlxsw_sp_fid *mlxsw_sp_bridge_fid_get(struct mlxsw_sp *mlxsw_sp,
2965 struct net_device *l3_dev)
2969 if (is_vlan_dev(l3_dev))
2970 fid = vlan_dev_vlan_id(l3_dev);
2971 else if (mlxsw_sp->master_bridge.dev == l3_dev)
2974 return mlxsw_sp_vfid_find(mlxsw_sp, l3_dev);
2976 return mlxsw_sp_fid_find(mlxsw_sp, fid);
2979 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
2981 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
2984 static enum mlxsw_flood_table_type mlxsw_sp_flood_table_type_get(u16 fid)
2986 return mlxsw_sp_fid_is_vfid(fid) ? MLXSW_REG_SFGC_TABLE_TYPE_FID :
2987 MLXSW_REG_SFGC_TABLE_TYPE_FID_OFFEST;
2990 static u16 mlxsw_sp_flood_table_index_get(u16 fid)
2992 return mlxsw_sp_fid_is_vfid(fid) ? mlxsw_sp_fid_to_vfid(fid) : fid;
2995 static int mlxsw_sp_router_port_flood_set(struct mlxsw_sp *mlxsw_sp, u16 fid,
2998 u8 router_port = mlxsw_sp_router_port(mlxsw_sp);
2999 enum mlxsw_flood_table_type table_type;
3004 sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL);
3008 table_type = mlxsw_sp_flood_table_type_get(fid);
3009 index = mlxsw_sp_flood_table_index_get(fid);
3010 mlxsw_reg_sftr_pack(sftr_pl, MLXSW_SP_FLOOD_TABLE_BC, index, table_type,
3011 1, router_port, set);
3012 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sftr), sftr_pl);
3018 static enum mlxsw_reg_ritr_if_type mlxsw_sp_rif_type_get(u16 fid)
3020 if (mlxsw_sp_fid_is_vfid(fid))
3021 return MLXSW_REG_RITR_FID_IF;
3023 return MLXSW_REG_RITR_VLAN_IF;
3026 static int mlxsw_sp_rif_bridge_op(struct mlxsw_sp *mlxsw_sp, u16 vr_id,
3027 struct net_device *l3_dev,
3031 enum mlxsw_reg_ritr_if_type rif_type;
3032 char ritr_pl[MLXSW_REG_RITR_LEN];
3034 rif_type = mlxsw_sp_rif_type_get(fid);
3035 mlxsw_reg_ritr_pack(ritr_pl, create, rif_type, rif, vr_id, l3_dev->mtu,
3037 mlxsw_reg_ritr_fid_set(ritr_pl, rif_type, fid);
3039 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3042 static int mlxsw_sp_rif_bridge_create(struct mlxsw_sp *mlxsw_sp,
3043 struct net_device *l3_dev,
3044 struct mlxsw_sp_fid *f)
3046 u32 tb_id = l3mdev_fib_table(l3_dev);
3047 struct mlxsw_sp_rif *rif;
3048 struct mlxsw_sp_vr *vr;
3052 rif_index = mlxsw_sp_avail_rif_get(mlxsw_sp);
3053 if (rif_index == MLXSW_SP_INVALID_INDEX_RIF)
3056 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3060 err = mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, true);
3062 goto err_port_flood_set;
3064 err = mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid,
3067 goto err_rif_bridge_op;
3069 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, true);
3071 goto err_rif_fdb_op;
3073 rif = mlxsw_sp_rif_alloc(rif_index, vr->id, l3_dev, f);
3080 mlxsw_sp->rifs[rif_index] = rif;
3083 netdev_dbg(l3_dev, "RIF=%d created\n", rif_index);
3088 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3090 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3093 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3095 mlxsw_sp_vr_put(vr);
3099 void mlxsw_sp_rif_bridge_destroy(struct mlxsw_sp *mlxsw_sp,
3100 struct mlxsw_sp_rif *rif)
3102 struct mlxsw_sp_vr *vr = &mlxsw_sp->router.vrs[rif->vr_id];
3103 struct net_device *l3_dev = rif->dev;
3104 struct mlxsw_sp_fid *f = rif->f;
3105 u16 rif_index = rif->rif_index;
3107 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3110 mlxsw_sp->rifs[rif_index] = NULL;
3115 mlxsw_sp_rif_fdb_op(mlxsw_sp, l3_dev->dev_addr, f->fid, false);
3117 mlxsw_sp_rif_bridge_op(mlxsw_sp, vr->id, l3_dev, f->fid, rif_index,
3120 mlxsw_sp_router_port_flood_set(mlxsw_sp, f->fid, false);
3122 mlxsw_sp_vr_put(vr);
3124 netdev_dbg(l3_dev, "RIF=%d destroyed\n", rif_index);
3127 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3128 struct net_device *br_dev,
3129 unsigned long event)
3131 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3132 struct mlxsw_sp_fid *f;
3134 /* FID can either be an actual FID if the L3 device is the
3135 * VLAN-aware bridge or a VLAN device on top. Otherwise, the
3136 * L3 device is a VLAN-unaware bridge and we get a vFID.
3138 f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3144 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3146 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3153 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3154 unsigned long event)
3156 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3157 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(vlan_dev);
3158 u16 vid = vlan_dev_vlan_id(vlan_dev);
3160 if (mlxsw_sp_port_dev_check(real_dev))
3161 return mlxsw_sp_inetaddr_vport_event(vlan_dev, real_dev, event,
3163 else if (netif_is_lag_master(real_dev))
3164 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3166 else if (netif_is_bridge_master(real_dev) &&
3167 mlxsw_sp->master_bridge.dev == real_dev)
3168 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, real_dev,
3174 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3175 unsigned long event, void *ptr)
3177 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3178 struct net_device *dev = ifa->ifa_dev->dev;
3179 struct mlxsw_sp *mlxsw_sp;
3180 struct mlxsw_sp_rif *rif;
3183 mlxsw_sp = mlxsw_sp_lower_get(dev);
3187 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3188 if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3191 if (mlxsw_sp_port_dev_check(dev))
3192 err = mlxsw_sp_inetaddr_port_event(dev, event);
3193 else if (netif_is_lag_master(dev))
3194 err = mlxsw_sp_inetaddr_lag_event(dev, event);
3195 else if (netif_is_bridge_master(dev))
3196 err = mlxsw_sp_inetaddr_bridge_event(dev, dev, event);
3197 else if (is_vlan_dev(dev))
3198 err = mlxsw_sp_inetaddr_vlan_event(dev, event);
3201 return notifier_from_errno(err);
3204 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3205 const char *mac, int mtu)
3207 char ritr_pl[MLXSW_REG_RITR_LEN];
3210 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3211 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3215 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3216 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3217 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3218 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3221 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3223 struct mlxsw_sp *mlxsw_sp;
3224 struct mlxsw_sp_rif *rif;
3227 mlxsw_sp = mlxsw_sp_lower_get(dev);
3231 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3235 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, false);
3239 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3244 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, rif->f->fid, true);
3246 goto err_rif_fdb_op;
3248 ether_addr_copy(rif->addr, dev->dev_addr);
3249 rif->mtu = dev->mtu;
3251 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3256 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3258 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, rif->f->fid, true);
3262 int mlxsw_sp_vport_vrf_join(struct mlxsw_sp_port *mlxsw_sp_vport)
3264 struct mlxsw_sp_fid *f = mlxsw_sp_vport_fid_get(mlxsw_sp_vport);
3265 struct net_device *dev = mlxsw_sp_vport->dev;
3267 /* In case vPort already has a RIF, then we need to drop it.
3268 * A new one will be created using the VRF's VR.
3271 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3273 return mlxsw_sp_vport_rif_sp_join(mlxsw_sp_vport, dev);
3276 void mlxsw_sp_vport_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_vport)
3278 mlxsw_sp_vport_rif_sp_leave(mlxsw_sp_vport);
3281 int mlxsw_sp_port_vrf_join(struct mlxsw_sp_port *mlxsw_sp_port)
3283 struct mlxsw_sp_port *mlxsw_sp_vport;
3285 mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3286 if (WARN_ON(!mlxsw_sp_vport))
3289 return mlxsw_sp_vport_vrf_join(mlxsw_sp_vport);
3292 void mlxsw_sp_port_vrf_leave(struct mlxsw_sp_port *mlxsw_sp_port)
3294 struct mlxsw_sp_port *mlxsw_sp_vport;
3296 mlxsw_sp_vport = mlxsw_sp_port_vport_find(mlxsw_sp_port, 1);
3297 if (WARN_ON(!mlxsw_sp_vport))
3300 mlxsw_sp_vport_vrf_leave(mlxsw_sp_vport);
3303 int mlxsw_sp_bridge_vrf_join(struct mlxsw_sp *mlxsw_sp,
3304 struct net_device *l3_dev)
3306 struct mlxsw_sp_fid *f;
3308 f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3313 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3315 return mlxsw_sp_rif_bridge_create(mlxsw_sp, l3_dev, f);
3318 void mlxsw_sp_bridge_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3319 struct net_device *l3_dev)
3321 struct mlxsw_sp_fid *f;
3323 f = mlxsw_sp_bridge_fid_get(mlxsw_sp, l3_dev);
3326 mlxsw_sp_rif_bridge_destroy(mlxsw_sp, f->rif);
3329 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3331 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
3333 /* Flush pending FIB notifications and then flush the device's
3334 * table before requesting another dump. The FIB notification
3335 * block is unregistered, so no need to take RTNL.
3337 mlxsw_core_flush_owq();
3338 mlxsw_sp_router_fib_flush(mlxsw_sp);
3341 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3343 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3347 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3350 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3351 mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
3353 if (!mlxsw_sp->rifs)
3356 mlxsw_reg_rgcr_pack(rgcr_pl, true);
3357 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3358 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3365 kfree(mlxsw_sp->rifs);
3369 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3371 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3374 mlxsw_reg_rgcr_pack(rgcr_pl, false);
3375 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3377 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3378 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
3380 kfree(mlxsw_sp->rifs);
3383 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3387 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
3388 err = __mlxsw_sp_router_init(mlxsw_sp);
3392 err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
3393 &mlxsw_sp_nexthop_ht_params);
3395 goto err_nexthop_ht_init;
3397 err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
3398 &mlxsw_sp_nexthop_group_ht_params);
3400 goto err_nexthop_group_ht_init;
3402 err = mlxsw_sp_lpm_init(mlxsw_sp);
3406 err = mlxsw_sp_vrs_init(mlxsw_sp);
3410 err = mlxsw_sp_neigh_init(mlxsw_sp);
3412 goto err_neigh_init;
3414 mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3415 err = register_fib_notifier(&mlxsw_sp->fib_nb,
3416 mlxsw_sp_router_fib_dump_flush);
3418 goto err_register_fib_notifier;
3422 err_register_fib_notifier:
3423 mlxsw_sp_neigh_fini(mlxsw_sp);
3425 mlxsw_sp_vrs_fini(mlxsw_sp);
3427 mlxsw_sp_lpm_fini(mlxsw_sp);
3429 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3430 err_nexthop_group_ht_init:
3431 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3432 err_nexthop_ht_init:
3433 __mlxsw_sp_router_fini(mlxsw_sp);
3437 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3439 unregister_fib_notifier(&mlxsw_sp->fib_nb);
3440 mlxsw_sp_neigh_fini(mlxsw_sp);
3441 mlxsw_sp_vrs_fini(mlxsw_sp);
3442 mlxsw_sp_lpm_fini(mlxsw_sp);
3443 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
3444 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
3445 __mlxsw_sp_router_fini(mlxsw_sp);