2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <linux/inetdevice.h>
44 #include <net/netevent.h>
45 #include <net/neighbour.h>
47 #include <net/ip_fib.h>
53 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
54 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
57 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
58 struct mlxsw_sp_prefix_usage *prefix_usage2)
62 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
63 if (!test_bit(prefix, prefix_usage2->b))
70 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
71 struct mlxsw_sp_prefix_usage *prefix_usage2)
73 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
77 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
79 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
81 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
85 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
86 struct mlxsw_sp_prefix_usage *prefix_usage2)
88 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
92 mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
94 memset(prefix_usage, 0, sizeof(*prefix_usage));
98 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
99 unsigned char prefix_len)
101 set_bit(prefix_len, prefix_usage->b);
105 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
106 unsigned char prefix_len)
108 clear_bit(prefix_len, prefix_usage->b);
111 struct mlxsw_sp_fib_key {
112 unsigned char addr[sizeof(struct in6_addr)];
113 unsigned char prefix_len;
116 enum mlxsw_sp_fib_entry_type {
117 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
118 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
119 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
122 struct mlxsw_sp_nexthop_group;
124 struct mlxsw_sp_fib_node {
125 struct list_head entry_list;
126 struct list_head list;
127 struct rhash_head ht_node;
128 struct mlxsw_sp_vr *vr;
129 struct mlxsw_sp_fib_key key;
132 struct mlxsw_sp_fib_entry_params {
139 struct mlxsw_sp_fib_entry {
140 struct list_head list;
141 struct mlxsw_sp_fib_node *fib_node;
142 enum mlxsw_sp_fib_entry_type type;
143 struct list_head nexthop_group_node;
144 struct mlxsw_sp_nexthop_group *nh_group;
145 struct mlxsw_sp_fib_entry_params params;
149 struct mlxsw_sp_fib {
150 struct rhashtable ht;
151 struct list_head node_list;
152 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
153 struct mlxsw_sp_prefix_usage prefix_usage;
156 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
158 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
160 struct mlxsw_sp_fib *fib;
163 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
165 return ERR_PTR(-ENOMEM);
166 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
168 goto err_rhashtable_init;
169 INIT_LIST_HEAD(&fib->node_list);
177 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
179 WARN_ON(!list_empty(&fib->node_list));
180 rhashtable_destroy(&fib->ht);
184 static struct mlxsw_sp_lpm_tree *
185 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
187 static struct mlxsw_sp_lpm_tree *lpm_tree;
190 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
191 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
192 if (lpm_tree->ref_count == 0) {
194 one_reserved = false;
202 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
203 struct mlxsw_sp_lpm_tree *lpm_tree)
205 char ralta_pl[MLXSW_REG_RALTA_LEN];
207 mlxsw_reg_ralta_pack(ralta_pl, true,
208 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
210 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
213 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
214 struct mlxsw_sp_lpm_tree *lpm_tree)
216 char ralta_pl[MLXSW_REG_RALTA_LEN];
218 mlxsw_reg_ralta_pack(ralta_pl, false,
219 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
221 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
225 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
226 struct mlxsw_sp_prefix_usage *prefix_usage,
227 struct mlxsw_sp_lpm_tree *lpm_tree)
229 char ralst_pl[MLXSW_REG_RALST_LEN];
232 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
234 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
237 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
238 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
241 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
242 MLXSW_REG_RALST_BIN_NO_CHILD);
243 last_prefix = prefix;
245 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
248 static struct mlxsw_sp_lpm_tree *
249 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
250 struct mlxsw_sp_prefix_usage *prefix_usage,
251 enum mlxsw_sp_l3proto proto, bool one_reserved)
253 struct mlxsw_sp_lpm_tree *lpm_tree;
256 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
258 return ERR_PTR(-EBUSY);
259 lpm_tree->proto = proto;
260 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
264 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
267 goto err_left_struct_set;
268 memcpy(&lpm_tree->prefix_usage, prefix_usage,
269 sizeof(lpm_tree->prefix_usage));
273 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
277 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
278 struct mlxsw_sp_lpm_tree *lpm_tree)
280 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
283 static struct mlxsw_sp_lpm_tree *
284 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
285 struct mlxsw_sp_prefix_usage *prefix_usage,
286 enum mlxsw_sp_l3proto proto, bool one_reserved)
288 struct mlxsw_sp_lpm_tree *lpm_tree;
291 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
292 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
293 if (lpm_tree->ref_count != 0 &&
294 lpm_tree->proto == proto &&
295 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
299 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
300 proto, one_reserved);
301 if (IS_ERR(lpm_tree))
305 lpm_tree->ref_count++;
309 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
310 struct mlxsw_sp_lpm_tree *lpm_tree)
312 if (--lpm_tree->ref_count == 0)
313 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
317 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
319 struct mlxsw_sp_lpm_tree *lpm_tree;
322 for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
323 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
324 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
328 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
330 struct mlxsw_sp_vr *vr;
333 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
334 vr = &mlxsw_sp->router.vrs[i];
341 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
342 struct mlxsw_sp_vr *vr)
344 char raltb_pl[MLXSW_REG_RALTB_LEN];
346 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
347 (enum mlxsw_reg_ralxx_protocol) vr->proto,
349 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
352 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
353 struct mlxsw_sp_vr *vr)
355 char raltb_pl[MLXSW_REG_RALTB_LEN];
357 /* Bind to tree 0 which is default */
358 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
359 (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
360 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
363 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
365 /* For our purpose, squash main and local table into one */
366 if (tb_id == RT_TABLE_LOCAL)
367 tb_id = RT_TABLE_MAIN;
371 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
373 enum mlxsw_sp_l3proto proto)
375 struct mlxsw_sp_vr *vr;
378 tb_id = mlxsw_sp_fix_tb_id(tb_id);
380 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
381 vr = &mlxsw_sp->router.vrs[i];
382 if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
388 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
389 unsigned char prefix_len,
391 enum mlxsw_sp_l3proto proto)
393 struct mlxsw_sp_prefix_usage req_prefix_usage;
394 struct mlxsw_sp_lpm_tree *lpm_tree;
395 struct mlxsw_sp_vr *vr;
398 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
400 return ERR_PTR(-EBUSY);
401 vr->fib = mlxsw_sp_fib_create();
403 return ERR_CAST(vr->fib);
407 mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
408 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
409 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
411 if (IS_ERR(lpm_tree)) {
412 err = PTR_ERR(lpm_tree);
415 vr->lpm_tree = lpm_tree;
416 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
424 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
426 mlxsw_sp_fib_destroy(vr->fib);
431 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
432 struct mlxsw_sp_vr *vr)
434 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
435 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
436 mlxsw_sp_fib_destroy(vr->fib);
441 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
442 struct mlxsw_sp_prefix_usage *req_prefix_usage)
444 struct mlxsw_sp_lpm_tree *lpm_tree;
446 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage,
447 &vr->lpm_tree->prefix_usage))
450 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
452 if (IS_ERR(lpm_tree)) {
453 /* We failed to get a tree according to the required
454 * prefix usage. However, the current tree might be still good
455 * for us if our requirement is subset of the prefixes used
458 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
459 &vr->lpm_tree->prefix_usage))
461 return PTR_ERR(lpm_tree);
464 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
465 mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
466 vr->lpm_tree = lpm_tree;
467 return mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
470 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
471 unsigned char prefix_len,
473 enum mlxsw_sp_l3proto proto)
475 struct mlxsw_sp_vr *vr;
478 tb_id = mlxsw_sp_fix_tb_id(tb_id);
479 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
481 vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
485 struct mlxsw_sp_prefix_usage req_prefix_usage;
487 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
488 &vr->fib->prefix_usage);
489 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
490 /* Need to replace LPM tree in case new prefix is required. */
491 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
499 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
501 /* Destroy virtual router entity in case the associated FIB is empty
502 * and allow it to be used for other tables in future. Otherwise,
503 * check if some prefix usage did not disappear and change tree if
504 * that is the case. Note that in case new, smaller tree cannot be
505 * allocated, the original one will be kept being used.
507 if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
508 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
510 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
511 &vr->fib->prefix_usage);
514 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
516 struct mlxsw_sp_vr *vr;
520 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
523 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
524 mlxsw_sp->router.vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
526 if (!mlxsw_sp->router.vrs)
529 for (i = 0; i < max_vrs; i++) {
530 vr = &mlxsw_sp->router.vrs[i];
537 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
539 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
541 /* At this stage we're guaranteed not to have new incoming
542 * FIB notifications and the work queue is free from FIBs
543 * sitting on top of mlxsw netdevs. However, we can still
544 * have other FIBs queued. Flush the queue before flushing
545 * the device's tables. No need for locks, as we're the only
548 mlxsw_core_flush_owq();
549 mlxsw_sp_router_fib_flush(mlxsw_sp);
550 kfree(mlxsw_sp->router.vrs);
553 struct mlxsw_sp_neigh_key {
557 struct mlxsw_sp_neigh_entry {
558 struct list_head rif_list_node;
559 struct rhash_head ht_node;
560 struct mlxsw_sp_neigh_key key;
563 unsigned char ha[ETH_ALEN];
564 struct list_head nexthop_list; /* list of nexthops using
567 struct list_head nexthop_neighs_list_node;
570 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
571 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
572 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
573 .key_len = sizeof(struct mlxsw_sp_neigh_key),
576 static struct mlxsw_sp_neigh_entry *
577 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
580 struct mlxsw_sp_neigh_entry *neigh_entry;
582 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
586 neigh_entry->key.n = n;
587 neigh_entry->rif = rif;
588 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
593 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
599 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
600 struct mlxsw_sp_neigh_entry *neigh_entry)
602 return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
603 &neigh_entry->ht_node,
604 mlxsw_sp_neigh_ht_params);
608 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
609 struct mlxsw_sp_neigh_entry *neigh_entry)
611 rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
612 &neigh_entry->ht_node,
613 mlxsw_sp_neigh_ht_params);
616 static struct mlxsw_sp_neigh_entry *
617 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
619 struct mlxsw_sp_neigh_entry *neigh_entry;
620 struct mlxsw_sp_rif *r;
623 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
625 return ERR_PTR(-EINVAL);
627 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, r->rif);
629 return ERR_PTR(-ENOMEM);
631 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
633 goto err_neigh_entry_insert;
635 list_add(&neigh_entry->rif_list_node, &r->neigh_list);
639 err_neigh_entry_insert:
640 mlxsw_sp_neigh_entry_free(neigh_entry);
645 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
646 struct mlxsw_sp_neigh_entry *neigh_entry)
648 list_del(&neigh_entry->rif_list_node);
649 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
650 mlxsw_sp_neigh_entry_free(neigh_entry);
653 static struct mlxsw_sp_neigh_entry *
654 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
656 struct mlxsw_sp_neigh_key key;
659 return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
660 &key, mlxsw_sp_neigh_ht_params);
664 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
666 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
668 mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
671 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
675 struct net_device *dev;
681 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
683 if (!mlxsw_sp->rifs[rif]) {
684 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
689 dev = mlxsw_sp->rifs[rif]->dev;
690 n = neigh_lookup(&arp_tbl, &dipn, dev);
692 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
697 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
698 neigh_event_send(n, NULL);
702 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
709 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
711 /* Hardware starts counting at 0, so add 1. */
714 /* Each record consists of several neighbour entries. */
715 for (i = 0; i < num_entries; i++) {
718 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
719 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
725 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
726 char *rauhtd_pl, int rec_index)
728 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
729 case MLXSW_REG_RAUHTD_TYPE_IPV4:
730 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
733 case MLXSW_REG_RAUHTD_TYPE_IPV6:
739 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
741 u8 num_rec, last_rec_index, num_entries;
743 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
744 last_rec_index = num_rec - 1;
746 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
748 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
749 MLXSW_REG_RAUHTD_TYPE_IPV6)
752 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
754 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
759 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
765 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
769 /* Make sure the neighbour's netdev isn't removed in the
774 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
775 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
778 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
781 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
782 for (i = 0; i < num_rec; i++)
783 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
785 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
792 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
794 struct mlxsw_sp_neigh_entry *neigh_entry;
796 /* Take RTNL mutex here to prevent lists from changes */
798 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
799 nexthop_neighs_list_node)
800 /* If this neigh have nexthops, make the kernel think this neigh
801 * is active regardless of the traffic.
803 neigh_event_send(neigh_entry->key.n, NULL);
808 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
810 unsigned long interval = mlxsw_sp->router.neighs_update.interval;
812 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
813 msecs_to_jiffies(interval));
816 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
818 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
819 router.neighs_update.dw.work);
822 err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
824 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
826 mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
828 mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
831 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
833 struct mlxsw_sp_neigh_entry *neigh_entry;
834 struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
835 router.nexthop_probe_dw.work);
837 /* Iterate over nexthop neighbours, find those who are unresolved and
838 * send arp on them. This solves the chicken-egg problem when
839 * the nexthop wouldn't get offloaded until the neighbor is resolved
840 * but it wouldn't get resolved ever in case traffic is flowing in HW
841 * using different nexthop.
843 * Take RTNL mutex here to prevent lists from changes.
846 list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
847 nexthop_neighs_list_node)
848 if (!neigh_entry->connected)
849 neigh_event_send(neigh_entry->key.n, NULL);
852 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
853 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
857 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
858 struct mlxsw_sp_neigh_entry *neigh_entry,
861 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
863 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
864 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
868 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
869 struct mlxsw_sp_neigh_entry *neigh_entry,
870 enum mlxsw_reg_rauht_op op)
872 struct neighbour *n = neigh_entry->key.n;
873 u32 dip = ntohl(*((__be32 *) n->primary_key));
874 char rauht_pl[MLXSW_REG_RAUHT_LEN];
876 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
878 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
882 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
883 struct mlxsw_sp_neigh_entry *neigh_entry,
886 if (!adding && !neigh_entry->connected)
888 neigh_entry->connected = adding;
889 if (neigh_entry->key.n->tbl == &arp_tbl)
890 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
891 mlxsw_sp_rauht_op(adding));
896 struct mlxsw_sp_neigh_event_work {
897 struct work_struct work;
898 struct mlxsw_sp *mlxsw_sp;
902 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
904 struct mlxsw_sp_neigh_event_work *neigh_work =
905 container_of(work, struct mlxsw_sp_neigh_event_work, work);
906 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
907 struct mlxsw_sp_neigh_entry *neigh_entry;
908 struct neighbour *n = neigh_work->n;
909 unsigned char ha[ETH_ALEN];
910 bool entry_connected;
913 /* If these parameters are changed after we release the lock,
914 * then we are guaranteed to receive another event letting us
917 read_lock_bh(&n->lock);
918 memcpy(ha, n->ha, ETH_ALEN);
919 nud_state = n->nud_state;
921 read_unlock_bh(&n->lock);
924 entry_connected = nud_state & NUD_VALID && !dead;
925 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
926 if (!entry_connected && !neigh_entry)
929 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
930 if (IS_ERR(neigh_entry))
934 memcpy(neigh_entry->ha, ha, ETH_ALEN);
935 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
936 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
938 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
939 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
947 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
948 unsigned long event, void *ptr)
950 struct mlxsw_sp_neigh_event_work *neigh_work;
951 struct mlxsw_sp_port *mlxsw_sp_port;
952 struct mlxsw_sp *mlxsw_sp;
953 unsigned long interval;
954 struct neigh_parms *p;
958 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
961 /* We don't care about changes in the default table. */
962 if (!p->dev || p->tbl != &arp_tbl)
965 /* We are in atomic context and can't take RTNL mutex,
966 * so use RCU variant to walk the device chain.
968 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
972 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
973 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
974 mlxsw_sp->router.neighs_update.interval = interval;
976 mlxsw_sp_port_dev_put(mlxsw_sp_port);
978 case NETEVENT_NEIGH_UPDATE:
981 if (n->tbl != &arp_tbl)
984 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
988 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
990 mlxsw_sp_port_dev_put(mlxsw_sp_port);
994 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
995 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
998 /* Take a reference to ensure the neighbour won't be
999 * destructed until we drop the reference in delayed
1003 mlxsw_core_schedule_work(&neigh_work->work);
1004 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1011 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1015 err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1016 &mlxsw_sp_neigh_ht_params);
1020 /* Initialize the polling interval according to the default
1023 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1025 /* Create the delayed works for the activity_update */
1026 INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1027 mlxsw_sp_router_neighs_update_work);
1028 INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1029 mlxsw_sp_router_probe_unresolved_nexthops);
1030 mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1031 mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1035 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1037 cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1038 cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1039 rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1042 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1043 const struct mlxsw_sp_rif *r)
1045 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1047 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1049 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1052 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1053 struct mlxsw_sp_rif *r)
1055 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1057 mlxsw_sp_neigh_rif_flush(mlxsw_sp, r);
1058 list_for_each_entry_safe(neigh_entry, tmp, &r->neigh_list,
1060 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1063 struct mlxsw_sp_nexthop_key {
1064 struct fib_nh *fib_nh;
1067 struct mlxsw_sp_nexthop {
1068 struct list_head neigh_list_node; /* member of neigh entry list */
1069 struct list_head rif_list_node;
1070 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1073 struct rhash_head ht_node;
1074 struct mlxsw_sp_nexthop_key key;
1075 struct mlxsw_sp_rif *r;
1076 u8 should_offload:1, /* set indicates this neigh is connected and
1077 * should be put to KVD linear area of this group.
1079 offloaded:1, /* set in case the neigh is actually put into
1080 * KVD linear area of this group.
1082 update:1; /* set indicates that MAC of this neigh should be
1085 struct mlxsw_sp_neigh_entry *neigh_entry;
1088 struct mlxsw_sp_nexthop_group_key {
1089 struct fib_info *fi;
1092 struct mlxsw_sp_nexthop_group {
1093 struct rhash_head ht_node;
1094 struct list_head fib_list; /* list of fib entries that use this group */
1095 struct mlxsw_sp_nexthop_group_key key;
1096 u8 adj_index_valid:1,
1097 gateway:1; /* routes using the group use a gateway */
1101 struct mlxsw_sp_nexthop nexthops[0];
1102 #define nh_rif nexthops[0].r
1105 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1106 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1107 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1108 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1111 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1112 struct mlxsw_sp_nexthop_group *nh_grp)
1114 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_group_ht,
1116 mlxsw_sp_nexthop_group_ht_params);
1119 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1120 struct mlxsw_sp_nexthop_group *nh_grp)
1122 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_group_ht,
1124 mlxsw_sp_nexthop_group_ht_params);
1127 static struct mlxsw_sp_nexthop_group *
1128 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1129 struct mlxsw_sp_nexthop_group_key key)
1131 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_group_ht, &key,
1132 mlxsw_sp_nexthop_group_ht_params);
1135 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1136 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1137 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1138 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1141 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1142 struct mlxsw_sp_nexthop *nh)
1144 return rhashtable_insert_fast(&mlxsw_sp->router.nexthop_ht,
1145 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1148 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1149 struct mlxsw_sp_nexthop *nh)
1151 rhashtable_remove_fast(&mlxsw_sp->router.nexthop_ht, &nh->ht_node,
1152 mlxsw_sp_nexthop_ht_params);
1155 static struct mlxsw_sp_nexthop *
1156 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1157 struct mlxsw_sp_nexthop_key key)
1159 return rhashtable_lookup_fast(&mlxsw_sp->router.nexthop_ht, &key,
1160 mlxsw_sp_nexthop_ht_params);
1163 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1164 struct mlxsw_sp_vr *vr,
1165 u32 adj_index, u16 ecmp_size,
1169 char raleu_pl[MLXSW_REG_RALEU_LEN];
1171 mlxsw_reg_raleu_pack(raleu_pl,
1172 (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
1173 adj_index, ecmp_size, new_adj_index,
1175 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1178 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1179 struct mlxsw_sp_nexthop_group *nh_grp,
1180 u32 old_adj_index, u16 old_ecmp_size)
1182 struct mlxsw_sp_fib_entry *fib_entry;
1183 struct mlxsw_sp_vr *vr = NULL;
1186 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1187 if (vr == fib_entry->fib_node->vr)
1189 vr = fib_entry->fib_node->vr;
1190 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
1201 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1202 struct mlxsw_sp_nexthop *nh)
1204 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1205 char ratr_pl[MLXSW_REG_RATR_LEN];
1207 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1208 true, adj_index, neigh_entry->rif);
1209 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1210 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1214 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1215 struct mlxsw_sp_nexthop_group *nh_grp,
1218 u32 adj_index = nh_grp->adj_index; /* base */
1219 struct mlxsw_sp_nexthop *nh;
1223 for (i = 0; i < nh_grp->count; i++) {
1224 nh = &nh_grp->nexthops[i];
1226 if (!nh->should_offload) {
1231 if (nh->update || reallocate) {
1232 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1244 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1245 struct mlxsw_sp_fib_entry *fib_entry);
1248 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1249 struct mlxsw_sp_nexthop_group *nh_grp)
1251 struct mlxsw_sp_fib_entry *fib_entry;
1254 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1255 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1263 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1264 struct mlxsw_sp_nexthop_group *nh_grp)
1266 struct mlxsw_sp_nexthop *nh;
1267 bool offload_change = false;
1270 bool old_adj_index_valid;
1277 if (!nh_grp->gateway) {
1278 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1282 for (i = 0; i < nh_grp->count; i++) {
1283 nh = &nh_grp->nexthops[i];
1285 if (nh->should_offload ^ nh->offloaded) {
1286 offload_change = true;
1287 if (nh->should_offload)
1290 if (nh->should_offload)
1293 if (!offload_change) {
1294 /* Nothing was added or removed, so no need to reallocate. Just
1295 * update MAC on existing adjacency indexes.
1297 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1300 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1306 /* No neigh of this group is connected so we just set
1307 * the trap and let everthing flow through kernel.
1311 ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1313 /* We ran out of KVD linear space, just set the
1314 * trap and let everything flow through kernel.
1316 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1320 old_adj_index_valid = nh_grp->adj_index_valid;
1321 old_adj_index = nh_grp->adj_index;
1322 old_ecmp_size = nh_grp->ecmp_size;
1323 nh_grp->adj_index_valid = 1;
1324 nh_grp->adj_index = adj_index;
1325 nh_grp->ecmp_size = ecmp_size;
1326 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1328 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1332 if (!old_adj_index_valid) {
1333 /* The trap was set for fib entries, so we have to call
1334 * fib entry update to unset it and use adjacency index.
1336 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1338 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1344 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1345 old_adj_index, old_ecmp_size);
1346 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1348 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1354 old_adj_index_valid = nh_grp->adj_index_valid;
1355 nh_grp->adj_index_valid = 0;
1356 for (i = 0; i < nh_grp->count; i++) {
1357 nh = &nh_grp->nexthops[i];
1360 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1362 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1363 if (old_adj_index_valid)
1364 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1367 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1370 if (!removing && !nh->should_offload)
1371 nh->should_offload = 1;
1372 else if (removing && nh->offloaded)
1373 nh->should_offload = 0;
1378 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1379 struct mlxsw_sp_neigh_entry *neigh_entry,
1382 struct mlxsw_sp_nexthop *nh;
1384 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1386 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1387 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1391 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1392 struct mlxsw_sp_rif *r)
1398 list_add(&nh->rif_list_node, &r->nexthop_list);
1401 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1406 list_del(&nh->rif_list_node);
1410 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1411 struct mlxsw_sp_nexthop *nh)
1413 struct mlxsw_sp_neigh_entry *neigh_entry;
1414 struct fib_nh *fib_nh = nh->key.fib_nh;
1415 struct neighbour *n;
1419 if (!nh->nh_grp->gateway || nh->neigh_entry)
1422 /* Take a reference of neigh here ensuring that neigh would
1423 * not be detructed before the nexthop entry is finished.
1424 * The reference is taken either in neigh_lookup() or
1425 * in neigh_create() in case n is not found.
1427 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1429 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1432 neigh_event_send(n, NULL);
1434 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1436 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1437 if (IS_ERR(neigh_entry)) {
1439 goto err_neigh_entry_create;
1443 /* If that is the first nexthop connected to that neigh, add to
1444 * nexthop_neighs_list
1446 if (list_empty(&neigh_entry->nexthop_list))
1447 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1448 &mlxsw_sp->router.nexthop_neighs_list);
1450 nh->neigh_entry = neigh_entry;
1451 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1452 read_lock_bh(&n->lock);
1453 nud_state = n->nud_state;
1455 read_unlock_bh(&n->lock);
1456 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1460 err_neigh_entry_create:
1465 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1466 struct mlxsw_sp_nexthop *nh)
1468 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1469 struct neighbour *n;
1473 n = neigh_entry->key.n;
1475 __mlxsw_sp_nexthop_neigh_update(nh, true);
1476 list_del(&nh->neigh_list_node);
1477 nh->neigh_entry = NULL;
1479 /* If that is the last nexthop connected to that neigh, remove from
1480 * nexthop_neighs_list
1482 if (list_empty(&neigh_entry->nexthop_list))
1483 list_del(&neigh_entry->nexthop_neighs_list_node);
1485 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1486 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1491 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1492 struct mlxsw_sp_nexthop_group *nh_grp,
1493 struct mlxsw_sp_nexthop *nh,
1494 struct fib_nh *fib_nh)
1496 struct net_device *dev = fib_nh->nh_dev;
1497 struct in_device *in_dev;
1498 struct mlxsw_sp_rif *r;
1501 nh->nh_grp = nh_grp;
1502 nh->key.fib_nh = fib_nh;
1503 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1507 in_dev = __in_dev_get_rtnl(dev);
1508 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1509 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1512 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1515 mlxsw_sp_nexthop_rif_init(nh, r);
1517 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1519 goto err_nexthop_neigh_init;
1523 err_nexthop_neigh_init:
1524 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1528 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1529 struct mlxsw_sp_nexthop *nh)
1531 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1532 mlxsw_sp_nexthop_rif_fini(nh);
1533 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1536 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1537 unsigned long event, struct fib_nh *fib_nh)
1539 struct mlxsw_sp_nexthop_key key;
1540 struct mlxsw_sp_nexthop *nh;
1541 struct mlxsw_sp_rif *r;
1543 if (mlxsw_sp->router.aborted)
1546 key.fib_nh = fib_nh;
1547 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1548 if (WARN_ON_ONCE(!nh))
1551 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1556 case FIB_EVENT_NH_ADD:
1557 mlxsw_sp_nexthop_rif_init(nh, r);
1558 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1560 case FIB_EVENT_NH_DEL:
1561 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1562 mlxsw_sp_nexthop_rif_fini(nh);
1566 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1569 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1570 struct mlxsw_sp_rif *r)
1572 struct mlxsw_sp_nexthop *nh, *tmp;
1574 list_for_each_entry_safe(nh, tmp, &r->nexthop_list, rif_list_node) {
1575 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1576 mlxsw_sp_nexthop_rif_fini(nh);
1577 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1581 static struct mlxsw_sp_nexthop_group *
1582 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1584 struct mlxsw_sp_nexthop_group *nh_grp;
1585 struct mlxsw_sp_nexthop *nh;
1586 struct fib_nh *fib_nh;
1591 alloc_size = sizeof(*nh_grp) +
1592 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1593 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1595 return ERR_PTR(-ENOMEM);
1596 INIT_LIST_HEAD(&nh_grp->fib_list);
1597 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1598 nh_grp->count = fi->fib_nhs;
1599 nh_grp->key.fi = fi;
1600 for (i = 0; i < nh_grp->count; i++) {
1601 nh = &nh_grp->nexthops[i];
1602 fib_nh = &fi->fib_nh[i];
1603 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1605 goto err_nexthop_init;
1607 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1609 goto err_nexthop_group_insert;
1610 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1613 err_nexthop_group_insert:
1615 for (i--; i >= 0; i--) {
1616 nh = &nh_grp->nexthops[i];
1617 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1620 return ERR_PTR(err);
1624 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1625 struct mlxsw_sp_nexthop_group *nh_grp)
1627 struct mlxsw_sp_nexthop *nh;
1630 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1631 for (i = 0; i < nh_grp->count; i++) {
1632 nh = &nh_grp->nexthops[i];
1633 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1635 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1636 WARN_ON_ONCE(nh_grp->adj_index_valid);
1640 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1641 struct mlxsw_sp_fib_entry *fib_entry,
1642 struct fib_info *fi)
1644 struct mlxsw_sp_nexthop_group_key key;
1645 struct mlxsw_sp_nexthop_group *nh_grp;
1648 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1650 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1652 return PTR_ERR(nh_grp);
1654 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1655 fib_entry->nh_group = nh_grp;
1659 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1660 struct mlxsw_sp_fib_entry *fib_entry)
1662 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1664 list_del(&fib_entry->nexthop_group_node);
1665 if (!list_empty(&nh_grp->fib_list))
1667 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1671 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1673 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1675 if (fib_entry->params.tos)
1678 switch (fib_entry->type) {
1679 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1680 return !!nh_group->adj_index_valid;
1681 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1682 return !!nh_group->nh_rif;
1688 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1690 fib_entry->offloaded = true;
1692 switch (fib_entry->fib_node->vr->proto) {
1693 case MLXSW_SP_L3_PROTO_IPV4:
1694 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1696 case MLXSW_SP_L3_PROTO_IPV6:
1702 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1704 switch (fib_entry->fib_node->vr->proto) {
1705 case MLXSW_SP_L3_PROTO_IPV4:
1706 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1708 case MLXSW_SP_L3_PROTO_IPV6:
1712 fib_entry->offloaded = false;
1716 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1717 enum mlxsw_reg_ralue_op op, int err)
1720 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1721 if (!fib_entry->offloaded)
1723 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1724 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
1727 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1728 !fib_entry->offloaded)
1729 mlxsw_sp_fib_entry_offload_set(fib_entry);
1730 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
1731 fib_entry->offloaded)
1732 mlxsw_sp_fib_entry_offload_unset(fib_entry);
1739 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1740 struct mlxsw_sp_fib_entry *fib_entry,
1741 enum mlxsw_reg_ralue_op op)
1743 char ralue_pl[MLXSW_REG_RALUE_LEN];
1744 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1745 struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
1746 enum mlxsw_reg_ralue_trap_action trap_action;
1748 u32 adjacency_index = 0;
1751 /* In case the nexthop group adjacency index is valid, use it
1752 * with provided ECMP size. Otherwise, setup trap and pass
1753 * traffic to kernel.
1755 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1756 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1757 adjacency_index = fib_entry->nh_group->adj_index;
1758 ecmp_size = fib_entry->nh_group->ecmp_size;
1760 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1761 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1764 mlxsw_reg_ralue_pack4(ralue_pl,
1765 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1766 vr->id, fib_entry->fib_node->key.prefix_len,
1768 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1769 adjacency_index, ecmp_size);
1770 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1773 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1774 struct mlxsw_sp_fib_entry *fib_entry,
1775 enum mlxsw_reg_ralue_op op)
1777 struct mlxsw_sp_rif *r = fib_entry->nh_group->nh_rif;
1778 enum mlxsw_reg_ralue_trap_action trap_action;
1779 char ralue_pl[MLXSW_REG_RALUE_LEN];
1780 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1781 struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
1785 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
1786 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1789 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1790 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1793 mlxsw_reg_ralue_pack4(ralue_pl,
1794 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1795 vr->id, fib_entry->fib_node->key.prefix_len,
1797 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, rif);
1798 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1801 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1802 struct mlxsw_sp_fib_entry *fib_entry,
1803 enum mlxsw_reg_ralue_op op)
1805 char ralue_pl[MLXSW_REG_RALUE_LEN];
1806 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
1807 struct mlxsw_sp_vr *vr = fib_entry->fib_node->vr;
1809 mlxsw_reg_ralue_pack4(ralue_pl,
1810 (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1811 vr->id, fib_entry->fib_node->key.prefix_len,
1813 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1814 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1817 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1818 struct mlxsw_sp_fib_entry *fib_entry,
1819 enum mlxsw_reg_ralue_op op)
1821 switch (fib_entry->type) {
1822 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1823 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1824 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1825 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1826 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1827 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1832 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1833 struct mlxsw_sp_fib_entry *fib_entry,
1834 enum mlxsw_reg_ralue_op op)
1838 switch (fib_entry->fib_node->vr->proto) {
1839 case MLXSW_SP_L3_PROTO_IPV4:
1840 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1842 case MLXSW_SP_L3_PROTO_IPV6:
1845 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
1849 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1850 struct mlxsw_sp_fib_entry *fib_entry)
1852 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1853 MLXSW_REG_RALUE_OP_WRITE_WRITE);
1856 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1857 struct mlxsw_sp_fib_entry *fib_entry)
1859 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1860 MLXSW_REG_RALUE_OP_WRITE_DELETE);
1864 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
1865 const struct fib_entry_notifier_info *fen_info,
1866 struct mlxsw_sp_fib_entry *fib_entry)
1868 struct fib_info *fi = fen_info->fi;
1870 if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
1871 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1874 if (fen_info->type != RTN_UNICAST)
1876 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
1877 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1879 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1883 static struct mlxsw_sp_fib_entry *
1884 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
1885 struct mlxsw_sp_fib_node *fib_node,
1886 const struct fib_entry_notifier_info *fen_info)
1888 struct mlxsw_sp_fib_entry *fib_entry;
1891 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
1894 goto err_fib_entry_alloc;
1897 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
1899 goto err_fib4_entry_type_set;
1901 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
1903 goto err_nexthop_group_get;
1905 fib_entry->params.prio = fen_info->fi->fib_priority;
1906 fib_entry->params.tb_id = fen_info->tb_id;
1907 fib_entry->params.type = fen_info->type;
1908 fib_entry->params.tos = fen_info->tos;
1910 fib_entry->fib_node = fib_node;
1914 err_nexthop_group_get:
1915 err_fib4_entry_type_set:
1917 err_fib_entry_alloc:
1918 return ERR_PTR(err);
1921 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1922 struct mlxsw_sp_fib_entry *fib_entry)
1924 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1928 static struct mlxsw_sp_fib_node *
1929 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
1930 const struct fib_entry_notifier_info *fen_info);
1932 static struct mlxsw_sp_fib_entry *
1933 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
1934 const struct fib_entry_notifier_info *fen_info)
1936 struct mlxsw_sp_fib_entry *fib_entry;
1937 struct mlxsw_sp_fib_node *fib_node;
1939 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
1940 if (IS_ERR(fib_node))
1943 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
1944 if (fib_entry->params.tb_id == fen_info->tb_id &&
1945 fib_entry->params.tos == fen_info->tos &&
1946 fib_entry->params.type == fen_info->type &&
1947 fib_entry->nh_group->key.fi == fen_info->fi) {
1955 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
1956 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
1957 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
1958 .key_len = sizeof(struct mlxsw_sp_fib_key),
1959 .automatic_shrinking = true,
1962 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
1963 struct mlxsw_sp_fib_node *fib_node)
1965 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
1966 mlxsw_sp_fib_ht_params);
1969 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
1970 struct mlxsw_sp_fib_node *fib_node)
1972 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
1973 mlxsw_sp_fib_ht_params);
1976 static struct mlxsw_sp_fib_node *
1977 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1978 size_t addr_len, unsigned char prefix_len)
1980 struct mlxsw_sp_fib_key key;
1982 memset(&key, 0, sizeof(key));
1983 memcpy(key.addr, addr, addr_len);
1984 key.prefix_len = prefix_len;
1985 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
1988 static struct mlxsw_sp_fib_node *
1989 mlxsw_sp_fib_node_create(struct mlxsw_sp_vr *vr, const void *addr,
1990 size_t addr_len, unsigned char prefix_len)
1992 struct mlxsw_sp_fib_node *fib_node;
1994 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
1998 INIT_LIST_HEAD(&fib_node->entry_list);
1999 list_add(&fib_node->list, &vr->fib->node_list);
2000 memcpy(fib_node->key.addr, addr, addr_len);
2001 fib_node->key.prefix_len = prefix_len;
2002 mlxsw_sp_fib_node_insert(vr->fib, fib_node);
2008 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2010 mlxsw_sp_fib_node_remove(fib_node->vr->fib, fib_node);
2011 list_del(&fib_node->list);
2012 WARN_ON(!list_empty(&fib_node->entry_list));
2017 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2018 const struct mlxsw_sp_fib_entry *fib_entry)
2020 return list_first_entry(&fib_node->entry_list,
2021 struct mlxsw_sp_fib_entry, list) == fib_entry;
2024 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2026 unsigned char prefix_len = fib_node->key.prefix_len;
2027 struct mlxsw_sp_fib *fib = fib_node->vr->fib;
2029 if (fib->prefix_ref_count[prefix_len]++ == 0)
2030 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2033 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2035 unsigned char prefix_len = fib_node->key.prefix_len;
2036 struct mlxsw_sp_fib *fib = fib_node->vr->fib;
2038 if (--fib->prefix_ref_count[prefix_len] == 0)
2039 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2042 static struct mlxsw_sp_fib_node *
2043 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2044 const struct fib_entry_notifier_info *fen_info)
2046 struct mlxsw_sp_fib_node *fib_node;
2047 struct mlxsw_sp_vr *vr;
2050 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
2051 MLXSW_SP_L3_PROTO_IPV4);
2053 return ERR_CAST(vr);
2055 fib_node = mlxsw_sp_fib_node_lookup(vr->fib, &fen_info->dst,
2056 sizeof(fen_info->dst),
2061 fib_node = mlxsw_sp_fib_node_create(vr, &fen_info->dst,
2062 sizeof(fen_info->dst),
2066 goto err_fib_node_create;
2071 err_fib_node_create:
2072 mlxsw_sp_vr_put(mlxsw_sp, vr);
2073 return ERR_PTR(err);
2076 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2077 struct mlxsw_sp_fib_node *fib_node)
2079 struct mlxsw_sp_vr *vr = fib_node->vr;
2081 if (!list_empty(&fib_node->entry_list))
2083 mlxsw_sp_fib_node_destroy(fib_node);
2084 mlxsw_sp_vr_put(mlxsw_sp, vr);
2087 static struct mlxsw_sp_fib_entry *
2088 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2089 const struct mlxsw_sp_fib_entry_params *params)
2091 struct mlxsw_sp_fib_entry *fib_entry;
2093 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2094 if (fib_entry->params.tb_id > params->tb_id)
2096 if (fib_entry->params.tb_id != params->tb_id)
2098 if (fib_entry->params.tos > params->tos)
2100 if (fib_entry->params.prio >= params->prio ||
2101 fib_entry->params.tos < params->tos)
2108 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2109 struct mlxsw_sp_fib_entry *new_entry)
2111 struct mlxsw_sp_fib_node *fib_node;
2113 if (WARN_ON(!fib_entry))
2116 fib_node = fib_entry->fib_node;
2117 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2118 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2119 fib_entry->params.tos != new_entry->params.tos ||
2120 fib_entry->params.prio != new_entry->params.prio)
2124 list_add_tail(&new_entry->list, &fib_entry->list);
2129 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2130 struct mlxsw_sp_fib_entry *new_entry,
2131 bool replace, bool append)
2133 struct mlxsw_sp_fib_entry *fib_entry;
2135 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2138 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2139 if (replace && WARN_ON(!fib_entry))
2142 /* Insert new entry before replaced one, so that we can later
2143 * remove the second.
2146 list_add_tail(&new_entry->list, &fib_entry->list);
2148 struct mlxsw_sp_fib_entry *last;
2150 list_for_each_entry(last, &fib_node->entry_list, list) {
2151 if (new_entry->params.tb_id > last->params.tb_id)
2157 list_add(&new_entry->list, &fib_entry->list);
2159 list_add(&new_entry->list, &fib_node->entry_list);
2166 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2168 list_del(&fib_entry->list);
2172 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2173 const struct mlxsw_sp_fib_node *fib_node,
2174 struct mlxsw_sp_fib_entry *fib_entry)
2176 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2179 /* To prevent packet loss, overwrite the previously offloaded
2182 if (!list_is_singular(&fib_node->entry_list)) {
2183 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2184 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2186 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2189 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2193 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2194 const struct mlxsw_sp_fib_node *fib_node,
2195 struct mlxsw_sp_fib_entry *fib_entry)
2197 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2200 /* Promote the next entry by overwriting the deleted entry */
2201 if (!list_is_singular(&fib_node->entry_list)) {
2202 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2203 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2205 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2206 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2210 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2213 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2214 struct mlxsw_sp_fib_entry *fib_entry,
2215 bool replace, bool append)
2217 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2220 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2225 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2227 goto err_fib4_node_entry_add;
2229 mlxsw_sp_fib_node_prefix_inc(fib_node);
2233 err_fib4_node_entry_add:
2234 mlxsw_sp_fib4_node_list_remove(fib_entry);
2239 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2240 struct mlxsw_sp_fib_entry *fib_entry)
2242 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2244 mlxsw_sp_fib_node_prefix_dec(fib_node);
2245 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2246 mlxsw_sp_fib4_node_list_remove(fib_entry);
2249 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2250 struct mlxsw_sp_fib_entry *fib_entry,
2253 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2254 struct mlxsw_sp_fib_entry *replaced;
2259 /* We inserted the new entry before replaced one */
2260 replaced = list_next_entry(fib_entry, list);
2262 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2263 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2264 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2268 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2269 const struct fib_entry_notifier_info *fen_info,
2270 bool replace, bool append)
2272 struct mlxsw_sp_fib_entry *fib_entry;
2273 struct mlxsw_sp_fib_node *fib_node;
2276 if (mlxsw_sp->router.aborted)
2279 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2280 if (IS_ERR(fib_node)) {
2281 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2282 return PTR_ERR(fib_node);
2285 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2286 if (IS_ERR(fib_entry)) {
2287 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2288 err = PTR_ERR(fib_entry);
2289 goto err_fib4_entry_create;
2292 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2295 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2296 goto err_fib4_node_entry_link;
2299 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2303 err_fib4_node_entry_link:
2304 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2305 err_fib4_entry_create:
2306 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2310 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2311 struct fib_entry_notifier_info *fen_info)
2313 struct mlxsw_sp_fib_entry *fib_entry;
2314 struct mlxsw_sp_fib_node *fib_node;
2316 if (mlxsw_sp->router.aborted)
2319 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2320 if (WARN_ON(!fib_entry))
2322 fib_node = fib_entry->fib_node;
2324 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2325 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2326 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2329 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2331 char ralta_pl[MLXSW_REG_RALTA_LEN];
2332 char ralst_pl[MLXSW_REG_RALST_LEN];
2333 char raltb_pl[MLXSW_REG_RALTB_LEN];
2334 char ralue_pl[MLXSW_REG_RALUE_LEN];
2337 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2338 MLXSW_SP_LPM_TREE_MIN);
2339 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2343 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2344 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2348 mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2349 MLXSW_SP_LPM_TREE_MIN);
2350 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
2354 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2355 MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
2356 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2357 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2360 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2361 struct mlxsw_sp_fib_node *fib_node)
2363 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2365 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2366 bool do_break = &tmp->list == &fib_node->entry_list;
2368 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2369 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2370 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2371 /* Break when entry list is empty and node was freed.
2372 * Otherwise, we'll access freed memory in the next
2380 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2381 struct mlxsw_sp_fib_node *fib_node)
2383 switch (fib_node->vr->proto) {
2384 case MLXSW_SP_L3_PROTO_IPV4:
2385 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2387 case MLXSW_SP_L3_PROTO_IPV6:
2393 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2395 struct mlxsw_sp_fib_node *fib_node, *tmp;
2396 struct mlxsw_sp_vr *vr;
2399 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2400 vr = &mlxsw_sp->router.vrs[i];
2405 list_for_each_entry_safe(fib_node, tmp, &vr->fib->node_list,
2407 bool do_break = &tmp->list == &vr->fib->node_list;
2409 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2416 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2420 if (mlxsw_sp->router.aborted)
2422 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2423 mlxsw_sp_router_fib_flush(mlxsw_sp);
2424 mlxsw_sp->router.aborted = true;
2425 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2427 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2430 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2432 char ritr_pl[MLXSW_REG_RITR_LEN];
2435 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2436 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2437 if (WARN_ON_ONCE(err))
2440 mlxsw_reg_ritr_enable_set(ritr_pl, false);
2441 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2444 void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2445 struct mlxsw_sp_rif *r)
2447 mlxsw_sp_router_rif_disable(mlxsw_sp, r->rif);
2448 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, r);
2449 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, r);
2452 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
2454 char rgcr_pl[MLXSW_REG_RGCR_LEN];
2458 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
2461 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2462 mlxsw_sp->rifs = kcalloc(max_rifs, sizeof(struct mlxsw_sp_rif *),
2464 if (!mlxsw_sp->rifs)
2467 mlxsw_reg_rgcr_pack(rgcr_pl, true);
2468 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
2469 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
2476 kfree(mlxsw_sp->rifs);
2480 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2482 char rgcr_pl[MLXSW_REG_RGCR_LEN];
2485 mlxsw_reg_rgcr_pack(rgcr_pl, false);
2486 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
2488 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2489 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
2491 kfree(mlxsw_sp->rifs);
2494 struct mlxsw_sp_fib_event_work {
2495 struct work_struct work;
2497 struct fib_entry_notifier_info fen_info;
2498 struct fib_nh_notifier_info fnh_info;
2500 struct mlxsw_sp *mlxsw_sp;
2501 unsigned long event;
2504 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2506 struct mlxsw_sp_fib_event_work *fib_work =
2507 container_of(work, struct mlxsw_sp_fib_event_work, work);
2508 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2509 bool replace, append;
2512 /* Protect internal structures from changes */
2514 switch (fib_work->event) {
2515 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2516 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2517 case FIB_EVENT_ENTRY_ADD:
2518 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2519 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2520 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2523 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2524 fib_info_put(fib_work->fen_info.fi);
2526 case FIB_EVENT_ENTRY_DEL:
2527 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2528 fib_info_put(fib_work->fen_info.fi);
2530 case FIB_EVENT_RULE_ADD: /* fall through */
2531 case FIB_EVENT_RULE_DEL:
2532 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2534 case FIB_EVENT_NH_ADD: /* fall through */
2535 case FIB_EVENT_NH_DEL:
2536 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2537 fib_work->fnh_info.fib_nh);
2538 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2545 /* Called with rcu_read_lock() */
2546 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2547 unsigned long event, void *ptr)
2549 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2550 struct mlxsw_sp_fib_event_work *fib_work;
2551 struct fib_notifier_info *info = ptr;
2553 if (!net_eq(info->net, &init_net))
2556 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2557 if (WARN_ON(!fib_work))
2560 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2561 fib_work->mlxsw_sp = mlxsw_sp;
2562 fib_work->event = event;
2565 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2566 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2567 case FIB_EVENT_ENTRY_ADD: /* fall through */
2568 case FIB_EVENT_ENTRY_DEL:
2569 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2570 /* Take referece on fib_info to prevent it from being
2571 * freed while work is queued. Release it afterwards.
2573 fib_info_hold(fib_work->fen_info.fi);
2575 case FIB_EVENT_NH_ADD: /* fall through */
2576 case FIB_EVENT_NH_DEL:
2577 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2578 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2582 mlxsw_core_schedule_work(&fib_work->work);
2587 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
2589 struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
2591 /* Flush pending FIB notifications and then flush the device's
2592 * table before requesting another dump. The FIB notification
2593 * block is unregistered, so no need to take RTNL.
2595 mlxsw_core_flush_owq();
2596 mlxsw_sp_router_fib_flush(mlxsw_sp);
2599 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
2603 INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
2604 err = __mlxsw_sp_router_init(mlxsw_sp);
2608 err = rhashtable_init(&mlxsw_sp->router.nexthop_ht,
2609 &mlxsw_sp_nexthop_ht_params);
2611 goto err_nexthop_ht_init;
2613 err = rhashtable_init(&mlxsw_sp->router.nexthop_group_ht,
2614 &mlxsw_sp_nexthop_group_ht_params);
2616 goto err_nexthop_group_ht_init;
2618 mlxsw_sp_lpm_init(mlxsw_sp);
2619 err = mlxsw_sp_vrs_init(mlxsw_sp);
2623 err = mlxsw_sp_neigh_init(mlxsw_sp);
2625 goto err_neigh_init;
2627 mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
2628 err = register_fib_notifier(&mlxsw_sp->fib_nb,
2629 mlxsw_sp_router_fib_dump_flush);
2631 goto err_register_fib_notifier;
2635 err_register_fib_notifier:
2636 mlxsw_sp_neigh_fini(mlxsw_sp);
2638 mlxsw_sp_vrs_fini(mlxsw_sp);
2640 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
2641 err_nexthop_group_ht_init:
2642 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
2643 err_nexthop_ht_init:
2644 __mlxsw_sp_router_fini(mlxsw_sp);
2648 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2650 unregister_fib_notifier(&mlxsw_sp->fib_nb);
2651 mlxsw_sp_neigh_fini(mlxsw_sp);
2652 mlxsw_sp_vrs_fini(mlxsw_sp);
2653 rhashtable_destroy(&mlxsw_sp->router.nexthop_group_ht);
2654 rhashtable_destroy(&mlxsw_sp->router.nexthop_ht);
2655 __mlxsw_sp_router_fini(mlxsw_sp);