2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <net/flow_dissector.h>
34 #include <net/sch_generic.h>
35 #include <net/pkt_cls.h>
36 #include <net/tc_act/tc_gact.h>
37 #include <net/tc_act/tc_skbedit.h>
38 #include <linux/mlx5/fs.h>
39 #include <linux/mlx5/device.h>
40 #include <linux/rhashtable.h>
41 #include <net/switchdev.h>
42 #include <net/tc_act/tc_mirred.h>
43 #include <net/tc_act/tc_vlan.h>
44 #include <net/tc_act/tc_tunnel_key.h>
45 #include <net/vxlan.h>
51 struct mlx5e_tc_flow {
52 struct rhash_head node;
54 struct mlx5_flow_handle *rule;
55 struct list_head encap; /* flows sharing the same encap */
56 struct mlx5_esw_flow_attr *attr;
60 MLX5_HEADER_TYPE_VXLAN = 0x0,
61 MLX5_HEADER_TYPE_NVGRE = 0x1,
64 #define MLX5E_TC_TABLE_NUM_ENTRIES 1024
65 #define MLX5E_TC_TABLE_NUM_GROUPS 4
67 static struct mlx5_flow_handle *
68 mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
69 struct mlx5_flow_spec *spec,
70 u32 action, u32 flow_tag)
72 struct mlx5_core_dev *dev = priv->mdev;
73 struct mlx5_flow_destination dest = { 0 };
74 struct mlx5_flow_act flow_act = {
79 struct mlx5_fc *counter = NULL;
80 struct mlx5_flow_handle *rule;
81 bool table_created = false;
83 if (action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
84 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
85 dest.ft = priv->fs.vlan.ft.t;
86 } else if (action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
87 counter = mlx5_fc_create(dev, true);
89 return ERR_CAST(counter);
91 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
92 dest.counter = counter;
95 if (IS_ERR_OR_NULL(priv->fs.tc.t)) {
97 mlx5_create_auto_grouped_flow_table(priv->fs.ns,
99 MLX5E_TC_TABLE_NUM_ENTRIES,
100 MLX5E_TC_TABLE_NUM_GROUPS,
102 if (IS_ERR(priv->fs.tc.t)) {
103 netdev_err(priv->netdev,
104 "Failed to create tc offload table\n");
105 rule = ERR_CAST(priv->fs.tc.t);
109 table_created = true;
112 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
113 rule = mlx5_add_flow_rules(priv->fs.tc.t, spec, &flow_act, &dest, 1);
122 mlx5_destroy_flow_table(priv->fs.tc.t);
123 priv->fs.tc.t = NULL;
126 mlx5_fc_destroy(dev, counter);
131 static struct mlx5_flow_handle *
132 mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
133 struct mlx5_flow_spec *spec,
134 struct mlx5_esw_flow_attr *attr)
136 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
139 err = mlx5_eswitch_add_vlan_action(esw, attr);
143 return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
146 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
147 struct mlx5e_tc_flow *flow) {
148 struct list_head *next = flow->encap.next;
150 list_del(&flow->encap);
151 if (list_empty(next)) {
152 struct mlx5_encap_entry *e;
154 e = list_entry(next, struct mlx5_encap_entry, flows);
156 mlx5_encap_dealloc(priv->mdev, e->encap_id);
159 hlist_del_rcu(&e->encap_hlist);
164 static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
165 struct mlx5e_tc_flow *flow)
167 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
168 struct mlx5_fc *counter = NULL;
170 counter = mlx5_flow_rule_counter(flow->rule);
172 mlx5_del_flow_rules(flow->rule);
174 if (esw && esw->mode == SRIOV_OFFLOADS) {
175 mlx5_eswitch_del_vlan_action(esw, flow->attr);
176 if (flow->attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP)
177 mlx5e_detach_encap(priv, flow);
180 mlx5_fc_destroy(priv->mdev, counter);
182 if (!mlx5e_tc_num_filters(priv) && (priv->fs.tc.t)) {
183 mlx5_destroy_flow_table(priv->fs.tc.t);
184 priv->fs.tc.t = NULL;
188 static void parse_vxlan_attr(struct mlx5_flow_spec *spec,
189 struct tc_cls_flower_offload *f)
191 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
193 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
195 void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
197 void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
200 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
201 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
203 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
204 struct flow_dissector_key_keyid *key =
205 skb_flow_dissector_target(f->dissector,
206 FLOW_DISSECTOR_KEY_ENC_KEYID,
208 struct flow_dissector_key_keyid *mask =
209 skb_flow_dissector_target(f->dissector,
210 FLOW_DISSECTOR_KEY_ENC_KEYID,
212 MLX5_SET(fte_match_set_misc, misc_c, vxlan_vni,
213 be32_to_cpu(mask->keyid));
214 MLX5_SET(fte_match_set_misc, misc_v, vxlan_vni,
215 be32_to_cpu(key->keyid));
219 static int parse_tunnel_attr(struct mlx5e_priv *priv,
220 struct mlx5_flow_spec *spec,
221 struct tc_cls_flower_offload *f)
223 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
225 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
228 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
229 struct flow_dissector_key_ports *key =
230 skb_flow_dissector_target(f->dissector,
231 FLOW_DISSECTOR_KEY_ENC_PORTS,
233 struct flow_dissector_key_ports *mask =
234 skb_flow_dissector_target(f->dissector,
235 FLOW_DISSECTOR_KEY_ENC_PORTS,
238 /* Full udp dst port must be given */
239 if (memchr_inv(&mask->dst, 0xff, sizeof(mask->dst)))
242 /* udp src port isn't supported */
243 if (memchr_inv(&mask->src, 0, sizeof(mask->src)))
246 if (mlx5e_vxlan_lookup_port(priv, be16_to_cpu(key->dst)) &&
247 MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap))
248 parse_vxlan_attr(spec, f);
252 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
253 udp_dport, ntohs(mask->dst));
254 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
255 udp_dport, ntohs(key->dst));
257 } else { /* udp dst port must be given */
261 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
262 struct flow_dissector_key_ipv4_addrs *key =
263 skb_flow_dissector_target(f->dissector,
264 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
266 struct flow_dissector_key_ipv4_addrs *mask =
267 skb_flow_dissector_target(f->dissector,
268 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
270 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
271 src_ipv4_src_ipv6.ipv4_layout.ipv4,
273 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
274 src_ipv4_src_ipv6.ipv4_layout.ipv4,
277 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
278 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
280 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
281 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
285 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype);
286 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, ETH_P_IP);
288 /* Enforce DMAC when offloading incoming tunneled flows.
289 * Flow counters require a match on the DMAC.
291 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_47_16);
292 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, dmac_15_0);
293 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
294 dmac_47_16), priv->netdev->dev_addr);
296 /* let software handle IP fragments */
297 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
298 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
303 static int __parse_cls_flower(struct mlx5e_priv *priv,
304 struct mlx5_flow_spec *spec,
305 struct tc_cls_flower_offload *f,
308 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
310 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
315 *min_inline = MLX5_INLINE_MODE_L2;
317 if (f->dissector->used_keys &
318 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
319 BIT(FLOW_DISSECTOR_KEY_BASIC) |
320 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
321 BIT(FLOW_DISSECTOR_KEY_VLAN) |
322 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
323 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
324 BIT(FLOW_DISSECTOR_KEY_PORTS) |
325 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
326 BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
327 BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
328 BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
329 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL))) {
330 netdev_warn(priv->netdev, "Unsupported key used: 0x%x\n",
331 f->dissector->used_keys);
335 if ((dissector_uses_key(f->dissector,
336 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) ||
337 dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID) ||
338 dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) &&
339 dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
340 struct flow_dissector_key_control *key =
341 skb_flow_dissector_target(f->dissector,
342 FLOW_DISSECTOR_KEY_ENC_CONTROL,
344 switch (key->addr_type) {
345 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
346 if (parse_tunnel_attr(priv, spec, f))
353 /* In decap flow, header pointers should point to the inner
354 * headers, outer header were already set by parse_tunnel_attr
356 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
358 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
362 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
363 struct flow_dissector_key_control *key =
364 skb_flow_dissector_target(f->dissector,
365 FLOW_DISSECTOR_KEY_CONTROL,
368 struct flow_dissector_key_control *mask =
369 skb_flow_dissector_target(f->dissector,
370 FLOW_DISSECTOR_KEY_CONTROL,
372 addr_type = key->addr_type;
374 if (mask->flags & FLOW_DIS_IS_FRAGMENT) {
375 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
376 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
377 key->flags & FLOW_DIS_IS_FRAGMENT);
381 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) {
382 struct flow_dissector_key_basic *key =
383 skb_flow_dissector_target(f->dissector,
384 FLOW_DISSECTOR_KEY_BASIC,
386 struct flow_dissector_key_basic *mask =
387 skb_flow_dissector_target(f->dissector,
388 FLOW_DISSECTOR_KEY_BASIC,
390 ip_proto = key->ip_proto;
392 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
393 ntohs(mask->n_proto));
394 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
395 ntohs(key->n_proto));
397 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
399 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
403 *min_inline = MLX5_INLINE_MODE_IP;
406 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
407 struct flow_dissector_key_eth_addrs *key =
408 skb_flow_dissector_target(f->dissector,
409 FLOW_DISSECTOR_KEY_ETH_ADDRS,
411 struct flow_dissector_key_eth_addrs *mask =
412 skb_flow_dissector_target(f->dissector,
413 FLOW_DISSECTOR_KEY_ETH_ADDRS,
416 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
419 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
423 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
426 ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
431 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) {
432 struct flow_dissector_key_vlan *key =
433 skb_flow_dissector_target(f->dissector,
434 FLOW_DISSECTOR_KEY_VLAN,
436 struct flow_dissector_key_vlan *mask =
437 skb_flow_dissector_target(f->dissector,
438 FLOW_DISSECTOR_KEY_VLAN,
440 if (mask->vlan_id || mask->vlan_priority) {
441 MLX5_SET(fte_match_set_lyr_2_4, headers_c, vlan_tag, 1);
442 MLX5_SET(fte_match_set_lyr_2_4, headers_v, vlan_tag, 1);
444 MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid, mask->vlan_id);
445 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid, key->vlan_id);
447 MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio, mask->vlan_priority);
448 MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio, key->vlan_priority);
452 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
453 struct flow_dissector_key_ipv4_addrs *key =
454 skb_flow_dissector_target(f->dissector,
455 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
457 struct flow_dissector_key_ipv4_addrs *mask =
458 skb_flow_dissector_target(f->dissector,
459 FLOW_DISSECTOR_KEY_IPV4_ADDRS,
462 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
463 src_ipv4_src_ipv6.ipv4_layout.ipv4),
464 &mask->src, sizeof(mask->src));
465 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
466 src_ipv4_src_ipv6.ipv4_layout.ipv4),
467 &key->src, sizeof(key->src));
468 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
469 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
470 &mask->dst, sizeof(mask->dst));
471 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
472 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
473 &key->dst, sizeof(key->dst));
475 if (mask->src || mask->dst)
476 *min_inline = MLX5_INLINE_MODE_IP;
479 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
480 struct flow_dissector_key_ipv6_addrs *key =
481 skb_flow_dissector_target(f->dissector,
482 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
484 struct flow_dissector_key_ipv6_addrs *mask =
485 skb_flow_dissector_target(f->dissector,
486 FLOW_DISSECTOR_KEY_IPV6_ADDRS,
489 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
490 src_ipv4_src_ipv6.ipv6_layout.ipv6),
491 &mask->src, sizeof(mask->src));
492 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
493 src_ipv4_src_ipv6.ipv6_layout.ipv6),
494 &key->src, sizeof(key->src));
496 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
497 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
498 &mask->dst, sizeof(mask->dst));
499 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
500 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
501 &key->dst, sizeof(key->dst));
503 if (ipv6_addr_type(&mask->src) != IPV6_ADDR_ANY ||
504 ipv6_addr_type(&mask->dst) != IPV6_ADDR_ANY)
505 *min_inline = MLX5_INLINE_MODE_IP;
508 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) {
509 struct flow_dissector_key_ports *key =
510 skb_flow_dissector_target(f->dissector,
511 FLOW_DISSECTOR_KEY_PORTS,
513 struct flow_dissector_key_ports *mask =
514 skb_flow_dissector_target(f->dissector,
515 FLOW_DISSECTOR_KEY_PORTS,
519 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
520 tcp_sport, ntohs(mask->src));
521 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
522 tcp_sport, ntohs(key->src));
524 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
525 tcp_dport, ntohs(mask->dst));
526 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
527 tcp_dport, ntohs(key->dst));
531 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
532 udp_sport, ntohs(mask->src));
533 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
534 udp_sport, ntohs(key->src));
536 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
537 udp_dport, ntohs(mask->dst));
538 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
539 udp_dport, ntohs(key->dst));
542 netdev_err(priv->netdev,
543 "Only UDP and TCP transport are supported\n");
547 if (mask->src || mask->dst)
548 *min_inline = MLX5_INLINE_MODE_TCP_UDP;
554 static int parse_cls_flower(struct mlx5e_priv *priv,
555 struct mlx5_flow_spec *spec,
556 struct tc_cls_flower_offload *f)
558 struct mlx5_core_dev *dev = priv->mdev;
559 struct mlx5_eswitch *esw = dev->priv.eswitch;
560 struct mlx5_eswitch_rep *rep = priv->ppriv;
564 err = __parse_cls_flower(priv, spec, f, &min_inline);
566 if (!err && esw->mode == SRIOV_OFFLOADS &&
567 rep->vport != FDB_UPLINK_VPORT) {
568 if (min_inline > esw->offloads.inline_mode) {
569 netdev_warn(priv->netdev,
570 "Flow is not offloaded due to min inline setting, required %d actual %d\n",
571 min_inline, esw->offloads.inline_mode);
579 static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
580 u32 *action, u32 *flow_tag)
582 const struct tc_action *a;
585 if (tc_no_actions(exts))
588 *flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
591 tcf_exts_to_list(exts, &actions);
592 list_for_each_entry(a, &actions, list) {
593 /* Only support a single action per rule */
597 if (is_tcf_gact_shot(a)) {
598 *action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
599 if (MLX5_CAP_FLOWTABLE(priv->mdev,
600 flow_table_properties_nic_receive.flow_counter))
601 *action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
605 if (is_tcf_skbedit_mark(a)) {
606 u32 mark = tcf_skbedit_mark(a);
608 if (mark & ~MLX5E_TC_FLOW_ID_MASK) {
609 netdev_warn(priv->netdev, "Bad flow mark - only 16 bit is supported: 0x%x\n",
615 *action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
625 static inline int cmp_encap_info(struct mlx5_encap_info *a,
626 struct mlx5_encap_info *b)
628 return memcmp(a, b, sizeof(*a));
631 static inline int hash_encap_info(struct mlx5_encap_info *info)
633 return jhash(info, sizeof(*info), 0);
636 static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
637 struct net_device *mirred_dev,
638 struct net_device **out_dev,
640 struct neighbour **out_n,
645 struct neighbour *n = NULL;
648 #if IS_ENABLED(CONFIG_INET)
649 rt = ip_route_output_key(dev_net(mirred_dev), fl4);
651 pr_warn("%s: no route to %pI4\n", __func__, &fl4->daddr);
658 if (!switchdev_port_same_parent_id(priv->netdev, rt->dst.dev)) {
659 pr_warn("%s: Can't offload the flow, netdevices aren't on the same HW e-switch\n",
665 ttl = ip4_dst_hoplimit(&rt->dst);
666 n = dst_neigh_lookup(&rt->dst, &fl4->daddr);
674 *out_dev = rt->dst.dev;
679 static int gen_vxlan_header_ipv4(struct net_device *out_dev,
681 unsigned char h_dest[ETH_ALEN],
688 int encap_size = VXLAN_HLEN + sizeof(struct iphdr) + ETH_HLEN;
689 struct ethhdr *eth = (struct ethhdr *)buf;
690 struct iphdr *ip = (struct iphdr *)((char *)eth + sizeof(struct ethhdr));
691 struct udphdr *udp = (struct udphdr *)((char *)ip + sizeof(struct iphdr));
692 struct vxlanhdr *vxh = (struct vxlanhdr *)((char *)udp + sizeof(struct udphdr));
694 memset(buf, 0, encap_size);
696 ether_addr_copy(eth->h_dest, h_dest);
697 ether_addr_copy(eth->h_source, out_dev->dev_addr);
698 eth->h_proto = htons(ETH_P_IP);
704 ip->protocol = IPPROTO_UDP;
708 udp->dest = udp_dst_port;
709 vxh->vx_flags = VXLAN_HF_VNI;
710 vxh->vx_vni = vxlan_vni_field(vx_vni);
715 static int mlx5e_create_encap_header_ipv4(struct mlx5e_priv *priv,
716 struct net_device *mirred_dev,
717 struct mlx5_encap_entry *e,
718 struct net_device **out_dev)
720 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
721 struct flowi4 fl4 = {};
729 encap_header = kzalloc(max_encap_size, GFP_KERNEL);
733 switch (e->tunnel_type) {
734 case MLX5_HEADER_TYPE_VXLAN:
735 fl4.flowi4_proto = IPPROTO_UDP;
736 fl4.fl4_dport = e->tun_info.tp_dst;
742 fl4.daddr = e->tun_info.daddr;
744 err = mlx5e_route_lookup_ipv4(priv, mirred_dev, out_dev,
745 &fl4, &n, &saddr, &ttl);
750 e->out_dev = *out_dev;
752 if (!(n->nud_state & NUD_VALID)) {
757 neigh_ha_snapshot(e->h_dest, n, *out_dev);
759 switch (e->tunnel_type) {
760 case MLX5_HEADER_TYPE_VXLAN:
761 encap_size = gen_vxlan_header_ipv4(*out_dev, encap_header,
764 saddr, e->tun_info.tp_dst,
772 err = mlx5_encap_alloc(priv->mdev, e->tunnel_type,
773 encap_size, encap_header, &e->encap_id);
779 static int mlx5e_attach_encap(struct mlx5e_priv *priv,
780 struct ip_tunnel_info *tun_info,
781 struct net_device *mirred_dev,
782 struct mlx5_esw_flow_attr *attr)
784 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
785 unsigned short family = ip_tunnel_info_af(tun_info);
786 struct ip_tunnel_key *key = &tun_info->key;
787 struct mlx5_encap_info info;
788 struct mlx5_encap_entry *e;
789 struct net_device *out_dev;
795 /* udp dst port must be given */
796 if (!memchr_inv(&key->tp_dst, 0, sizeof(key->tp_dst)))
799 if (mlx5e_vxlan_lookup_port(priv, be16_to_cpu(key->tp_dst)) &&
800 MLX5_CAP_ESW(priv->mdev, vxlan_encap_decap)) {
801 info.tp_dst = key->tp_dst;
802 info.tun_id = tunnel_id_to_key32(key->tun_id);
803 tunnel_type = MLX5_HEADER_TYPE_VXLAN;
810 info.daddr = key->u.ipv4.dst;
816 hash_key = hash_encap_info(&info);
818 hash_for_each_possible_rcu(esw->offloads.encap_tbl, e,
819 encap_hlist, hash_key) {
820 if (!cmp_encap_info(&e->tun_info, &info)) {
831 e = kzalloc(sizeof(*e), GFP_KERNEL);
836 e->tunnel_type = tunnel_type;
837 INIT_LIST_HEAD(&e->flows);
839 err = mlx5e_create_encap_header_ipv4(priv, mirred_dev, e, &out_dev);
844 hash_add_rcu(esw->offloads.encap_tbl, &e->encap_hlist, hash_key);
853 static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
854 struct mlx5e_tc_flow *flow)
856 struct mlx5_esw_flow_attr *attr = flow->attr;
857 struct ip_tunnel_info *info = NULL;
858 const struct tc_action *a;
863 if (tc_no_actions(exts))
866 memset(attr, 0, sizeof(*attr));
867 attr->in_rep = priv->ppriv;
869 tcf_exts_to_list(exts, &actions);
870 list_for_each_entry(a, &actions, list) {
871 if (is_tcf_gact_shot(a)) {
872 attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP |
873 MLX5_FLOW_CONTEXT_ACTION_COUNT;
877 if (is_tcf_mirred_egress_redirect(a)) {
878 int ifindex = tcf_mirred_ifindex(a);
879 struct net_device *out_dev;
880 struct mlx5e_priv *out_priv;
882 out_dev = __dev_get_by_index(dev_net(priv->netdev), ifindex);
884 if (switchdev_port_same_parent_id(priv->netdev,
886 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
887 MLX5_FLOW_CONTEXT_ACTION_COUNT;
888 out_priv = netdev_priv(out_dev);
889 attr->out_rep = out_priv->ppriv;
891 err = mlx5e_attach_encap(priv, info,
895 list_add(&flow->encap, &attr->encap->flows);
896 attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
897 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
898 MLX5_FLOW_CONTEXT_ACTION_COUNT;
899 out_priv = netdev_priv(attr->encap->out_dev);
900 attr->out_rep = out_priv->ppriv;
902 pr_err("devices %s %s not on same switch HW, can't offload forwarding\n",
903 priv->netdev->name, out_dev->name);
909 if (is_tcf_tunnel_set(a)) {
910 info = tcf_tunnel_info(a);
918 if (is_tcf_vlan(a)) {
919 if (tcf_vlan_action(a) == VLAN_F_POP) {
920 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
921 } else if (tcf_vlan_action(a) == VLAN_F_PUSH) {
922 if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
925 attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
926 attr->vlan = tcf_vlan_push_vid(a);
931 if (is_tcf_tunnel_release(a)) {
932 attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
941 int mlx5e_configure_flower(struct mlx5e_priv *priv, __be16 protocol,
942 struct tc_cls_flower_offload *f)
944 struct mlx5e_tc_table *tc = &priv->fs.tc;
946 bool fdb_flow = false;
947 u32 flow_tag, action;
948 struct mlx5e_tc_flow *flow;
949 struct mlx5_flow_spec *spec;
950 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
952 if (esw && esw->mode == SRIOV_OFFLOADS)
956 flow = kzalloc(sizeof(*flow) +
957 sizeof(struct mlx5_esw_flow_attr),
960 flow = kzalloc(sizeof(*flow), GFP_KERNEL);
962 spec = mlx5_vzalloc(sizeof(*spec));
963 if (!spec || !flow) {
968 flow->cookie = f->cookie;
970 err = parse_cls_flower(priv, spec, f);
975 flow->attr = (struct mlx5_esw_flow_attr *)(flow + 1);
976 err = parse_tc_fdb_actions(priv, f->exts, flow);
979 flow->rule = mlx5e_tc_add_fdb_flow(priv, spec, flow->attr);
981 err = parse_tc_nic_actions(priv, f->exts, &action, &flow_tag);
984 flow->rule = mlx5e_tc_add_nic_flow(priv, spec, action, flow_tag);
987 if (IS_ERR(flow->rule)) {
988 err = PTR_ERR(flow->rule);
992 err = rhashtable_insert_fast(&tc->ht, &flow->node,
1000 mlx5_del_flow_rules(flow->rule);
1009 int mlx5e_delete_flower(struct mlx5e_priv *priv,
1010 struct tc_cls_flower_offload *f)
1012 struct mlx5e_tc_flow *flow;
1013 struct mlx5e_tc_table *tc = &priv->fs.tc;
1015 flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1020 rhashtable_remove_fast(&tc->ht, &flow->node, tc->ht_params);
1022 mlx5e_tc_del_flow(priv, flow);
1030 int mlx5e_stats_flower(struct mlx5e_priv *priv,
1031 struct tc_cls_flower_offload *f)
1033 struct mlx5e_tc_table *tc = &priv->fs.tc;
1034 struct mlx5e_tc_flow *flow;
1035 struct tc_action *a;
1036 struct mlx5_fc *counter;
1042 flow = rhashtable_lookup_fast(&tc->ht, &f->cookie,
1047 counter = mlx5_flow_rule_counter(flow->rule);
1051 mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse);
1053 tcf_exts_to_list(f->exts, &actions);
1054 list_for_each_entry(a, &actions, list)
1055 tcf_action_stats_update(a, bytes, packets, lastuse);
1060 static const struct rhashtable_params mlx5e_tc_flow_ht_params = {
1061 .head_offset = offsetof(struct mlx5e_tc_flow, node),
1062 .key_offset = offsetof(struct mlx5e_tc_flow, cookie),
1063 .key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
1064 .automatic_shrinking = true,
1067 int mlx5e_tc_init(struct mlx5e_priv *priv)
1069 struct mlx5e_tc_table *tc = &priv->fs.tc;
1071 tc->ht_params = mlx5e_tc_flow_ht_params;
1072 return rhashtable_init(&tc->ht, &tc->ht_params);
1075 static void _mlx5e_tc_del_flow(void *ptr, void *arg)
1077 struct mlx5e_tc_flow *flow = ptr;
1078 struct mlx5e_priv *priv = arg;
1080 mlx5e_tc_del_flow(priv, flow);
1084 void mlx5e_tc_cleanup(struct mlx5e_priv *priv)
1086 struct mlx5e_tc_table *tc = &priv->fs.tc;
1088 rhashtable_free_and_destroy(&tc->ht, _mlx5e_tc_del_flow, priv);
1090 if (!IS_ERR_OR_NULL(tc->t)) {
1091 mlx5_destroy_flow_table(tc->t);