drivers/net/ethernet/rocker/rocker.c

   1 /*
   2  * drivers/net/ethernet/rocker/rocker.c - Rocker switch device driver
   3  * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
   4  * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <linux/kernel.h>
  13 #include <linux/module.h>
  14 #include <linux/pci.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/sched.h>
  17 #include <linux/wait.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/hashtable.h>
  20 #include <linux/crc32.h>
  21 #include <linux/sort.h>
  22 #include <linux/random.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/inetdevice.h>
  25 #include <linux/skbuff.h>
  26 #include <linux/socket.h>
  27 #include <linux/etherdevice.h>
  28 #include <linux/ethtool.h>
  29 #include <linux/if_ether.h>
  30 #include <linux/if_vlan.h>
  31 #include <linux/if_bridge.h>
  32 #include <linux/bitops.h>
  33 #include <linux/ctype.h>
  34 #include <net/switchdev.h>
  35 #include <net/rtnetlink.h>
  36 #include <net/ip_fib.h>
  37 #include <net/netevent.h>
  38 #include <net/arp.h>
  39 #include <linux/io-64-nonatomic-lo-hi.h>
  40 #include <generated/utsrelease.h>
  41
  42 #include "rocker_hw.h"
  43
  44 static const char rocker_driver_name[] = "rocker";
  45
  46 static const struct pci_device_id rocker_pci_id_table[] = {
  47         {PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_ROCKER), 0},
  48         {0, }
  49 };
  50
  51 struct rocker_flow_tbl_key {
  52         u32 priority;
  53         enum rocker_of_dpa_table_id tbl_id;
  54         union {
  55                 struct {
  56                         u32 in_pport;
  57                         u32 in_pport_mask;
  58                         enum rocker_of_dpa_table_id goto_tbl;
  59                 } ig_port;
  60                 struct {
  61                         u32 in_pport;
  62                         __be16 vlan_id;
  63                         __be16 vlan_id_mask;
  64                         enum rocker_of_dpa_table_id goto_tbl;
  65                         bool untagged;
  66                         __be16 new_vlan_id;
  67                 } vlan;
  68                 struct {
  69                         u32 in_pport;
  70                         u32 in_pport_mask;
  71                         __be16 eth_type;
  72                         u8 eth_dst[ETH_ALEN];
  73                         u8 eth_dst_mask[ETH_ALEN];
  74                         __be16 vlan_id;
  75                         __be16 vlan_id_mask;
  76                         enum rocker_of_dpa_table_id goto_tbl;
  77                         bool copy_to_cpu;
  78                 } term_mac;
  79                 struct {
  80                         __be16 eth_type;
  81                         __be32 dst4;
  82                         __be32 dst4_mask;
  83                         enum rocker_of_dpa_table_id goto_tbl;
  84                         u32 group_id;
  85                 } ucast_routing;
  86                 struct {
  87                         u8 eth_dst[ETH_ALEN];
  88                         u8 eth_dst_mask[ETH_ALEN];
  89                         int has_eth_dst;
  90                         int has_eth_dst_mask;
  91                         __be16 vlan_id;
  92                         u32 tunnel_id;
  93                         enum rocker_of_dpa_table_id goto_tbl;
  94                         u32 group_id;
  95                         bool copy_to_cpu;
  96                 } bridge;
  97                 struct {
  98                         u32 in_pport;
  99                         u32 in_pport_mask;
 100                         u8 eth_src[ETH_ALEN];
 101                         u8 eth_src_mask[ETH_ALEN];
 102                         u8 eth_dst[ETH_ALEN];
 103                         u8 eth_dst_mask[ETH_ALEN];
 104                         __be16 eth_type;
 105                         __be16 vlan_id;
 106                         __be16 vlan_id_mask;
 107                         u8 ip_proto;
 108                         u8 ip_proto_mask;
 109                         u8 ip_tos;
 110                         u8 ip_tos_mask;
 111                         u32 group_id;
 112                 } acl;
 113         };
 114 };
 115
 116 struct rocker_flow_tbl_entry {
 117         struct hlist_node entry;
 118         u32 cmd;
 119         u64 cookie;
 120         struct rocker_flow_tbl_key key;
 121         size_t key_len;
 122         u32 key_crc32; /* key */
 123 };
 124
 125 struct rocker_group_tbl_entry {
 126         struct hlist_node entry;
 127         u32 cmd;
 128         u32 group_id; /* key */
 129         u16 group_count;
 130         u32 *group_ids;
 131         union {
 132                 struct {
 133                         u8 pop_vlan;
 134                 } l2_interface;
 135                 struct {
 136                         u8 eth_src[ETH_ALEN];
 137                         u8 eth_dst[ETH_ALEN];
 138                         __be16 vlan_id;
 139                         u32 group_id;
 140                 } l2_rewrite;
 141                 struct {
 142                         u8 eth_src[ETH_ALEN];
 143                         u8 eth_dst[ETH_ALEN];
 144                         __be16 vlan_id;
 145                         bool ttl_check;
 146                         u32 group_id;
 147                 } l3_unicast;
 148         };
 149 };
 150
 151 struct rocker_fdb_tbl_entry {
 152         struct hlist_node entry;
 153         u32 key_crc32; /* key */
 154         bool learned;
 155         unsigned long touched;
 156         struct rocker_fdb_tbl_key {
 157                 struct rocker_port *rocker_port;
 158                 u8 addr[ETH_ALEN];
 159                 __be16 vlan_id;
 160         } key;
 161 };
 162
 163 struct rocker_internal_vlan_tbl_entry {
 164         struct hlist_node entry;
 165         int ifindex; /* key */
 166         u32 ref_count;
 167         __be16 vlan_id;
 168 };
 169
 170 struct rocker_neigh_tbl_entry {
 171         struct hlist_node entry;
 172         __be32 ip_addr; /* key */
 173         struct net_device *dev;
 174         u32 ref_count;
 175         u32 index;
 176         u8 eth_dst[ETH_ALEN];
 177         bool ttl_check;
 178 };
 179
 180 struct rocker_desc_info {
 181         char *data; /* mapped */
 182         size_t data_size;
 183         size_t tlv_size;
 184         struct rocker_desc *desc;
 185         dma_addr_t mapaddr;
 186 };
 187
 188 struct rocker_dma_ring_info {
 189         size_t size;
 190         u32 head;
 191         u32 tail;
 192         struct rocker_desc *desc; /* mapped */
 193         dma_addr_t mapaddr;
 194         struct rocker_desc_info *desc_info;
 195         unsigned int type;
 196 };
 197
 198 struct rocker;
 199
 200 enum {
 201         ROCKER_CTRL_LINK_LOCAL_MCAST,
 202         ROCKER_CTRL_LOCAL_ARP,
 203         ROCKER_CTRL_IPV4_MCAST,
 204         ROCKER_CTRL_IPV6_MCAST,
 205         ROCKER_CTRL_DFLT_BRIDGING,
 206         ROCKER_CTRL_DFLT_OVS,
 207         ROCKER_CTRL_MAX,
 208 };
 209
 210 #define ROCKER_INTERNAL_VLAN_ID_BASE    0x0f00
 211 #define ROCKER_N_INTERNAL_VLANS         255
 212 #define ROCKER_VLAN_BITMAP_LEN          BITS_TO_LONGS(VLAN_N_VID)
 213 #define ROCKER_INTERNAL_VLAN_BITMAP_LEN BITS_TO_LONGS(ROCKER_N_INTERNAL_VLANS)
 214
 215 struct rocker_port {
 216         struct net_device *dev;
 217         struct net_device *bridge_dev;
 218         struct rocker *rocker;
 219         unsigned int port_number;
 220         u32 pport;
 221         __be16 internal_vlan_id;
 222         int stp_state;
 223         u32 brport_flags;
 224         unsigned long ageing_time;
 225         bool ctrls[ROCKER_CTRL_MAX];
 226         unsigned long vlan_bitmap[ROCKER_VLAN_BITMAP_LEN];
 227         struct napi_struct napi_tx;
 228         struct napi_struct napi_rx;
 229         struct rocker_dma_ring_info tx_ring;
 230         struct rocker_dma_ring_info rx_ring;
 231 };
 232
 233 struct rocker {
 234         struct pci_dev *pdev;
 235         u8 __iomem *hw_addr;
 236         struct msix_entry *msix_entries;
 237         unsigned int port_count;
 238         struct rocker_port **ports;
 239         struct {
 240                 u64 id;
 241         } hw;
 242         spinlock_t cmd_ring_lock;               /* for cmd ring accesses */
 243         struct rocker_dma_ring_info cmd_ring;
 244         struct rocker_dma_ring_info event_ring;
 245         DECLARE_HASHTABLE(flow_tbl, 16);
 246         spinlock_t flow_tbl_lock;               /* for flow tbl accesses */
 247         u64 flow_tbl_next_cookie;
 248         DECLARE_HASHTABLE(group_tbl, 16);
 249         spinlock_t group_tbl_lock;              /* for group tbl accesses */
 250         struct timer_list fdb_cleanup_timer;
 251         DECLARE_HASHTABLE(fdb_tbl, 16);
 252         spinlock_t fdb_tbl_lock;                /* for fdb tbl accesses */
 253         unsigned long internal_vlan_bitmap[ROCKER_INTERNAL_VLAN_BITMAP_LEN];
 254         DECLARE_HASHTABLE(internal_vlan_tbl, 8);
 255         spinlock_t internal_vlan_tbl_lock;      /* for vlan tbl accesses */
 256         DECLARE_HASHTABLE(neigh_tbl, 16);
 257         spinlock_t neigh_tbl_lock;              /* for neigh tbl accesses */
 258         u32 neigh_tbl_next_index;
 259 };
 260
 261 static const u8 zero_mac[ETH_ALEN]   = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
 262 static const u8 ff_mac[ETH_ALEN]     = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 263 static const u8 ll_mac[ETH_ALEN]     = { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x00 };
 264 static const u8 ll_mask[ETH_ALEN]    = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0 };
 265 static const u8 mcast_mac[ETH_ALEN]  = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 };
 266 static const u8 ipv4_mcast[ETH_ALEN] = { 0x01, 0x00, 0x5e, 0x00, 0x00, 0x00 };
 267 static const u8 ipv4_mask[ETH_ALEN]  = { 0xff, 0xff, 0xff, 0x80, 0x00, 0x00 };
 268 static const u8 ipv6_mcast[ETH_ALEN] = { 0x33, 0x33, 0x00, 0x00, 0x00, 0x00 };
 269 static const u8 ipv6_mask[ETH_ALEN]  = { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 };
 270
 271 /* Rocker priority levels for flow table entries.  Higher
 272  * priority match takes precedence over lower priority match.
 273  */
 274
 275 enum {
 276         ROCKER_PRIORITY_UNKNOWN = 0,
 277         ROCKER_PRIORITY_IG_PORT = 1,
 278         ROCKER_PRIORITY_VLAN = 1,
 279         ROCKER_PRIORITY_TERM_MAC_UCAST = 0,
 280         ROCKER_PRIORITY_TERM_MAC_MCAST = 1,
 281         ROCKER_PRIORITY_BRIDGING_VLAN_DFLT_EXACT = 1,
 282         ROCKER_PRIORITY_BRIDGING_VLAN_DFLT_WILD = 2,
 283         ROCKER_PRIORITY_BRIDGING_VLAN = 3,
 284         ROCKER_PRIORITY_BRIDGING_TENANT_DFLT_EXACT = 1,
 285         ROCKER_PRIORITY_BRIDGING_TENANT_DFLT_WILD = 2,
 286         ROCKER_PRIORITY_BRIDGING_TENANT = 3,
 287         ROCKER_PRIORITY_ACL_CTRL = 3,
 288         ROCKER_PRIORITY_ACL_NORMAL = 2,
 289         ROCKER_PRIORITY_ACL_DFLT = 1,
 290 };
 291
 292 static bool rocker_vlan_id_is_internal(__be16 vlan_id)
 293 {
 294         u16 start = ROCKER_INTERNAL_VLAN_ID_BASE;
 295         u16 end = 0xffe;
 296         u16 _vlan_id = ntohs(vlan_id);
 297
 298         return (_vlan_id >= start && _vlan_id <= end);
 299 }
 300
 301 static __be16 rocker_port_vid_to_vlan(const struct rocker_port *rocker_port,
 302                                       u16 vid, bool *pop_vlan)
 303 {
 304         __be16 vlan_id;
 305
 306         if (pop_vlan)
 307                 *pop_vlan = false;
 308         vlan_id = htons(vid);
 309         if (!vlan_id) {
 310                 vlan_id = rocker_port->internal_vlan_id;
 311                 if (pop_vlan)
 312                         *pop_vlan = true;
 313         }
 314
 315         return vlan_id;
 316 }
 317
 318 static u16 rocker_port_vlan_to_vid(const struct rocker_port *rocker_port,
 319                                    __be16 vlan_id)
 320 {
 321         if (rocker_vlan_id_is_internal(vlan_id))
 322                 return 0;
 323
 324         return ntohs(vlan_id);
 325 }
 326
 327 static bool rocker_port_is_bridged(const struct rocker_port *rocker_port)
 328 {
 329         return rocker_port->bridge_dev &&
 330                netif_is_bridge_master(rocker_port->bridge_dev);
 331 }
 332
 333 static bool rocker_port_is_ovsed(const struct rocker_port *rocker_port)
 334 {
 335         return rocker_port->bridge_dev &&
 336                netif_is_ovs_master(rocker_port->bridge_dev);
 337 }
 338
 339 #define ROCKER_OP_FLAG_REMOVE           BIT(0)
 340 #define ROCKER_OP_FLAG_NOWAIT           BIT(1)
 341 #define ROCKER_OP_FLAG_LEARNED          BIT(2)
 342 #define ROCKER_OP_FLAG_REFRESH          BIT(3)
 343
 344 static void *__rocker_mem_alloc(struct switchdev_trans *trans, int flags,
 345                                 size_t size)
 346 {
 347         struct switchdev_trans_item *elem = NULL;
 348         gfp_t gfp_flags = (flags & ROCKER_OP_FLAG_NOWAIT) ?
 349                           GFP_ATOMIC : GFP_KERNEL;
 350
 351         /* If in transaction prepare phase, allocate the memory
 352          * and enqueue it on a transaction.  If in transaction
 353          * commit phase, dequeue the memory from the transaction
 354          * rather than re-allocating the memory.  The idea is the
 355          * driver code paths for prepare and commit are identical
 356          * so the memory allocated in the prepare phase is the
 357          * memory used in the commit phase.
 358          */
 359
 360         if (!trans) {
 361                 elem = kzalloc(size + sizeof(*elem), gfp_flags);
 362         } else if (switchdev_trans_ph_prepare(trans)) {
 363                 elem = kzalloc(size + sizeof(*elem), gfp_flags);
 364                 if (!elem)
 365                         return NULL;
 366                 switchdev_trans_item_enqueue(trans, elem, kfree, elem);
 367         } else {
 368                 elem = switchdev_trans_item_dequeue(trans);
 369         }
 370
 371         return elem ? elem + 1 : NULL;
 372 }
 373
 374 static void *rocker_kzalloc(struct switchdev_trans *trans, int flags,
 375                             size_t size)
 376 {
 377         return __rocker_mem_alloc(trans, flags, size);
 378 }
 379
 380 static void *rocker_kcalloc(struct switchdev_trans *trans, int flags,
 381                             size_t n, size_t size)
 382 {
 383         return __rocker_mem_alloc(trans, flags, n * size);
 384 }
 385
 386 static void rocker_kfree(struct switchdev_trans *trans, const void *mem)
 387 {
 388         struct switchdev_trans_item *elem;
 389
 390         /* Frees are ignored if in transaction prepare phase.  The
 391          * memory remains on the per-port list until freed in the
 392          * commit phase.
 393          */
 394
 395         if (switchdev_trans_ph_prepare(trans))
 396                 return;
 397
 398         elem = (struct switchdev_trans_item *) mem - 1;
 399         kfree(elem);
 400 }
 401
 402 struct rocker_wait {
 403         wait_queue_head_t wait;
 404         bool done;
 405         bool nowait;
 406 };
 407
 408 static void rocker_wait_reset(struct rocker_wait *wait)
 409 {
 410         wait->done = false;
 411         wait->nowait = false;
 412 }
 413
 414 static void rocker_wait_init(struct rocker_wait *wait)
 415 {
 416         init_waitqueue_head(&wait->wait);
 417         rocker_wait_reset(wait);
 418 }
 419
 420 static struct rocker_wait *rocker_wait_create(struct rocker_port *rocker_port,
 421                                               struct switchdev_trans *trans,
 422                                               int flags)
 423 {
 424         struct rocker_wait *wait;
 425
 426         wait = rocker_kzalloc(trans, flags, sizeof(*wait));
 427         if (!wait)
 428                 return NULL;
 429         rocker_wait_init(wait);
 430         return wait;
 431 }
 432
 433 static void rocker_wait_destroy(struct switchdev_trans *trans,
 434                                 struct rocker_wait *wait)
 435 {
 436         rocker_kfree(trans, wait);
 437 }
 438
 439 static bool rocker_wait_event_timeout(struct rocker_wait *wait,
 440                                       unsigned long timeout)
 441 {
 442         wait_event_timeout(wait->wait, wait->done, HZ / 10);
 443         if (!wait->done)
 444                 return false;
 445         return true;
 446 }
 447
 448 static void rocker_wait_wake_up(struct rocker_wait *wait)
 449 {
 450         wait->done = true;
 451         wake_up(&wait->wait);
 452 }
 453
 454 static u32 rocker_msix_vector(const struct rocker *rocker, unsigned int vector)
 455 {
 456         return rocker->msix_entries[vector].vector;
 457 }
 458
 459 static u32 rocker_msix_tx_vector(const struct rocker_port *rocker_port)
 460 {
 461         return rocker_msix_vector(rocker_port->rocker,
 462                                   ROCKER_MSIX_VEC_TX(rocker_port->port_number));
 463 }
 464
 465 static u32 rocker_msix_rx_vector(const struct rocker_port *rocker_port)
 466 {
 467         return rocker_msix_vector(rocker_port->rocker,
 468                                   ROCKER_MSIX_VEC_RX(rocker_port->port_number));
 469 }
 470
 471 #define rocker_write32(rocker, reg, val)        \
 472         writel((val), (rocker)->hw_addr + (ROCKER_ ## reg))
 473 #define rocker_read32(rocker, reg)      \
 474         readl((rocker)->hw_addr + (ROCKER_ ## reg))
 475 #define rocker_write64(rocker, reg, val)        \
 476         writeq((val), (rocker)->hw_addr + (ROCKER_ ## reg))
 477 #define rocker_read64(rocker, reg)      \
 478         readq((rocker)->hw_addr + (ROCKER_ ## reg))
 479
 480 /*****************************
 481  * HW basic testing functions
 482  *****************************/
 483
 484 static int rocker_reg_test(const struct rocker *rocker)
 485 {
 486         const struct pci_dev *pdev = rocker->pdev;
 487         u64 test_reg;
 488         u64 rnd;
 489
 490         rnd = prandom_u32();
 491         rnd >>= 1;
 492         rocker_write32(rocker, TEST_REG, rnd);
 493         test_reg = rocker_read32(rocker, TEST_REG);
 494         if (test_reg != rnd * 2) {
 495                 dev_err(&pdev->dev, "unexpected 32bit register value %08llx, expected %08llx\n",
 496                         test_reg, rnd * 2);
 497                 return -EIO;
 498         }
 499
 500         rnd = prandom_u32();
 501         rnd <<= 31;
 502         rnd |= prandom_u32();
 503         rocker_write64(rocker, TEST_REG64, rnd);
 504         test_reg = rocker_read64(rocker, TEST_REG64);
 505         if (test_reg != rnd * 2) {
 506                 dev_err(&pdev->dev, "unexpected 64bit register value %16llx, expected %16llx\n",
 507                         test_reg, rnd * 2);
 508                 return -EIO;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int rocker_dma_test_one(const struct rocker *rocker,
 515                                struct rocker_wait *wait, u32 test_type,
 516                                dma_addr_t dma_handle, const unsigned char *buf,
 517                                const unsigned char *expect, size_t size)
 518 {
 519         const struct pci_dev *pdev = rocker->pdev;
 520         int i;
 521
 522         rocker_wait_reset(wait);
 523         rocker_write32(rocker, TEST_DMA_CTRL, test_type);
 524
 525         if (!rocker_wait_event_timeout(wait, HZ / 10)) {
 526                 dev_err(&pdev->dev, "no interrupt received within a timeout\n");
 527                 return -EIO;
 528         }
 529
 530         for (i = 0; i < size; i++) {
 531                 if (buf[i] != expect[i]) {
 532                         dev_err(&pdev->dev, "unexpected memory content %02x at byte %x\n, %02x expected",
 533                                 buf[i], i, expect[i]);
 534                         return -EIO;
 535                 }
 536         }
 537         return 0;
 538 }
 539
 540 #define ROCKER_TEST_DMA_BUF_SIZE (PAGE_SIZE * 4)
 541 #define ROCKER_TEST_DMA_FILL_PATTERN 0x96
 542
 543 static int rocker_dma_test_offset(const struct rocker *rocker,
 544                                   struct rocker_wait *wait, int offset)
 545 {
 546         struct pci_dev *pdev = rocker->pdev;
 547         unsigned char *alloc;
 548         unsigned char *buf;
 549         unsigned char *expect;
 550         dma_addr_t dma_handle;
 551         int i;
 552         int err;
 553
 554         alloc = kzalloc(ROCKER_TEST_DMA_BUF_SIZE * 2 + offset,
 555                         GFP_KERNEL | GFP_DMA);
 556         if (!alloc)
 557                 return -ENOMEM;
 558         buf = alloc + offset;
 559         expect = buf + ROCKER_TEST_DMA_BUF_SIZE;
 560
 561         dma_handle = pci_map_single(pdev, buf, ROCKER_TEST_DMA_BUF_SIZE,
 562                                     PCI_DMA_BIDIRECTIONAL);
 563         if (pci_dma_mapping_error(pdev, dma_handle)) {
 564                 err = -EIO;
 565                 goto free_alloc;
 566         }
 567
 568         rocker_write64(rocker, TEST_DMA_ADDR, dma_handle);
 569         rocker_write32(rocker, TEST_DMA_SIZE, ROCKER_TEST_DMA_BUF_SIZE);
 570
 571         memset(expect, ROCKER_TEST_DMA_FILL_PATTERN, ROCKER_TEST_DMA_BUF_SIZE);
 572         err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_FILL,
 573                                   dma_handle, buf, expect,
 574                                   ROCKER_TEST_DMA_BUF_SIZE);
 575         if (err)
 576                 goto unmap;
 577
 578         memset(expect, 0, ROCKER_TEST_DMA_BUF_SIZE);
 579         err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_CLEAR,
 580                                   dma_handle, buf, expect,
 581                                   ROCKER_TEST_DMA_BUF_SIZE);
 582         if (err)
 583                 goto unmap;
 584
 585         prandom_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE);
 586         for (i = 0; i < ROCKER_TEST_DMA_BUF_SIZE; i++)
 587                 expect[i] = ~buf[i];
 588         err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_INVERT,
 589                                   dma_handle, buf, expect,
 590                                   ROCKER_TEST_DMA_BUF_SIZE);
 591         if (err)
 592                 goto unmap;
 593
 594 unmap:
 595         pci_unmap_single(pdev, dma_handle, ROCKER_TEST_DMA_BUF_SIZE,
 596                          PCI_DMA_BIDIRECTIONAL);
 597 free_alloc:
 598         kfree(alloc);
 599
 600         return err;
 601 }
 602
 603 static int rocker_dma_test(const struct rocker *rocker,
 604                            struct rocker_wait *wait)
 605 {
 606         int i;
 607         int err;
 608
 609         for (i = 0; i < 8; i++) {
 610                 err = rocker_dma_test_offset(rocker, wait, i);
 611                 if (err)
 612                         return err;
 613         }
 614         return 0;
 615 }
 616
 617 static irqreturn_t rocker_test_irq_handler(int irq, void *dev_id)
 618 {
 619         struct rocker_wait *wait = dev_id;
 620
 621         rocker_wait_wake_up(wait);
 622
 623         return IRQ_HANDLED;
 624 }
 625
 626 static int rocker_basic_hw_test(const struct rocker *rocker)
 627 {
 628         const struct pci_dev *pdev = rocker->pdev;
 629         struct rocker_wait wait;
 630         int err;
 631
 632         err = rocker_reg_test(rocker);
 633         if (err) {
 634                 dev_err(&pdev->dev, "reg test failed\n");
 635                 return err;
 636         }
 637
 638         err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_TEST),
 639                           rocker_test_irq_handler, 0,
 640                           rocker_driver_name, &wait);
 641         if (err) {
 642                 dev_err(&pdev->dev, "cannot assign test irq\n");
 643                 return err;
 644         }
 645
 646         rocker_wait_init(&wait);
 647         rocker_write32(rocker, TEST_IRQ, ROCKER_MSIX_VEC_TEST);
 648
 649         if (!rocker_wait_event_timeout(&wait, HZ / 10)) {
 650                 dev_err(&pdev->dev, "no interrupt received within a timeout\n");
 651                 err = -EIO;
 652                 goto free_irq;
 653         }
 654
 655         err = rocker_dma_test(rocker, &wait);
 656         if (err)
 657                 dev_err(&pdev->dev, "dma test failed\n");
 658
 659 free_irq:
 660         free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_TEST), &wait);
 661         return err;
 662 }
 663
 664 /******
 665  * TLV
 666  ******/
 667
 668 #define ROCKER_TLV_ALIGNTO 8U
 669 #define ROCKER_TLV_ALIGN(len) \
 670         (((len) + ROCKER_TLV_ALIGNTO - 1) & ~(ROCKER_TLV_ALIGNTO - 1))
 671 #define ROCKER_TLV_HDRLEN ROCKER_TLV_ALIGN(sizeof(struct rocker_tlv))
 672
 673 /*  <------- ROCKER_TLV_HDRLEN -------> <--- ROCKER_TLV_ALIGN(payload) --->
 674  * +-----------------------------+- - -+- - - - - - - - - - - - - - -+- - -+
 675  * |             Header          | Pad |           Payload           | Pad |
 676  * |      (struct rocker_tlv)    | ing |                             | ing |
 677  * +-----------------------------+- - -+- - - - - - - - - - - - - - -+- - -+
 678  *  <--------------------------- tlv->len -------------------------->
 679  */
 680
 681 static struct rocker_tlv *rocker_tlv_next(const struct rocker_tlv *tlv,
 682                                           int *remaining)
 683 {
 684         int totlen = ROCKER_TLV_ALIGN(tlv->len);
 685
 686         *remaining -= totlen;
 687         return (struct rocker_tlv *) ((char *) tlv + totlen);
 688 }
 689
 690 static int rocker_tlv_ok(const struct rocker_tlv *tlv, int remaining)
 691 {
 692         return remaining >= (int) ROCKER_TLV_HDRLEN &&
 693                tlv->len >= ROCKER_TLV_HDRLEN &&
 694                tlv->len <= remaining;
 695 }
 696
 697 #define rocker_tlv_for_each(pos, head, len, rem)        \
 698         for (pos = head, rem = len;                     \
 699              rocker_tlv_ok(pos, rem);                   \
 700              pos = rocker_tlv_next(pos, &(rem)))
 701
 702 #define rocker_tlv_for_each_nested(pos, tlv, rem)       \
 703         rocker_tlv_for_each(pos, rocker_tlv_data(tlv),  \
 704                             rocker_tlv_len(tlv), rem)
 705
 706 static int rocker_tlv_attr_size(int payload)
 707 {
 708         return ROCKER_TLV_HDRLEN + payload;
 709 }
 710
 711 static int rocker_tlv_total_size(int payload)
 712 {
 713         return ROCKER_TLV_ALIGN(rocker_tlv_attr_size(payload));
 714 }
 715
 716 static int rocker_tlv_padlen(int payload)
 717 {
 718         return rocker_tlv_total_size(payload) - rocker_tlv_attr_size(payload);
 719 }
 720
 721 static int rocker_tlv_type(const struct rocker_tlv *tlv)
 722 {
 723         return tlv->type;
 724 }
 725
 726 static void *rocker_tlv_data(const struct rocker_tlv *tlv)
 727 {
 728         return (char *) tlv + ROCKER_TLV_HDRLEN;
 729 }
 730
 731 static int rocker_tlv_len(const struct rocker_tlv *tlv)
 732 {
 733         return tlv->len - ROCKER_TLV_HDRLEN;
 734 }
 735
 736 static u8 rocker_tlv_get_u8(const struct rocker_tlv *tlv)
 737 {
 738         return *(u8 *) rocker_tlv_data(tlv);
 739 }
 740
 741 static u16 rocker_tlv_get_u16(const struct rocker_tlv *tlv)
 742 {
 743         return *(u16 *) rocker_tlv_data(tlv);
 744 }
 745
 746 static __be16 rocker_tlv_get_be16(const struct rocker_tlv *tlv)
 747 {
 748         return *(__be16 *) rocker_tlv_data(tlv);
 749 }
 750
 751 static u32 rocker_tlv_get_u32(const struct rocker_tlv *tlv)
 752 {
 753         return *(u32 *) rocker_tlv_data(tlv);
 754 }
 755
 756 static u64 rocker_tlv_get_u64(const struct rocker_tlv *tlv)
 757 {
 758         return *(u64 *) rocker_tlv_data(tlv);
 759 }
 760
 761 static void rocker_tlv_parse(const struct rocker_tlv **tb, int maxtype,
 762                              const char *buf, int buf_len)
 763 {
 764         const struct rocker_tlv *tlv;
 765         const struct rocker_tlv *head = (const struct rocker_tlv *) buf;
 766         int rem;
 767
 768         memset(tb, 0, sizeof(struct rocker_tlv *) * (maxtype + 1));
 769
 770         rocker_tlv_for_each(tlv, head, buf_len, rem) {
 771                 u32 type = rocker_tlv_type(tlv);
 772
 773                 if (type > 0 && type <= maxtype)
 774                         tb[type] = tlv;
 775         }
 776 }
 777
 778 static void rocker_tlv_parse_nested(const struct rocker_tlv **tb, int maxtype,
 779                                     const struct rocker_tlv *tlv)
 780 {
 781         rocker_tlv_parse(tb, maxtype, rocker_tlv_data(tlv),
 782                          rocker_tlv_len(tlv));
 783 }
 784
 785 static void rocker_tlv_parse_desc(const struct rocker_tlv **tb, int maxtype,
 786                                   const struct rocker_desc_info *desc_info)
 787 {
 788         rocker_tlv_parse(tb, maxtype, desc_info->data,
 789                          desc_info->desc->tlv_size);
 790 }
 791
 792 static struct rocker_tlv *rocker_tlv_start(struct rocker_desc_info *desc_info)
 793 {
 794         return (struct rocker_tlv *) ((char *) desc_info->data +
 795                                                desc_info->tlv_size);
 796 }
 797
 798 static int rocker_tlv_put(struct rocker_desc_info *desc_info,
 799                           int attrtype, int attrlen, const void *data)
 800 {
 801         int tail_room = desc_info->data_size - desc_info->tlv_size;
 802         int total_size = rocker_tlv_total_size(attrlen);
 803         struct rocker_tlv *tlv;
 804
 805         if (unlikely(tail_room < total_size))
 806                 return -EMSGSIZE;
 807
 808         tlv = rocker_tlv_start(desc_info);
 809         desc_info->tlv_size += total_size;
 810         tlv->type = attrtype;
 811         tlv->len = rocker_tlv_attr_size(attrlen);
 812         memcpy(rocker_tlv_data(tlv), data, attrlen);
 813         memset((char *) tlv + tlv->len, 0, rocker_tlv_padlen(attrlen));
 814         return 0;
 815 }
 816
 817 static int rocker_tlv_put_u8(struct rocker_desc_info *desc_info,
 818                              int attrtype, u8 value)
 819 {
 820         return rocker_tlv_put(desc_info, attrtype, sizeof(u8), &value);
 821 }
 822
 823 static int rocker_tlv_put_u16(struct rocker_desc_info *desc_info,
 824                               int attrtype, u16 value)
 825 {
 826         return rocker_tlv_put(desc_info, attrtype, sizeof(u16), &value);
 827 }
 828
 829 static int rocker_tlv_put_be16(struct rocker_desc_info *desc_info,
 830                                int attrtype, __be16 value)
 831 {
 832         return rocker_tlv_put(desc_info, attrtype, sizeof(__be16), &value);
 833 }
 834
 835 static int rocker_tlv_put_u32(struct rocker_desc_info *desc_info,
 836                               int attrtype, u32 value)
 837 {
 838         return rocker_tlv_put(desc_info, attrtype, sizeof(u32), &value);
 839 }
 840
 841 static int rocker_tlv_put_be32(struct rocker_desc_info *desc_info,
 842                                int attrtype, __be32 value)
 843 {
 844         return rocker_tlv_put(desc_info, attrtype, sizeof(__be32), &value);
 845 }
 846
 847 static int rocker_tlv_put_u64(struct rocker_desc_info *desc_info,
 848                               int attrtype, u64 value)
 849 {
 850         return rocker_tlv_put(desc_info, attrtype, sizeof(u64), &value);
 851 }
 852
 853 static struct rocker_tlv *
 854 rocker_tlv_nest_start(struct rocker_desc_info *desc_info, int attrtype)
 855 {
 856         struct rocker_tlv *start = rocker_tlv_start(desc_info);
 857
 858         if (rocker_tlv_put(desc_info, attrtype, 0, NULL) < 0)
 859                 return NULL;
 860
 861         return start;
 862 }
 863
 864 static void rocker_tlv_nest_end(struct rocker_desc_info *desc_info,
 865                                 struct rocker_tlv *start)
 866 {
 867         start->len = (char *) rocker_tlv_start(desc_info) - (char *) start;
 868 }
 869
 870 static void rocker_tlv_nest_cancel(struct rocker_desc_info *desc_info,
 871                                    const struct rocker_tlv *start)
 872 {
 873         desc_info->tlv_size = (const char *) start - desc_info->data;
 874 }
 875
 876 /******************************************
 877  * DMA rings and descriptors manipulations
 878  ******************************************/
 879
 880 static u32 __pos_inc(u32 pos, size_t limit)
 881 {
 882         return ++pos == limit ? 0 : pos;
 883 }
 884
 885 static int rocker_desc_err(const struct rocker_desc_info *desc_info)
 886 {
 887         int err = desc_info->desc->comp_err & ~ROCKER_DMA_DESC_COMP_ERR_GEN;
 888
 889         switch (err) {
 890         case ROCKER_OK:
 891                 return 0;
 892         case -ROCKER_ENOENT:
 893                 return -ENOENT;
 894         case -ROCKER_ENXIO:
 895                 return -ENXIO;
 896         case -ROCKER_ENOMEM:
 897                 return -ENOMEM;
 898         case -ROCKER_EEXIST:
 899                 return -EEXIST;
 900         case -ROCKER_EINVAL:
 901                 return -EINVAL;
 902         case -ROCKER_EMSGSIZE:
 903                 return -EMSGSIZE;
 904         case -ROCKER_ENOTSUP:
 905                 return -EOPNOTSUPP;
 906         case -ROCKER_ENOBUFS:
 907                 return -ENOBUFS;
 908         }
 909
 910         return -EINVAL;
 911 }
 912
 913 static void rocker_desc_gen_clear(const struct rocker_desc_info *desc_info)
 914 {
 915         desc_info->desc->comp_err &= ~ROCKER_DMA_DESC_COMP_ERR_GEN;
 916 }
 917
 918 static bool rocker_desc_gen(const struct rocker_desc_info *desc_info)
 919 {
 920         u32 comp_err = desc_info->desc->comp_err;
 921
 922         return comp_err & ROCKER_DMA_DESC_COMP_ERR_GEN ? true : false;
 923 }
 924
 925 static void *rocker_desc_cookie_ptr_get(const struct rocker_desc_info *desc_info)
 926 {
 927         return (void *)(uintptr_t)desc_info->desc->cookie;
 928 }
 929
 930 static void rocker_desc_cookie_ptr_set(const struct rocker_desc_info *desc_info,
 931                                        void *ptr)
 932 {
 933         desc_info->desc->cookie = (uintptr_t) ptr;
 934 }
 935
 936 static struct rocker_desc_info *
 937 rocker_desc_head_get(const struct rocker_dma_ring_info *info)
 938 {
 939         static struct rocker_desc_info *desc_info;
 940         u32 head = __pos_inc(info->head, info->size);
 941
 942         desc_info = &info->desc_info[info->head];
 943         if (head == info->tail)
 944                 return NULL; /* ring full */
 945         desc_info->tlv_size = 0;
 946         return desc_info;
 947 }
 948
 949 static void rocker_desc_commit(const struct rocker_desc_info *desc_info)
 950 {
 951         desc_info->desc->buf_size = desc_info->data_size;
 952         desc_info->desc->tlv_size = desc_info->tlv_size;
 953 }
 954
 955 static void rocker_desc_head_set(const struct rocker *rocker,
 956                                  struct rocker_dma_ring_info *info,
 957                                  const struct rocker_desc_info *desc_info)
 958 {
 959         u32 head = __pos_inc(info->head, info->size);
 960
 961         BUG_ON(head == info->tail);
 962         rocker_desc_commit(desc_info);
 963         info->head = head;
 964         rocker_write32(rocker, DMA_DESC_HEAD(info->type), head);
 965 }
 966
 967 static struct rocker_desc_info *
 968 rocker_desc_tail_get(struct rocker_dma_ring_info *info)
 969 {
 970         static struct rocker_desc_info *desc_info;
 971
 972         if (info->tail == info->head)
 973                 return NULL; /* nothing to be done between head and tail */
 974         desc_info = &info->desc_info[info->tail];
 975         if (!rocker_desc_gen(desc_info))
 976                 return NULL; /* gen bit not set, desc is not ready yet */
 977         info->tail = __pos_inc(info->tail, info->size);
 978         desc_info->tlv_size = desc_info->desc->tlv_size;
 979         return desc_info;
 980 }
 981
 982 static void rocker_dma_ring_credits_set(const struct rocker *rocker,
 983                                         const struct rocker_dma_ring_info *info,
 984                                         u32 credits)
 985 {
 986         if (credits)
 987                 rocker_write32(rocker, DMA_DESC_CREDITS(info->type), credits);
 988 }
 989
 990 static unsigned long rocker_dma_ring_size_fix(size_t size)
 991 {
 992         return max(ROCKER_DMA_SIZE_MIN,
 993                    min(roundup_pow_of_two(size), ROCKER_DMA_SIZE_MAX));
 994 }
 995
 996 static int rocker_dma_ring_create(const struct rocker *rocker,
 997                                   unsigned int type,
 998                                   size_t size,
 999                                   struct rocker_dma_ring_info *info)
1000 {
1001         int i;
1002
1003         BUG_ON(size != rocker_dma_ring_size_fix(size));
1004         info->size = size;
1005         info->type = type;
1006         info->head = 0;
1007         info->tail = 0;
1008         info->desc_info = kcalloc(info->size, sizeof(*info->desc_info),
1009                                   GFP_KERNEL);
1010         if (!info->desc_info)
1011                 return -ENOMEM;
1012
1013         info->desc = pci_alloc_consistent(rocker->pdev,
1014                                           info->size * sizeof(*info->desc),
1015                                           &info->mapaddr);
1016         if (!info->desc) {
1017                 kfree(info->desc_info);
1018                 return -ENOMEM;
1019         }
1020
1021         for (i = 0; i < info->size; i++)
1022                 info->desc_info[i].desc = &info->desc[i];
1023
1024         rocker_write32(rocker, DMA_DESC_CTRL(info->type),
1025                        ROCKER_DMA_DESC_CTRL_RESET);
1026         rocker_write64(rocker, DMA_DESC_ADDR(info->type), info->mapaddr);
1027         rocker_write32(rocker, DMA_DESC_SIZE(info->type), info->size);
1028
1029         return 0;
1030 }
1031
1032 static void rocker_dma_ring_destroy(const struct rocker *rocker,
1033                                     const struct rocker_dma_ring_info *info)
1034 {
1035         rocker_write64(rocker, DMA_DESC_ADDR(info->type), 0);
1036
1037         pci_free_consistent(rocker->pdev,
1038                             info->size * sizeof(struct rocker_desc),
1039                             info->desc, info->mapaddr);
1040         kfree(info->desc_info);
1041 }
1042
1043 static void rocker_dma_ring_pass_to_producer(const struct rocker *rocker,
1044                                              struct rocker_dma_ring_info *info)
1045 {
1046         int i;
1047
1048         BUG_ON(info->head || info->tail);
1049
1050         /* When ring is consumer, we need to advance head for each desc.
1051          * That tells hw that the desc is ready to be used by it.
1052          */
1053         for (i = 0; i < info->size - 1; i++)
1054                 rocker_desc_head_set(rocker, info, &info->desc_info[i]);
1055         rocker_desc_commit(&info->desc_info[i]);
1056 }
1057
1058 static int rocker_dma_ring_bufs_alloc(const struct rocker *rocker,
1059                                       const struct rocker_dma_ring_info *info,
1060                                       int direction, size_t buf_size)
1061 {
1062         struct pci_dev *pdev = rocker->pdev;
1063         int i;
1064         int err;
1065
1066         for (i = 0; i < info->size; i++) {
1067                 struct rocker_desc_info *desc_info = &info->desc_info[i];
1068                 struct rocker_desc *desc = &info->desc[i];
1069                 dma_addr_t dma_handle;
1070                 char *buf;
1071
1072                 buf = kzalloc(buf_size, GFP_KERNEL | GFP_DMA);
1073                 if (!buf) {
1074                         err = -ENOMEM;
1075                         goto rollback;
1076                 }
1077
1078                 dma_handle = pci_map_single(pdev, buf, buf_size, direction);
1079                 if (pci_dma_mapping_error(pdev, dma_handle)) {
1080                         kfree(buf);
1081                         err = -EIO;
1082                         goto rollback;
1083                 }
1084
1085                 desc_info->data = buf;
1086                 desc_info->data_size = buf_size;
1087                 dma_unmap_addr_set(desc_info, mapaddr, dma_handle);
1088
1089                 desc->buf_addr = dma_handle;
1090                 desc->buf_size = buf_size;
1091         }
1092         return 0;
1093
1094 rollback:
1095         for (i--; i >= 0; i--) {
1096                 const struct rocker_desc_info *desc_info = &info->desc_info[i];
1097
1098                 pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),
1099                                  desc_info->data_size, direction);
1100                 kfree(desc_info->data);
1101         }
1102         return err;
1103 }
1104
1105 static void rocker_dma_ring_bufs_free(const struct rocker *rocker,
1106                                       const struct rocker_dma_ring_info *info,
1107                                       int direction)
1108 {
1109         struct pci_dev *pdev = rocker->pdev;
1110         int i;
1111
1112         for (i = 0; i < info->size; i++) {
1113                 const struct rocker_desc_info *desc_info = &info->desc_info[i];
1114                 struct rocker_desc *desc = &info->desc[i];
1115
1116                 desc->buf_addr = 0;
1117                 desc->buf_size = 0;
1118                 pci_unmap_single(pdev, dma_unmap_addr(desc_info, mapaddr),
1119                                  desc_info->data_size, direction);
1120                 kfree(desc_info->data);
1121         }
1122 }
1123
1124 static int rocker_dma_rings_init(struct rocker *rocker)
1125 {
1126         const struct pci_dev *pdev = rocker->pdev;
1127         int err;
1128
1129         err = rocker_dma_ring_create(rocker, ROCKER_DMA_CMD,
1130                                      ROCKER_DMA_CMD_DEFAULT_SIZE,
1131                                      &rocker->cmd_ring);
1132         if (err) {
1133                 dev_err(&pdev->dev, "failed to create command dma ring\n");
1134                 return err;
1135         }
1136
1137         spin_lock_init(&rocker->cmd_ring_lock);
1138
1139         err = rocker_dma_ring_bufs_alloc(rocker, &rocker->cmd_ring,
1140                                          PCI_DMA_BIDIRECTIONAL, PAGE_SIZE);
1141         if (err) {
1142                 dev_err(&pdev->dev, "failed to alloc command dma ring buffers\n");
1143                 goto err_dma_cmd_ring_bufs_alloc;
1144         }
1145
1146         err = rocker_dma_ring_create(rocker, ROCKER_DMA_EVENT,
1147                                      ROCKER_DMA_EVENT_DEFAULT_SIZE,
1148                                      &rocker->event_ring);
1149         if (err) {
1150                 dev_err(&pdev->dev, "failed to create event dma ring\n");
1151                 goto err_dma_event_ring_create;
1152         }
1153
1154         err = rocker_dma_ring_bufs_alloc(rocker, &rocker->event_ring,
1155                                          PCI_DMA_FROMDEVICE, PAGE_SIZE);
1156         if (err) {
1157                 dev_err(&pdev->dev, "failed to alloc event dma ring buffers\n");
1158                 goto err_dma_event_ring_bufs_alloc;
1159         }
1160         rocker_dma_ring_pass_to_producer(rocker, &rocker->event_ring);
1161         return 0;
1162
1163 err_dma_event_ring_bufs_alloc:
1164         rocker_dma_ring_destroy(rocker, &rocker->event_ring);
1165 err_dma_event_ring_create:
1166         rocker_dma_ring_bufs_free(rocker, &rocker->cmd_ring,
1167                                   PCI_DMA_BIDIRECTIONAL);
1168 err_dma_cmd_ring_bufs_alloc:
1169         rocker_dma_ring_destroy(rocker, &rocker->cmd_ring);
1170         return err;
1171 }
1172
1173 static void rocker_dma_rings_fini(struct rocker *rocker)
1174 {
1175         rocker_dma_ring_bufs_free(rocker, &rocker->event_ring,
1176                                   PCI_DMA_BIDIRECTIONAL);
1177         rocker_dma_ring_destroy(rocker, &rocker->event_ring);
1178         rocker_dma_ring_bufs_free(rocker, &rocker->cmd_ring,
1179                                   PCI_DMA_BIDIRECTIONAL);
1180         rocker_dma_ring_destroy(rocker, &rocker->cmd_ring);
1181 }
1182
1183 static int rocker_dma_rx_ring_skb_map(const struct rocker_port *rocker_port,
1184                                       struct rocker_desc_info *desc_info,
1185                                       struct sk_buff *skb, size_t buf_len)
1186 {
1187         const struct rocker *rocker = rocker_port->rocker;
1188         struct pci_dev *pdev = rocker->pdev;
1189         dma_addr_t dma_handle;
1190
1191         dma_handle = pci_map_single(pdev, skb->data, buf_len,
1192                                     PCI_DMA_FROMDEVICE);
1193         if (pci_dma_mapping_error(pdev, dma_handle))
1194                 return -EIO;
1195         if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_RX_FRAG_ADDR, dma_handle))
1196                 goto tlv_put_failure;
1197         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_RX_FRAG_MAX_LEN, buf_len))
1198                 goto tlv_put_failure;
1199         return 0;
1200
1201 tlv_put_failure:
1202         pci_unmap_single(pdev, dma_handle, buf_len, PCI_DMA_FROMDEVICE);
1203         desc_info->tlv_size = 0;
1204         return -EMSGSIZE;
1205 }
1206
1207 static size_t rocker_port_rx_buf_len(const struct rocker_port *rocker_port)
1208 {
1209         return rocker_port->dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
1210 }
1211
1212 static int rocker_dma_rx_ring_skb_alloc(const struct rocker_port *rocker_port,
1213                                         struct rocker_desc_info *desc_info)
1214 {
1215         struct net_device *dev = rocker_port->dev;
1216         struct sk_buff *skb;
1217         size_t buf_len = rocker_port_rx_buf_len(rocker_port);
1218         int err;
1219
1220         /* Ensure that hw will see tlv_size zero in case of an error.
1221          * That tells hw to use another descriptor.
1222          */
1223         rocker_desc_cookie_ptr_set(desc_info, NULL);
1224         desc_info->tlv_size = 0;
1225
1226         skb = netdev_alloc_skb_ip_align(dev, buf_len);
1227         if (!skb)
1228                 return -ENOMEM;
1229         err = rocker_dma_rx_ring_skb_map(rocker_port, desc_info, skb, buf_len);
1230         if (err) {
1231                 dev_kfree_skb_any(skb);
1232                 return err;
1233         }
1234         rocker_desc_cookie_ptr_set(desc_info, skb);
1235         return 0;
1236 }
1237
1238 static void rocker_dma_rx_ring_skb_unmap(const struct rocker *rocker,
1239                                          const struct rocker_tlv **attrs)
1240 {
1241         struct pci_dev *pdev = rocker->pdev;
1242         dma_addr_t dma_handle;
1243         size_t len;
1244
1245         if (!attrs[ROCKER_TLV_RX_FRAG_ADDR] ||
1246             !attrs[ROCKER_TLV_RX_FRAG_MAX_LEN])
1247                 return;
1248         dma_handle = rocker_tlv_get_u64(attrs[ROCKER_TLV_RX_FRAG_ADDR]);
1249         len = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FRAG_MAX_LEN]);
1250         pci_unmap_single(pdev, dma_handle, len, PCI_DMA_FROMDEVICE);
1251 }
1252
1253 static void rocker_dma_rx_ring_skb_free(const struct rocker *rocker,
1254                                         const struct rocker_desc_info *desc_info)
1255 {
1256         const struct rocker_tlv *attrs[ROCKER_TLV_RX_MAX + 1];
1257         struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);
1258
1259         if (!skb)
1260                 return;
1261         rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);
1262         rocker_dma_rx_ring_skb_unmap(rocker, attrs);
1263         dev_kfree_skb_any(skb);
1264 }
1265
1266 static int rocker_dma_rx_ring_skbs_alloc(const struct rocker_port *rocker_port)
1267 {
1268         const struct rocker_dma_ring_info *rx_ring = &rocker_port->rx_ring;
1269         const struct rocker *rocker = rocker_port->rocker;
1270         int i;
1271         int err;
1272
1273         for (i = 0; i < rx_ring->size; i++) {
1274                 err = rocker_dma_rx_ring_skb_alloc(rocker_port,
1275                                                    &rx_ring->desc_info[i]);
1276                 if (err)
1277                         goto rollback;
1278         }
1279         return 0;
1280
1281 rollback:
1282         for (i--; i >= 0; i--)
1283                 rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);
1284         return err;
1285 }
1286
1287 static void rocker_dma_rx_ring_skbs_free(const struct rocker_port *rocker_port)
1288 {
1289         const struct rocker_dma_ring_info *rx_ring = &rocker_port->rx_ring;
1290         const struct rocker *rocker = rocker_port->rocker;
1291         int i;
1292
1293         for (i = 0; i < rx_ring->size; i++)
1294                 rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);
1295 }
1296
1297 static int rocker_port_dma_rings_init(struct rocker_port *rocker_port)
1298 {
1299         struct rocker *rocker = rocker_port->rocker;
1300         int err;
1301
1302         err = rocker_dma_ring_create(rocker,
1303                                      ROCKER_DMA_TX(rocker_port->port_number),
1304                                      ROCKER_DMA_TX_DEFAULT_SIZE,
1305                                      &rocker_port->tx_ring);
1306         if (err) {
1307                 netdev_err(rocker_port->dev, "failed to create tx dma ring\n");
1308                 return err;
1309         }
1310
1311         err = rocker_dma_ring_bufs_alloc(rocker, &rocker_port->tx_ring,
1312                                          PCI_DMA_TODEVICE,
1313                                          ROCKER_DMA_TX_DESC_SIZE);
1314         if (err) {
1315                 netdev_err(rocker_port->dev, "failed to alloc tx dma ring buffers\n");
1316                 goto err_dma_tx_ring_bufs_alloc;
1317         }
1318
1319         err = rocker_dma_ring_create(rocker,
1320                                      ROCKER_DMA_RX(rocker_port->port_number),
1321                                      ROCKER_DMA_RX_DEFAULT_SIZE,
1322                                      &rocker_port->rx_ring);
1323         if (err) {
1324                 netdev_err(rocker_port->dev, "failed to create rx dma ring\n");
1325                 goto err_dma_rx_ring_create;
1326         }
1327
1328         err = rocker_dma_ring_bufs_alloc(rocker, &rocker_port->rx_ring,
1329                                          PCI_DMA_BIDIRECTIONAL,
1330                                          ROCKER_DMA_RX_DESC_SIZE);
1331         if (err) {
1332                 netdev_err(rocker_port->dev, "failed to alloc rx dma ring buffers\n");
1333                 goto err_dma_rx_ring_bufs_alloc;
1334         }
1335
1336         err = rocker_dma_rx_ring_skbs_alloc(rocker_port);
1337         if (err) {
1338                 netdev_err(rocker_port->dev, "failed to alloc rx dma ring skbs\n");
1339                 goto err_dma_rx_ring_skbs_alloc;
1340         }
1341         rocker_dma_ring_pass_to_producer(rocker, &rocker_port->rx_ring);
1342
1343         return 0;
1344
1345 err_dma_rx_ring_skbs_alloc:
1346         rocker_dma_ring_bufs_free(rocker, &rocker_port->rx_ring,
1347                                   PCI_DMA_BIDIRECTIONAL);
1348 err_dma_rx_ring_bufs_alloc:
1349         rocker_dma_ring_destroy(rocker, &rocker_port->rx_ring);
1350 err_dma_rx_ring_create:
1351         rocker_dma_ring_bufs_free(rocker, &rocker_port->tx_ring,
1352                                   PCI_DMA_TODEVICE);
1353 err_dma_tx_ring_bufs_alloc:
1354         rocker_dma_ring_destroy(rocker, &rocker_port->tx_ring);
1355         return err;
1356 }
1357
1358 static void rocker_port_dma_rings_fini(struct rocker_port *rocker_port)
1359 {
1360         struct rocker *rocker = rocker_port->rocker;
1361
1362         rocker_dma_rx_ring_skbs_free(rocker_port);
1363         rocker_dma_ring_bufs_free(rocker, &rocker_port->rx_ring,
1364                                   PCI_DMA_BIDIRECTIONAL);
1365         rocker_dma_ring_destroy(rocker, &rocker_port->rx_ring);
1366         rocker_dma_ring_bufs_free(rocker, &rocker_port->tx_ring,
1367                                   PCI_DMA_TODEVICE);
1368         rocker_dma_ring_destroy(rocker, &rocker_port->tx_ring);
1369 }
1370
1371 static void rocker_port_set_enable(const struct rocker_port *rocker_port,
1372                                    bool enable)
1373 {
1374         u64 val = rocker_read64(rocker_port->rocker, PORT_PHYS_ENABLE);
1375
1376         if (enable)
1377                 val |= 1ULL << rocker_port->pport;
1378         else
1379                 val &= ~(1ULL << rocker_port->pport);
1380         rocker_write64(rocker_port->rocker, PORT_PHYS_ENABLE, val);
1381 }
1382
1383 /********************************
1384  * Interrupt handler and helpers
1385  ********************************/
1386
1387 static irqreturn_t rocker_cmd_irq_handler(int irq, void *dev_id)
1388 {
1389         struct rocker *rocker = dev_id;
1390         const struct rocker_desc_info *desc_info;
1391         struct rocker_wait *wait;
1392         u32 credits = 0;
1393
1394         spin_lock(&rocker->cmd_ring_lock);
1395         while ((desc_info = rocker_desc_tail_get(&rocker->cmd_ring))) {
1396                 wait = rocker_desc_cookie_ptr_get(desc_info);
1397                 if (wait->nowait) {
1398                         rocker_desc_gen_clear(desc_info);
1399                         rocker_wait_destroy(NULL, wait);
1400                 } else {
1401                         rocker_wait_wake_up(wait);
1402                 }
1403                 credits++;
1404         }
1405         spin_unlock(&rocker->cmd_ring_lock);
1406         rocker_dma_ring_credits_set(rocker, &rocker->cmd_ring, credits);
1407
1408         return IRQ_HANDLED;
1409 }
1410
1411 static void rocker_port_link_up(const struct rocker_port *rocker_port)
1412 {
1413         netif_carrier_on(rocker_port->dev);
1414         netdev_info(rocker_port->dev, "Link is up\n");
1415 }
1416
1417 static void rocker_port_link_down(const struct rocker_port *rocker_port)
1418 {
1419         netif_carrier_off(rocker_port->dev);
1420         netdev_info(rocker_port->dev, "Link is down\n");
1421 }
1422
1423 static int rocker_event_link_change(const struct rocker *rocker,
1424                                     const struct rocker_tlv *info)
1425 {
1426         const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_LINK_CHANGED_MAX + 1];
1427         unsigned int port_number;
1428         bool link_up;
1429         struct rocker_port *rocker_port;
1430
1431         rocker_tlv_parse_nested(attrs, ROCKER_TLV_EVENT_LINK_CHANGED_MAX, info);
1432         if (!attrs[ROCKER_TLV_EVENT_LINK_CHANGED_PPORT] ||
1433             !attrs[ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP])
1434                 return -EIO;
1435         port_number =
1436                 rocker_tlv_get_u32(attrs[ROCKER_TLV_EVENT_LINK_CHANGED_PPORT]) - 1;
1437         link_up = rocker_tlv_get_u8(attrs[ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP]);
1438
1439         if (port_number >= rocker->port_count)
1440                 return -EINVAL;
1441
1442         rocker_port = rocker->ports[port_number];
1443         if (netif_carrier_ok(rocker_port->dev) != link_up) {
1444                 if (link_up)
1445                         rocker_port_link_up(rocker_port);
1446                 else
1447                         rocker_port_link_down(rocker_port);
1448         }
1449
1450         return 0;
1451 }
1452
1453 static int rocker_port_fdb(struct rocker_port *rocker_port,
1454                            struct switchdev_trans *trans,
1455                            const unsigned char *addr,
1456                            __be16 vlan_id, int flags);
1457
1458 static int rocker_event_mac_vlan_seen(const struct rocker *rocker,
1459                                       const struct rocker_tlv *info)
1460 {
1461         const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAX + 1];
1462         unsigned int port_number;
1463         struct rocker_port *rocker_port;
1464         const unsigned char *addr;
1465         int flags = ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_LEARNED;
1466         __be16 vlan_id;
1467
1468         rocker_tlv_parse_nested(attrs, ROCKER_TLV_EVENT_MAC_VLAN_MAX, info);
1469         if (!attrs[ROCKER_TLV_EVENT_MAC_VLAN_PPORT] ||
1470             !attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAC] ||
1471             !attrs[ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID])
1472                 return -EIO;
1473         port_number =
1474                 rocker_tlv_get_u32(attrs[ROCKER_TLV_EVENT_MAC_VLAN_PPORT]) - 1;
1475         addr = rocker_tlv_data(attrs[ROCKER_TLV_EVENT_MAC_VLAN_MAC]);
1476         vlan_id = rocker_tlv_get_be16(attrs[ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID]);
1477
1478         if (port_number >= rocker->port_count)
1479                 return -EINVAL;
1480
1481         rocker_port = rocker->ports[port_number];
1482
1483         if (rocker_port->stp_state != BR_STATE_LEARNING &&
1484             rocker_port->stp_state != BR_STATE_FORWARDING)
1485                 return 0;
1486
1487         return rocker_port_fdb(rocker_port, NULL, addr, vlan_id, flags);
1488 }
1489
1490 static int rocker_event_process(const struct rocker *rocker,
1491                                 const struct rocker_desc_info *desc_info)
1492 {
1493         const struct rocker_tlv *attrs[ROCKER_TLV_EVENT_MAX + 1];
1494         const struct rocker_tlv *info;
1495         u16 type;
1496
1497         rocker_tlv_parse_desc(attrs, ROCKER_TLV_EVENT_MAX, desc_info);
1498         if (!attrs[ROCKER_TLV_EVENT_TYPE] ||
1499             !attrs[ROCKER_TLV_EVENT_INFO])
1500                 return -EIO;
1501
1502         type = rocker_tlv_get_u16(attrs[ROCKER_TLV_EVENT_TYPE]);
1503         info = attrs[ROCKER_TLV_EVENT_INFO];
1504
1505         switch (type) {
1506         case ROCKER_TLV_EVENT_TYPE_LINK_CHANGED:
1507                 return rocker_event_link_change(rocker, info);
1508         case ROCKER_TLV_EVENT_TYPE_MAC_VLAN_SEEN:
1509                 return rocker_event_mac_vlan_seen(rocker, info);
1510         }
1511
1512         return -EOPNOTSUPP;
1513 }
1514
1515 static irqreturn_t rocker_event_irq_handler(int irq, void *dev_id)
1516 {
1517         struct rocker *rocker = dev_id;
1518         const struct pci_dev *pdev = rocker->pdev;
1519         const struct rocker_desc_info *desc_info;
1520         u32 credits = 0;
1521         int err;
1522
1523         while ((desc_info = rocker_desc_tail_get(&rocker->event_ring))) {
1524                 err = rocker_desc_err(desc_info);
1525                 if (err) {
1526                         dev_err(&pdev->dev, "event desc received with err %d\n",
1527                                 err);
1528                 } else {
1529                         err = rocker_event_process(rocker, desc_info);
1530                         if (err)
1531                                 dev_err(&pdev->dev, "event processing failed with err %d\n",
1532                                         err);
1533                 }
1534                 rocker_desc_gen_clear(desc_info);
1535                 rocker_desc_head_set(rocker, &rocker->event_ring, desc_info);
1536                 credits++;
1537         }
1538         rocker_dma_ring_credits_set(rocker, &rocker->event_ring, credits);
1539
1540         return IRQ_HANDLED;
1541 }
1542
1543 static irqreturn_t rocker_tx_irq_handler(int irq, void *dev_id)
1544 {
1545         struct rocker_port *rocker_port = dev_id;
1546
1547         napi_schedule(&rocker_port->napi_tx);
1548         return IRQ_HANDLED;
1549 }
1550
1551 static irqreturn_t rocker_rx_irq_handler(int irq, void *dev_id)
1552 {
1553         struct rocker_port *rocker_port = dev_id;
1554
1555         napi_schedule(&rocker_port->napi_rx);
1556         return IRQ_HANDLED;
1557 }
1558
1559 /********************
1560  * Command interface
1561  ********************/
1562
1563 typedef int (*rocker_cmd_prep_cb_t)(const struct rocker_port *rocker_port,
1564                                     struct rocker_desc_info *desc_info,
1565                                     void *priv);
1566
1567 typedef int (*rocker_cmd_proc_cb_t)(const struct rocker_port *rocker_port,
1568                                     const struct rocker_desc_info *desc_info,
1569                                     void *priv);
1570
1571 static int rocker_cmd_exec(struct rocker_port *rocker_port,
1572                            struct switchdev_trans *trans, int flags,
1573                            rocker_cmd_prep_cb_t prepare, void *prepare_priv,
1574                            rocker_cmd_proc_cb_t process, void *process_priv)
1575 {
1576         struct rocker *rocker = rocker_port->rocker;
1577         struct rocker_desc_info *desc_info;
1578         struct rocker_wait *wait;
1579         bool nowait = !!(flags & ROCKER_OP_FLAG_NOWAIT);
1580         unsigned long lock_flags;
1581         int err;
1582
1583         wait = rocker_wait_create(rocker_port, trans, flags);
1584         if (!wait)
1585                 return -ENOMEM;
1586         wait->nowait = nowait;
1587
1588         spin_lock_irqsave(&rocker->cmd_ring_lock, lock_flags);
1589
1590         desc_info = rocker_desc_head_get(&rocker->cmd_ring);
1591         if (!desc_info) {
1592                 spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1593                 err = -EAGAIN;
1594                 goto out;
1595         }
1596
1597         err = prepare(rocker_port, desc_info, prepare_priv);
1598         if (err) {
1599                 spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1600                 goto out;
1601         }
1602
1603         rocker_desc_cookie_ptr_set(desc_info, wait);
1604
1605         if (!switchdev_trans_ph_prepare(trans))
1606                 rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);
1607
1608         spin_unlock_irqrestore(&rocker->cmd_ring_lock, lock_flags);
1609
1610         if (nowait)
1611                 return 0;
1612
1613         if (!switchdev_trans_ph_prepare(trans))
1614                 if (!rocker_wait_event_timeout(wait, HZ / 10))
1615                         return -EIO;
1616
1617         err = rocker_desc_err(desc_info);
1618         if (err)
1619                 return err;
1620
1621         if (process)
1622                 err = process(rocker_port, desc_info, process_priv);
1623
1624         rocker_desc_gen_clear(desc_info);
1625 out:
1626         rocker_wait_destroy(trans, wait);
1627         return err;
1628 }
1629
1630 static int
1631 rocker_cmd_get_port_settings_prep(const struct rocker_port *rocker_port,
1632                                   struct rocker_desc_info *desc_info,
1633                                   void *priv)
1634 {
1635         struct rocker_tlv *cmd_info;
1636
1637         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1638                                ROCKER_TLV_CMD_TYPE_GET_PORT_SETTINGS))
1639                 return -EMSGSIZE;
1640         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1641         if (!cmd_info)
1642                 return -EMSGSIZE;
1643         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1644                                rocker_port->pport))
1645                 return -EMSGSIZE;
1646         rocker_tlv_nest_end(desc_info, cmd_info);
1647         return 0;
1648 }
1649
1650 static int
1651 rocker_cmd_get_port_settings_ethtool_proc(const struct rocker_port *rocker_port,
1652                                           const struct rocker_desc_info *desc_info,
1653                                           void *priv)
1654 {
1655         struct ethtool_cmd *ecmd = priv;
1656         const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1657         const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1658         u32 speed;
1659         u8 duplex;
1660         u8 autoneg;
1661
1662         rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1663         if (!attrs[ROCKER_TLV_CMD_INFO])
1664                 return -EIO;
1665
1666         rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1667                                 attrs[ROCKER_TLV_CMD_INFO]);
1668         if (!info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED] ||
1669             !info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX] ||
1670             !info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG])
1671                 return -EIO;
1672
1673         speed = rocker_tlv_get_u32(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED]);
1674         duplex = rocker_tlv_get_u8(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX]);
1675         autoneg = rocker_tlv_get_u8(info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG]);
1676
1677         ecmd->transceiver = XCVR_INTERNAL;
1678         ecmd->supported = SUPPORTED_TP;
1679         ecmd->phy_address = 0xff;
1680         ecmd->port = PORT_TP;
1681         ethtool_cmd_speed_set(ecmd, speed);
1682         ecmd->duplex = duplex ? DUPLEX_FULL : DUPLEX_HALF;
1683         ecmd->autoneg = autoneg ? AUTONEG_ENABLE : AUTONEG_DISABLE;
1684
1685         return 0;
1686 }
1687
1688 static int
1689 rocker_cmd_get_port_settings_macaddr_proc(const struct rocker_port *rocker_port,
1690                                           const struct rocker_desc_info *desc_info,
1691                                           void *priv)
1692 {
1693         unsigned char *macaddr = priv;
1694         const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1695         const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1696         const struct rocker_tlv *attr;
1697
1698         rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1699         if (!attrs[ROCKER_TLV_CMD_INFO])
1700                 return -EIO;
1701
1702         rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1703                                 attrs[ROCKER_TLV_CMD_INFO]);
1704         attr = info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR];
1705         if (!attr)
1706                 return -EIO;
1707
1708         if (rocker_tlv_len(attr) != ETH_ALEN)
1709                 return -EINVAL;
1710
1711         ether_addr_copy(macaddr, rocker_tlv_data(attr));
1712         return 0;
1713 }
1714
1715 struct port_name {
1716         char *buf;
1717         size_t len;
1718 };
1719
1720 static int
1721 rocker_cmd_get_port_settings_phys_name_proc(const struct rocker_port *rocker_port,
1722                                             const struct rocker_desc_info *desc_info,
1723                                             void *priv)
1724 {
1725         const struct rocker_tlv *info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
1726         const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
1727         struct port_name *name = priv;
1728         const struct rocker_tlv *attr;
1729         size_t i, j, len;
1730         const char *str;
1731
1732         rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
1733         if (!attrs[ROCKER_TLV_CMD_INFO])
1734                 return -EIO;
1735
1736         rocker_tlv_parse_nested(info_attrs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
1737                                 attrs[ROCKER_TLV_CMD_INFO]);
1738         attr = info_attrs[ROCKER_TLV_CMD_PORT_SETTINGS_PHYS_NAME];
1739         if (!attr)
1740                 return -EIO;
1741
1742         len = min_t(size_t, rocker_tlv_len(attr), name->len);
1743         str = rocker_tlv_data(attr);
1744
1745         /* make sure name only contains alphanumeric characters */
1746         for (i = j = 0; i < len; ++i) {
1747                 if (isalnum(str[i])) {
1748                         name->buf[j] = str[i];
1749                         j++;
1750                 }
1751         }
1752
1753         if (j == 0)
1754                 return -EIO;
1755
1756         name->buf[j] = '\0';
1757
1758         return 0;
1759 }
1760
1761 static int
1762 rocker_cmd_set_port_settings_ethtool_prep(const struct rocker_port *rocker_port,
1763                                           struct rocker_desc_info *desc_info,
1764                                           void *priv)
1765 {
1766         struct ethtool_cmd *ecmd = priv;
1767         struct rocker_tlv *cmd_info;
1768
1769         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1770                                ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1771                 return -EMSGSIZE;
1772         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1773         if (!cmd_info)
1774                 return -EMSGSIZE;
1775         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1776                                rocker_port->pport))
1777                 return -EMSGSIZE;
1778         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,
1779                                ethtool_cmd_speed(ecmd)))
1780                 return -EMSGSIZE;
1781         if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,
1782                               ecmd->duplex))
1783                 return -EMSGSIZE;
1784         if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,
1785                               ecmd->autoneg))
1786                 return -EMSGSIZE;
1787         rocker_tlv_nest_end(desc_info, cmd_info);
1788         return 0;
1789 }
1790
1791 static int
1792 rocker_cmd_set_port_settings_macaddr_prep(const struct rocker_port *rocker_port,
1793                                           struct rocker_desc_info *desc_info,
1794                                           void *priv)
1795 {
1796         const unsigned char *macaddr = priv;
1797         struct rocker_tlv *cmd_info;
1798
1799         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1800                                ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1801                 return -EMSGSIZE;
1802         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1803         if (!cmd_info)
1804                 return -EMSGSIZE;
1805         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1806                                rocker_port->pport))
1807                 return -EMSGSIZE;
1808         if (rocker_tlv_put(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,
1809                            ETH_ALEN, macaddr))
1810                 return -EMSGSIZE;
1811         rocker_tlv_nest_end(desc_info, cmd_info);
1812         return 0;
1813 }
1814
1815 static int
1816 rocker_cmd_set_port_settings_mtu_prep(const struct rocker_port *rocker_port,
1817                                       struct rocker_desc_info *desc_info,
1818                                       void *priv)
1819 {
1820         int mtu = *(int *)priv;
1821         struct rocker_tlv *cmd_info;
1822
1823         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1824                                ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1825                 return -EMSGSIZE;
1826         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1827         if (!cmd_info)
1828                 return -EMSGSIZE;
1829         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1830                                rocker_port->pport))
1831                 return -EMSGSIZE;
1832         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MTU,
1833                                mtu))
1834                 return -EMSGSIZE;
1835         rocker_tlv_nest_end(desc_info, cmd_info);
1836         return 0;
1837 }
1838
1839 static int
1840 rocker_cmd_set_port_learning_prep(const struct rocker_port *rocker_port,
1841                                   struct rocker_desc_info *desc_info,
1842                                   void *priv)
1843 {
1844         struct rocker_tlv *cmd_info;
1845
1846         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
1847                                ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS))
1848                 return -EMSGSIZE;
1849         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
1850         if (!cmd_info)
1851                 return -EMSGSIZE;
1852         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,
1853                                rocker_port->pport))
1854                 return -EMSGSIZE;
1855         if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,
1856                               !!(rocker_port->brport_flags & BR_LEARNING)))
1857                 return -EMSGSIZE;
1858         rocker_tlv_nest_end(desc_info, cmd_info);
1859         return 0;
1860 }
1861
1862 static int rocker_cmd_get_port_settings_ethtool(struct rocker_port *rocker_port,
1863                                                 struct ethtool_cmd *ecmd)
1864 {
1865         return rocker_cmd_exec(rocker_port, NULL, 0,
1866                                rocker_cmd_get_port_settings_prep, NULL,
1867                                rocker_cmd_get_port_settings_ethtool_proc,
1868                                ecmd);
1869 }
1870
1871 static int rocker_cmd_get_port_settings_macaddr(struct rocker_port *rocker_port,
1872                                                 unsigned char *macaddr)
1873 {
1874         return rocker_cmd_exec(rocker_port, NULL, 0,
1875                                rocker_cmd_get_port_settings_prep, NULL,
1876                                rocker_cmd_get_port_settings_macaddr_proc,
1877                                macaddr);
1878 }
1879
1880 static int rocker_cmd_set_port_settings_ethtool(struct rocker_port *rocker_port,
1881                                                 struct ethtool_cmd *ecmd)
1882 {
1883         return rocker_cmd_exec(rocker_port, NULL, 0,
1884                                rocker_cmd_set_port_settings_ethtool_prep,
1885                                ecmd, NULL, NULL);
1886 }
1887
1888 static int rocker_cmd_set_port_settings_macaddr(struct rocker_port *rocker_port,
1889                                                 unsigned char *macaddr)
1890 {
1891         return rocker_cmd_exec(rocker_port, NULL, 0,
1892                                rocker_cmd_set_port_settings_macaddr_prep,
1893                                macaddr, NULL, NULL);
1894 }
1895
1896 static int rocker_cmd_set_port_settings_mtu(struct rocker_port *rocker_port,
1897                                             int mtu)
1898 {
1899         return rocker_cmd_exec(rocker_port, NULL, 0,
1900                                rocker_cmd_set_port_settings_mtu_prep,
1901                                &mtu, NULL, NULL);
1902 }
1903
1904 static int rocker_port_set_learning(struct rocker_port *rocker_port,
1905                                     struct switchdev_trans *trans)
1906 {
1907         return rocker_cmd_exec(rocker_port, trans, 0,
1908                                rocker_cmd_set_port_learning_prep,
1909                                NULL, NULL, NULL);
1910 }
1911
1912 static int
1913 rocker_cmd_flow_tbl_add_ig_port(struct rocker_desc_info *desc_info,
1914                                 const struct rocker_flow_tbl_entry *entry)
1915 {
1916         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT,
1917                                entry->key.ig_port.in_pport))
1918                 return -EMSGSIZE;
1919         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT_MASK,
1920                                entry->key.ig_port.in_pport_mask))
1921                 return -EMSGSIZE;
1922         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,
1923                                entry->key.ig_port.goto_tbl))
1924                 return -EMSGSIZE;
1925
1926         return 0;
1927 }
1928
1929 static int
1930 rocker_cmd_flow_tbl_add_vlan(struct rocker_desc_info *desc_info,
1931                              const struct rocker_flow_tbl_entry *entry)
1932 {
1933         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT,
1934                                entry->key.vlan.in_pport))
1935                 return -EMSGSIZE;
1936         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
1937                                 entry->key.vlan.vlan_id))
1938                 return -EMSGSIZE;
1939         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID_MASK,
1940                                 entry->key.vlan.vlan_id_mask))
1941                 return -EMSGSIZE;
1942         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,
1943                                entry->key.vlan.goto_tbl))
1944                 return -EMSGSIZE;
1945         if (entry->key.vlan.untagged &&
1946             rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_NEW_VLAN_ID,
1947                                 entry->key.vlan.new_vlan_id))
1948                 return -EMSGSIZE;
1949
1950         return 0;
1951 }
1952
1953 static int
1954 rocker_cmd_flow_tbl_add_term_mac(struct rocker_desc_info *desc_info,
1955                                  const struct rocker_flow_tbl_entry *entry)
1956 {
1957         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT,
1958                                entry->key.term_mac.in_pport))
1959                 return -EMSGSIZE;
1960         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT_MASK,
1961                                entry->key.term_mac.in_pport_mask))
1962                 return -EMSGSIZE;
1963         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_ETHERTYPE,
1964                                 entry->key.term_mac.eth_type))
1965                 return -EMSGSIZE;
1966         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC,
1967                            ETH_ALEN, entry->key.term_mac.eth_dst))
1968                 return -EMSGSIZE;
1969         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC_MASK,
1970                            ETH_ALEN, entry->key.term_mac.eth_dst_mask))
1971                 return -EMSGSIZE;
1972         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
1973                                 entry->key.term_mac.vlan_id))
1974                 return -EMSGSIZE;
1975         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID_MASK,
1976                                 entry->key.term_mac.vlan_id_mask))
1977                 return -EMSGSIZE;
1978         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,
1979                                entry->key.term_mac.goto_tbl))
1980                 return -EMSGSIZE;
1981         if (entry->key.term_mac.copy_to_cpu &&
1982             rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_COPY_CPU_ACTION,
1983                               entry->key.term_mac.copy_to_cpu))
1984                 return -EMSGSIZE;
1985
1986         return 0;
1987 }
1988
1989 static int
1990 rocker_cmd_flow_tbl_add_ucast_routing(struct rocker_desc_info *desc_info,
1991                                       const struct rocker_flow_tbl_entry *entry)
1992 {
1993         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_ETHERTYPE,
1994                                 entry->key.ucast_routing.eth_type))
1995                 return -EMSGSIZE;
1996         if (rocker_tlv_put_be32(desc_info, ROCKER_TLV_OF_DPA_DST_IP,
1997                                 entry->key.ucast_routing.dst4))
1998                 return -EMSGSIZE;
1999         if (rocker_tlv_put_be32(desc_info, ROCKER_TLV_OF_DPA_DST_IP_MASK,
2000                                 entry->key.ucast_routing.dst4_mask))
2001                 return -EMSGSIZE;
2002         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,
2003                                entry->key.ucast_routing.goto_tbl))
2004                 return -EMSGSIZE;
2005         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID,
2006                                entry->key.ucast_routing.group_id))
2007                 return -EMSGSIZE;
2008
2009         return 0;
2010 }
2011
2012 static int
2013 rocker_cmd_flow_tbl_add_bridge(struct rocker_desc_info *desc_info,
2014                                const struct rocker_flow_tbl_entry *entry)
2015 {
2016         if (entry->key.bridge.has_eth_dst &&
2017             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC,
2018                            ETH_ALEN, entry->key.bridge.eth_dst))
2019                 return -EMSGSIZE;
2020         if (entry->key.bridge.has_eth_dst_mask &&
2021             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC_MASK,
2022                            ETH_ALEN, entry->key.bridge.eth_dst_mask))
2023                 return -EMSGSIZE;
2024         if (entry->key.bridge.vlan_id &&
2025             rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
2026                                 entry->key.bridge.vlan_id))
2027                 return -EMSGSIZE;
2028         if (entry->key.bridge.tunnel_id &&
2029             rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_TUNNEL_ID,
2030                                entry->key.bridge.tunnel_id))
2031                 return -EMSGSIZE;
2032         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,
2033                                entry->key.bridge.goto_tbl))
2034                 return -EMSGSIZE;
2035         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID,
2036                                entry->key.bridge.group_id))
2037                 return -EMSGSIZE;
2038         if (entry->key.bridge.copy_to_cpu &&
2039             rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_COPY_CPU_ACTION,
2040                               entry->key.bridge.copy_to_cpu))
2041                 return -EMSGSIZE;
2042
2043         return 0;
2044 }
2045
2046 static int
2047 rocker_cmd_flow_tbl_add_acl(struct rocker_desc_info *desc_info,
2048                             const struct rocker_flow_tbl_entry *entry)
2049 {
2050         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT,
2051                                entry->key.acl.in_pport))
2052                 return -EMSGSIZE;
2053         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_IN_PPORT_MASK,
2054                                entry->key.acl.in_pport_mask))
2055                 return -EMSGSIZE;
2056         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_SRC_MAC,
2057                            ETH_ALEN, entry->key.acl.eth_src))
2058                 return -EMSGSIZE;
2059         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_SRC_MAC_MASK,
2060                            ETH_ALEN, entry->key.acl.eth_src_mask))
2061                 return -EMSGSIZE;
2062         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC,
2063                            ETH_ALEN, entry->key.acl.eth_dst))
2064                 return -EMSGSIZE;
2065         if (rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC_MASK,
2066                            ETH_ALEN, entry->key.acl.eth_dst_mask))
2067                 return -EMSGSIZE;
2068         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_ETHERTYPE,
2069                                 entry->key.acl.eth_type))
2070                 return -EMSGSIZE;
2071         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
2072                                 entry->key.acl.vlan_id))
2073                 return -EMSGSIZE;
2074         if (rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID_MASK,
2075                                 entry->key.acl.vlan_id_mask))
2076                 return -EMSGSIZE;
2077
2078         switch (ntohs(entry->key.acl.eth_type)) {
2079         case ETH_P_IP:
2080         case ETH_P_IPV6:
2081                 if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_IP_PROTO,
2082                                       entry->key.acl.ip_proto))
2083                         return -EMSGSIZE;
2084                 if (rocker_tlv_put_u8(desc_info,
2085                                       ROCKER_TLV_OF_DPA_IP_PROTO_MASK,
2086                                       entry->key.acl.ip_proto_mask))
2087                         return -EMSGSIZE;
2088                 if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_IP_DSCP,
2089                                       entry->key.acl.ip_tos & 0x3f))
2090                         return -EMSGSIZE;
2091                 if (rocker_tlv_put_u8(desc_info,
2092                                       ROCKER_TLV_OF_DPA_IP_DSCP_MASK,
2093                                       entry->key.acl.ip_tos_mask & 0x3f))
2094                         return -EMSGSIZE;
2095                 if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_IP_ECN,
2096                                       (entry->key.acl.ip_tos & 0xc0) >> 6))
2097                         return -EMSGSIZE;
2098                 if (rocker_tlv_put_u8(desc_info,
2099                                       ROCKER_TLV_OF_DPA_IP_ECN_MASK,
2100                                       (entry->key.acl.ip_tos_mask & 0xc0) >> 6))
2101                         return -EMSGSIZE;
2102                 break;
2103         }
2104
2105         if (entry->key.acl.group_id != ROCKER_GROUP_NONE &&
2106             rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID,
2107                                entry->key.acl.group_id))
2108                 return -EMSGSIZE;
2109
2110         return 0;
2111 }
2112
2113 static int rocker_cmd_flow_tbl_add(const struct rocker_port *rocker_port,
2114                                    struct rocker_desc_info *desc_info,
2115                                    void *priv)
2116 {
2117         const struct rocker_flow_tbl_entry *entry = priv;
2118         struct rocker_tlv *cmd_info;
2119         int err = 0;
2120
2121         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE, entry->cmd))
2122                 return -EMSGSIZE;
2123         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
2124         if (!cmd_info)
2125                 return -EMSGSIZE;
2126         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_TABLE_ID,
2127                                entry->key.tbl_id))
2128                 return -EMSGSIZE;
2129         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_PRIORITY,
2130                                entry->key.priority))
2131                 return -EMSGSIZE;
2132         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_HARDTIME, 0))
2133                 return -EMSGSIZE;
2134         if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_OF_DPA_COOKIE,
2135                                entry->cookie))
2136                 return -EMSGSIZE;
2137
2138         switch (entry->key.tbl_id) {
2139         case ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT:
2140                 err = rocker_cmd_flow_tbl_add_ig_port(desc_info, entry);
2141                 break;
2142         case ROCKER_OF_DPA_TABLE_ID_VLAN:
2143                 err = rocker_cmd_flow_tbl_add_vlan(desc_info, entry);
2144                 break;
2145         case ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC:
2146                 err = rocker_cmd_flow_tbl_add_term_mac(desc_info, entry);
2147                 break;
2148         case ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING:
2149                 err = rocker_cmd_flow_tbl_add_ucast_routing(desc_info, entry);
2150                 break;
2151         case ROCKER_OF_DPA_TABLE_ID_BRIDGING:
2152                 err = rocker_cmd_flow_tbl_add_bridge(desc_info, entry);
2153                 break;
2154         case ROCKER_OF_DPA_TABLE_ID_ACL_POLICY:
2155                 err = rocker_cmd_flow_tbl_add_acl(desc_info, entry);
2156                 break;
2157         default:
2158                 err = -ENOTSUPP;
2159                 break;
2160         }
2161
2162         if (err)
2163                 return err;
2164
2165         rocker_tlv_nest_end(desc_info, cmd_info);
2166
2167         return 0;
2168 }
2169
2170 static int rocker_cmd_flow_tbl_del(const struct rocker_port *rocker_port,
2171                                    struct rocker_desc_info *desc_info,
2172                                    void *priv)
2173 {
2174         const struct rocker_flow_tbl_entry *entry = priv;
2175         struct rocker_tlv *cmd_info;
2176
2177         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE, entry->cmd))
2178                 return -EMSGSIZE;
2179         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
2180         if (!cmd_info)
2181                 return -EMSGSIZE;
2182         if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_OF_DPA_COOKIE,
2183                                entry->cookie))
2184                 return -EMSGSIZE;
2185         rocker_tlv_nest_end(desc_info, cmd_info);
2186
2187         return 0;
2188 }
2189
2190 static int
2191 rocker_cmd_group_tbl_add_l2_interface(struct rocker_desc_info *desc_info,
2192                                       struct rocker_group_tbl_entry *entry)
2193 {
2194         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_OUT_PPORT,
2195                                ROCKER_GROUP_PORT_GET(entry->group_id)))
2196                 return -EMSGSIZE;
2197         if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_POP_VLAN,
2198                               entry->l2_interface.pop_vlan))
2199                 return -EMSGSIZE;
2200
2201         return 0;
2202 }
2203
2204 static int
2205 rocker_cmd_group_tbl_add_l2_rewrite(struct rocker_desc_info *desc_info,
2206                                     const struct rocker_group_tbl_entry *entry)
2207 {
2208         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID_LOWER,
2209                                entry->l2_rewrite.group_id))
2210                 return -EMSGSIZE;
2211         if (!is_zero_ether_addr(entry->l2_rewrite.eth_src) &&
2212             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_SRC_MAC,
2213                            ETH_ALEN, entry->l2_rewrite.eth_src))
2214                 return -EMSGSIZE;
2215         if (!is_zero_ether_addr(entry->l2_rewrite.eth_dst) &&
2216             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC,
2217                            ETH_ALEN, entry->l2_rewrite.eth_dst))
2218                 return -EMSGSIZE;
2219         if (entry->l2_rewrite.vlan_id &&
2220             rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
2221                                 entry->l2_rewrite.vlan_id))
2222                 return -EMSGSIZE;
2223
2224         return 0;
2225 }
2226
2227 static int
2228 rocker_cmd_group_tbl_add_group_ids(struct rocker_desc_info *desc_info,
2229                                    const struct rocker_group_tbl_entry *entry)
2230 {
2231         int i;
2232         struct rocker_tlv *group_ids;
2233
2234         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_OF_DPA_GROUP_COUNT,
2235                                entry->group_count))
2236                 return -EMSGSIZE;
2237
2238         group_ids = rocker_tlv_nest_start(desc_info,
2239                                           ROCKER_TLV_OF_DPA_GROUP_IDS);
2240         if (!group_ids)
2241                 return -EMSGSIZE;
2242
2243         for (i = 0; i < entry->group_count; i++)
2244                 /* Note TLV array is 1-based */
2245                 if (rocker_tlv_put_u32(desc_info, i + 1, entry->group_ids[i]))
2246                         return -EMSGSIZE;
2247
2248         rocker_tlv_nest_end(desc_info, group_ids);
2249
2250         return 0;
2251 }
2252
2253 static int
2254 rocker_cmd_group_tbl_add_l3_unicast(struct rocker_desc_info *desc_info,
2255                                     const struct rocker_group_tbl_entry *entry)
2256 {
2257         if (!is_zero_ether_addr(entry->l3_unicast.eth_src) &&
2258             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_SRC_MAC,
2259                            ETH_ALEN, entry->l3_unicast.eth_src))
2260                 return -EMSGSIZE;
2261         if (!is_zero_ether_addr(entry->l3_unicast.eth_dst) &&
2262             rocker_tlv_put(desc_info, ROCKER_TLV_OF_DPA_DST_MAC,
2263                            ETH_ALEN, entry->l3_unicast.eth_dst))
2264                 return -EMSGSIZE;
2265         if (entry->l3_unicast.vlan_id &&
2266             rocker_tlv_put_be16(desc_info, ROCKER_TLV_OF_DPA_VLAN_ID,
2267                                 entry->l3_unicast.vlan_id))
2268                 return -EMSGSIZE;
2269         if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_OF_DPA_TTL_CHECK,
2270                               entry->l3_unicast.ttl_check))
2271                 return -EMSGSIZE;
2272         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID_LOWER,
2273                                entry->l3_unicast.group_id))
2274                 return -EMSGSIZE;
2275
2276         return 0;
2277 }
2278
2279 static int rocker_cmd_group_tbl_add(const struct rocker_port *rocker_port,
2280                                     struct rocker_desc_info *desc_info,
2281                                     void *priv)
2282 {
2283         struct rocker_group_tbl_entry *entry = priv;
2284         struct rocker_tlv *cmd_info;
2285         int err = 0;
2286
2287         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE, entry->cmd))
2288                 return -EMSGSIZE;
2289         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
2290         if (!cmd_info)
2291                 return -EMSGSIZE;
2292
2293         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID,
2294                                entry->group_id))
2295                 return -EMSGSIZE;
2296
2297         switch (ROCKER_GROUP_TYPE_GET(entry->group_id)) {
2298         case ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE:
2299                 err = rocker_cmd_group_tbl_add_l2_interface(desc_info, entry);
2300                 break;
2301         case ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE:
2302                 err = rocker_cmd_group_tbl_add_l2_rewrite(desc_info, entry);
2303                 break;
2304         case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
2305         case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
2306                 err = rocker_cmd_group_tbl_add_group_ids(desc_info, entry);
2307                 break;
2308         case ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST:
2309                 err = rocker_cmd_group_tbl_add_l3_unicast(desc_info, entry);
2310                 break;
2311         default:
2312                 err = -ENOTSUPP;
2313                 break;
2314         }
2315
2316         if (err)
2317                 return err;
2318
2319         rocker_tlv_nest_end(desc_info, cmd_info);
2320
2321         return 0;
2322 }
2323
2324 static int rocker_cmd_group_tbl_del(const struct rocker_port *rocker_port,
2325                                     struct rocker_desc_info *desc_info,
2326                                     void *priv)
2327 {
2328         const struct rocker_group_tbl_entry *entry = priv;
2329         struct rocker_tlv *cmd_info;
2330
2331         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE, entry->cmd))
2332                 return -EMSGSIZE;
2333         cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
2334         if (!cmd_info)
2335                 return -EMSGSIZE;
2336         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_OF_DPA_GROUP_ID,
2337                                entry->group_id))
2338                 return -EMSGSIZE;
2339         rocker_tlv_nest_end(desc_info, cmd_info);
2340
2341         return 0;
2342 }
2343
2344 /***************************************************
2345  * Flow, group, FDB, internal VLAN and neigh tables
2346  ***************************************************/
2347
2348 static int rocker_init_tbls(struct rocker *rocker)
2349 {
2350         hash_init(rocker->flow_tbl);
2351         spin_lock_init(&rocker->flow_tbl_lock);
2352
2353         hash_init(rocker->group_tbl);
2354         spin_lock_init(&rocker->group_tbl_lock);
2355
2356         hash_init(rocker->fdb_tbl);
2357         spin_lock_init(&rocker->fdb_tbl_lock);
2358
2359         hash_init(rocker->internal_vlan_tbl);
2360         spin_lock_init(&rocker->internal_vlan_tbl_lock);
2361
2362         hash_init(rocker->neigh_tbl);
2363         spin_lock_init(&rocker->neigh_tbl_lock);
2364
2365         return 0;
2366 }
2367
2368 static void rocker_free_tbls(struct rocker *rocker)
2369 {
2370         unsigned long flags;
2371         struct rocker_flow_tbl_entry *flow_entry;
2372         struct rocker_group_tbl_entry *group_entry;
2373         struct rocker_fdb_tbl_entry *fdb_entry;
2374         struct rocker_internal_vlan_tbl_entry *internal_vlan_entry;
2375         struct rocker_neigh_tbl_entry *neigh_entry;
2376         struct hlist_node *tmp;
2377         int bkt;
2378
2379         spin_lock_irqsave(&rocker->flow_tbl_lock, flags);
2380         hash_for_each_safe(rocker->flow_tbl, bkt, tmp, flow_entry, entry)
2381                 hash_del(&flow_entry->entry);
2382         spin_unlock_irqrestore(&rocker->flow_tbl_lock, flags);
2383
2384         spin_lock_irqsave(&rocker->group_tbl_lock, flags);
2385         hash_for_each_safe(rocker->group_tbl, bkt, tmp, group_entry, entry)
2386                 hash_del(&group_entry->entry);
2387         spin_unlock_irqrestore(&rocker->group_tbl_lock, flags);
2388
2389         spin_lock_irqsave(&rocker->fdb_tbl_lock, flags);
2390         hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, fdb_entry, entry)
2391                 hash_del(&fdb_entry->entry);
2392         spin_unlock_irqrestore(&rocker->fdb_tbl_lock, flags);
2393
2394         spin_lock_irqsave(&rocker->internal_vlan_tbl_lock, flags);
2395         hash_for_each_safe(rocker->internal_vlan_tbl, bkt,
2396                            tmp, internal_vlan_entry, entry)
2397                 hash_del(&internal_vlan_entry->entry);
2398         spin_unlock_irqrestore(&rocker->internal_vlan_tbl_lock, flags);
2399
2400         spin_lock_irqsave(&rocker->neigh_tbl_lock, flags);
2401         hash_for_each_safe(rocker->neigh_tbl, bkt, tmp, neigh_entry, entry)
2402                 hash_del(&neigh_entry->entry);
2403         spin_unlock_irqrestore(&rocker->neigh_tbl_lock, flags);
2404 }
2405
2406 static struct rocker_flow_tbl_entry *
2407 rocker_flow_tbl_find(const struct rocker *rocker,
2408                      const struct rocker_flow_tbl_entry *match)
2409 {
2410         struct rocker_flow_tbl_entry *found;
2411         size_t key_len = match->key_len ? match->key_len : sizeof(found->key);
2412
2413         hash_for_each_possible(rocker->flow_tbl, found,
2414                                entry, match->key_crc32) {
2415                 if (memcmp(&found->key, &match->key, key_len) == 0)
2416                         return found;
2417         }
2418
2419         return NULL;
2420 }
2421
2422 static int rocker_flow_tbl_add(struct rocker_port *rocker_port,
2423                                struct switchdev_trans *trans, int flags,
2424                                struct rocker_flow_tbl_entry *match)
2425 {
2426         struct rocker *rocker = rocker_port->rocker;
2427         struct rocker_flow_tbl_entry *found;
2428         size_t key_len = match->key_len ? match->key_len : sizeof(found->key);
2429         unsigned long lock_flags;
2430
2431         match->key_crc32 = crc32(~0, &match->key, key_len);
2432
2433         spin_lock_irqsave(&rocker->flow_tbl_lock, lock_flags);
2434
2435         found = rocker_flow_tbl_find(rocker, match);
2436
2437         if (found) {
2438                 match->cookie = found->cookie;
2439                 if (!switchdev_trans_ph_prepare(trans))
2440                         hash_del(&found->entry);
2441                 rocker_kfree(trans, found);
2442                 found = match;
2443                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_MOD;
2444         } else {
2445                 found = match;
2446                 found->cookie = rocker->flow_tbl_next_cookie++;
2447                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD;
2448         }
2449
2450         if (!switchdev_trans_ph_prepare(trans))
2451                 hash_add(rocker->flow_tbl, &found->entry, found->key_crc32);
2452
2453         spin_unlock_irqrestore(&rocker->flow_tbl_lock, lock_flags);
2454
2455         return rocker_cmd_exec(rocker_port, trans, flags,
2456                                rocker_cmd_flow_tbl_add, found, NULL, NULL);
2457 }
2458
2459 static int rocker_flow_tbl_del(struct rocker_port *rocker_port,
2460                                struct switchdev_trans *trans, int flags,
2461                                struct rocker_flow_tbl_entry *match)
2462 {
2463         struct rocker *rocker = rocker_port->rocker;
2464         struct rocker_flow_tbl_entry *found;
2465         size_t key_len = match->key_len ? match->key_len : sizeof(found->key);
2466         unsigned long lock_flags;
2467         int err = 0;
2468
2469         match->key_crc32 = crc32(~0, &match->key, key_len);
2470
2471         spin_lock_irqsave(&rocker->flow_tbl_lock, lock_flags);
2472
2473         found = rocker_flow_tbl_find(rocker, match);
2474
2475         if (found) {
2476                 if (!switchdev_trans_ph_prepare(trans))
2477                         hash_del(&found->entry);
2478                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL;
2479         }
2480
2481         spin_unlock_irqrestore(&rocker->flow_tbl_lock, lock_flags);
2482
2483         rocker_kfree(trans, match);
2484
2485         if (found) {
2486                 err = rocker_cmd_exec(rocker_port, trans, flags,
2487                                       rocker_cmd_flow_tbl_del,
2488                                       found, NULL, NULL);
2489                 rocker_kfree(trans, found);
2490         }
2491
2492         return err;
2493 }
2494
2495 static int rocker_flow_tbl_do(struct rocker_port *rocker_port,
2496                               struct switchdev_trans *trans, int flags,
2497                               struct rocker_flow_tbl_entry *entry)
2498 {
2499         if (flags & ROCKER_OP_FLAG_REMOVE)
2500                 return rocker_flow_tbl_del(rocker_port, trans, flags, entry);
2501         else
2502                 return rocker_flow_tbl_add(rocker_port, trans, flags, entry);
2503 }
2504
2505 static int rocker_flow_tbl_ig_port(struct rocker_port *rocker_port,
2506                                    struct switchdev_trans *trans, int flags,
2507                                    u32 in_pport, u32 in_pport_mask,
2508                                    enum rocker_of_dpa_table_id goto_tbl)
2509 {
2510         struct rocker_flow_tbl_entry *entry;
2511
2512         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2513         if (!entry)
2514                 return -ENOMEM;
2515
2516         entry->key.priority = ROCKER_PRIORITY_IG_PORT;
2517         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT;
2518         entry->key.ig_port.in_pport = in_pport;
2519         entry->key.ig_port.in_pport_mask = in_pport_mask;
2520         entry->key.ig_port.goto_tbl = goto_tbl;
2521
2522         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2523 }
2524
2525 static int rocker_flow_tbl_vlan(struct rocker_port *rocker_port,
2526                                 struct switchdev_trans *trans, int flags,
2527                                 u32 in_pport, __be16 vlan_id,
2528                                 __be16 vlan_id_mask,
2529                                 enum rocker_of_dpa_table_id goto_tbl,
2530                                 bool untagged, __be16 new_vlan_id)
2531 {
2532         struct rocker_flow_tbl_entry *entry;
2533
2534         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2535         if (!entry)
2536                 return -ENOMEM;
2537
2538         entry->key.priority = ROCKER_PRIORITY_VLAN;
2539         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_VLAN;
2540         entry->key.vlan.in_pport = in_pport;
2541         entry->key.vlan.vlan_id = vlan_id;
2542         entry->key.vlan.vlan_id_mask = vlan_id_mask;
2543         entry->key.vlan.goto_tbl = goto_tbl;
2544
2545         entry->key.vlan.untagged = untagged;
2546         entry->key.vlan.new_vlan_id = new_vlan_id;
2547
2548         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2549 }
2550
2551 static int rocker_flow_tbl_term_mac(struct rocker_port *rocker_port,
2552                                     struct switchdev_trans *trans,
2553                                     u32 in_pport, u32 in_pport_mask,
2554                                     __be16 eth_type, const u8 *eth_dst,
2555                                     const u8 *eth_dst_mask, __be16 vlan_id,
2556                                     __be16 vlan_id_mask, bool copy_to_cpu,
2557                                     int flags)
2558 {
2559         struct rocker_flow_tbl_entry *entry;
2560
2561         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2562         if (!entry)
2563                 return -ENOMEM;
2564
2565         if (is_multicast_ether_addr(eth_dst)) {
2566                 entry->key.priority = ROCKER_PRIORITY_TERM_MAC_MCAST;
2567                 entry->key.term_mac.goto_tbl =
2568                          ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING;
2569         } else {
2570                 entry->key.priority = ROCKER_PRIORITY_TERM_MAC_UCAST;
2571                 entry->key.term_mac.goto_tbl =
2572                          ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING;
2573         }
2574
2575         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC;
2576         entry->key.term_mac.in_pport = in_pport;
2577         entry->key.term_mac.in_pport_mask = in_pport_mask;
2578         entry->key.term_mac.eth_type = eth_type;
2579         ether_addr_copy(entry->key.term_mac.eth_dst, eth_dst);
2580         ether_addr_copy(entry->key.term_mac.eth_dst_mask, eth_dst_mask);
2581         entry->key.term_mac.vlan_id = vlan_id;
2582         entry->key.term_mac.vlan_id_mask = vlan_id_mask;
2583         entry->key.term_mac.copy_to_cpu = copy_to_cpu;
2584
2585         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2586 }
2587
2588 static int rocker_flow_tbl_bridge(struct rocker_port *rocker_port,
2589                                   struct switchdev_trans *trans, int flags,
2590                                   const u8 *eth_dst, const u8 *eth_dst_mask,
2591                                   __be16 vlan_id, u32 tunnel_id,
2592                                   enum rocker_of_dpa_table_id goto_tbl,
2593                                   u32 group_id, bool copy_to_cpu)
2594 {
2595         struct rocker_flow_tbl_entry *entry;
2596         u32 priority;
2597         bool vlan_bridging = !!vlan_id;
2598         bool dflt = !eth_dst || (eth_dst && eth_dst_mask);
2599         bool wild = false;
2600
2601         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2602         if (!entry)
2603                 return -ENOMEM;
2604
2605         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_BRIDGING;
2606
2607         if (eth_dst) {
2608                 entry->key.bridge.has_eth_dst = 1;
2609                 ether_addr_copy(entry->key.bridge.eth_dst, eth_dst);
2610         }
2611         if (eth_dst_mask) {
2612                 entry->key.bridge.has_eth_dst_mask = 1;
2613                 ether_addr_copy(entry->key.bridge.eth_dst_mask, eth_dst_mask);
2614                 if (!ether_addr_equal(eth_dst_mask, ff_mac))
2615                         wild = true;
2616         }
2617
2618         priority = ROCKER_PRIORITY_UNKNOWN;
2619         if (vlan_bridging && dflt && wild)
2620                 priority = ROCKER_PRIORITY_BRIDGING_VLAN_DFLT_WILD;
2621         else if (vlan_bridging && dflt && !wild)
2622                 priority = ROCKER_PRIORITY_BRIDGING_VLAN_DFLT_EXACT;
2623         else if (vlan_bridging && !dflt)
2624                 priority = ROCKER_PRIORITY_BRIDGING_VLAN;
2625         else if (!vlan_bridging && dflt && wild)
2626                 priority = ROCKER_PRIORITY_BRIDGING_TENANT_DFLT_WILD;
2627         else if (!vlan_bridging && dflt && !wild)
2628                 priority = ROCKER_PRIORITY_BRIDGING_TENANT_DFLT_EXACT;
2629         else if (!vlan_bridging && !dflt)
2630                 priority = ROCKER_PRIORITY_BRIDGING_TENANT;
2631
2632         entry->key.priority = priority;
2633         entry->key.bridge.vlan_id = vlan_id;
2634         entry->key.bridge.tunnel_id = tunnel_id;
2635         entry->key.bridge.goto_tbl = goto_tbl;
2636         entry->key.bridge.group_id = group_id;
2637         entry->key.bridge.copy_to_cpu = copy_to_cpu;
2638
2639         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2640 }
2641
2642 static int rocker_flow_tbl_ucast4_routing(struct rocker_port *rocker_port,
2643                                           struct switchdev_trans *trans,
2644                                           __be16 eth_type, __be32 dst,
2645                                           __be32 dst_mask, u32 priority,
2646                                           enum rocker_of_dpa_table_id goto_tbl,
2647                                           u32 group_id, int flags)
2648 {
2649         struct rocker_flow_tbl_entry *entry;
2650
2651         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2652         if (!entry)
2653                 return -ENOMEM;
2654
2655         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING;
2656         entry->key.priority = priority;
2657         entry->key.ucast_routing.eth_type = eth_type;
2658         entry->key.ucast_routing.dst4 = dst;
2659         entry->key.ucast_routing.dst4_mask = dst_mask;
2660         entry->key.ucast_routing.goto_tbl = goto_tbl;
2661         entry->key.ucast_routing.group_id = group_id;
2662         entry->key_len = offsetof(struct rocker_flow_tbl_key,
2663                                   ucast_routing.group_id);
2664
2665         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2666 }
2667
2668 static int rocker_flow_tbl_acl(struct rocker_port *rocker_port,
2669                                struct switchdev_trans *trans, int flags,
2670                                u32 in_pport, u32 in_pport_mask,
2671                                const u8 *eth_src, const u8 *eth_src_mask,
2672                                const u8 *eth_dst, const u8 *eth_dst_mask,
2673                                __be16 eth_type, __be16 vlan_id,
2674                                __be16 vlan_id_mask, u8 ip_proto,
2675                                u8 ip_proto_mask, u8 ip_tos, u8 ip_tos_mask,
2676                                u32 group_id)
2677 {
2678         u32 priority;
2679         struct rocker_flow_tbl_entry *entry;
2680
2681         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2682         if (!entry)
2683                 return -ENOMEM;
2684
2685         priority = ROCKER_PRIORITY_ACL_NORMAL;
2686         if (eth_dst && eth_dst_mask) {
2687                 if (ether_addr_equal(eth_dst_mask, mcast_mac))
2688                         priority = ROCKER_PRIORITY_ACL_DFLT;
2689                 else if (is_link_local_ether_addr(eth_dst))
2690                         priority = ROCKER_PRIORITY_ACL_CTRL;
2691         }
2692
2693         entry->key.priority = priority;
2694         entry->key.tbl_id = ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
2695         entry->key.acl.in_pport = in_pport;
2696         entry->key.acl.in_pport_mask = in_pport_mask;
2697
2698         if (eth_src)
2699                 ether_addr_copy(entry->key.acl.eth_src, eth_src);
2700         if (eth_src_mask)
2701                 ether_addr_copy(entry->key.acl.eth_src_mask, eth_src_mask);
2702         if (eth_dst)
2703                 ether_addr_copy(entry->key.acl.eth_dst, eth_dst);
2704         if (eth_dst_mask)
2705                 ether_addr_copy(entry->key.acl.eth_dst_mask, eth_dst_mask);
2706
2707         entry->key.acl.eth_type = eth_type;
2708         entry->key.acl.vlan_id = vlan_id;
2709         entry->key.acl.vlan_id_mask = vlan_id_mask;
2710         entry->key.acl.ip_proto = ip_proto;
2711         entry->key.acl.ip_proto_mask = ip_proto_mask;
2712         entry->key.acl.ip_tos = ip_tos;
2713         entry->key.acl.ip_tos_mask = ip_tos_mask;
2714         entry->key.acl.group_id = group_id;
2715
2716         return rocker_flow_tbl_do(rocker_port, trans, flags, entry);
2717 }
2718
2719 static struct rocker_group_tbl_entry *
2720 rocker_group_tbl_find(const struct rocker *rocker,
2721                       const struct rocker_group_tbl_entry *match)
2722 {
2723         struct rocker_group_tbl_entry *found;
2724
2725         hash_for_each_possible(rocker->group_tbl, found,
2726                                entry, match->group_id) {
2727                 if (found->group_id == match->group_id)
2728                         return found;
2729         }
2730
2731         return NULL;
2732 }
2733
2734 static void rocker_group_tbl_entry_free(struct switchdev_trans *trans,
2735                                         struct rocker_group_tbl_entry *entry)
2736 {
2737         switch (ROCKER_GROUP_TYPE_GET(entry->group_id)) {
2738         case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
2739         case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
2740                 rocker_kfree(trans, entry->group_ids);
2741                 break;
2742         default:
2743                 break;
2744         }
2745         rocker_kfree(trans, entry);
2746 }
2747
2748 static int rocker_group_tbl_add(struct rocker_port *rocker_port,
2749                                 struct switchdev_trans *trans, int flags,
2750                                 struct rocker_group_tbl_entry *match)
2751 {
2752         struct rocker *rocker = rocker_port->rocker;
2753         struct rocker_group_tbl_entry *found;
2754         unsigned long lock_flags;
2755
2756         spin_lock_irqsave(&rocker->group_tbl_lock, lock_flags);
2757
2758         found = rocker_group_tbl_find(rocker, match);
2759
2760         if (found) {
2761                 if (!switchdev_trans_ph_prepare(trans))
2762                         hash_del(&found->entry);
2763                 rocker_group_tbl_entry_free(trans, found);
2764                 found = match;
2765                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD;
2766         } else {
2767                 found = match;
2768                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD;
2769         }
2770
2771         if (!switchdev_trans_ph_prepare(trans))
2772                 hash_add(rocker->group_tbl, &found->entry, found->group_id);
2773
2774         spin_unlock_irqrestore(&rocker->group_tbl_lock, lock_flags);
2775
2776         return rocker_cmd_exec(rocker_port, trans, flags,
2777                                rocker_cmd_group_tbl_add, found, NULL, NULL);
2778 }
2779
2780 static int rocker_group_tbl_del(struct rocker_port *rocker_port,
2781                                 struct switchdev_trans *trans, int flags,
2782                                 struct rocker_group_tbl_entry *match)
2783 {
2784         struct rocker *rocker = rocker_port->rocker;
2785         struct rocker_group_tbl_entry *found;
2786         unsigned long lock_flags;
2787         int err = 0;
2788
2789         spin_lock_irqsave(&rocker->group_tbl_lock, lock_flags);
2790
2791         found = rocker_group_tbl_find(rocker, match);
2792
2793         if (found) {
2794                 if (!switchdev_trans_ph_prepare(trans))
2795                         hash_del(&found->entry);
2796                 found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL;
2797         }
2798
2799         spin_unlock_irqrestore(&rocker->group_tbl_lock, lock_flags);
2800
2801         rocker_group_tbl_entry_free(trans, match);
2802
2803         if (found) {
2804                 err = rocker_cmd_exec(rocker_port, trans, flags,
2805                                       rocker_cmd_group_tbl_del,
2806                                       found, NULL, NULL);
2807                 rocker_group_tbl_entry_free(trans, found);
2808         }
2809
2810         return err;
2811 }
2812
2813 static int rocker_group_tbl_do(struct rocker_port *rocker_port,
2814                                struct switchdev_trans *trans, int flags,
2815                                struct rocker_group_tbl_entry *entry)
2816 {
2817         if (flags & ROCKER_OP_FLAG_REMOVE)
2818                 return rocker_group_tbl_del(rocker_port, trans, flags, entry);
2819         else
2820                 return rocker_group_tbl_add(rocker_port, trans, flags, entry);
2821 }
2822
2823 static int rocker_group_l2_interface(struct rocker_port *rocker_port,
2824                                      struct switchdev_trans *trans, int flags,
2825                                      __be16 vlan_id, u32 out_pport,
2826                                      int pop_vlan)
2827 {
2828         struct rocker_group_tbl_entry *entry;
2829
2830         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2831         if (!entry)
2832                 return -ENOMEM;
2833
2834         entry->group_id = ROCKER_GROUP_L2_INTERFACE(vlan_id, out_pport);
2835         entry->l2_interface.pop_vlan = pop_vlan;
2836
2837         return rocker_group_tbl_do(rocker_port, trans, flags, entry);
2838 }
2839
2840 static int rocker_group_l2_fan_out(struct rocker_port *rocker_port,
2841                                    struct switchdev_trans *trans,
2842                                    int flags, u8 group_count,
2843                                    const u32 *group_ids, u32 group_id)
2844 {
2845         struct rocker_group_tbl_entry *entry;
2846
2847         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2848         if (!entry)
2849                 return -ENOMEM;
2850
2851         entry->group_id = group_id;
2852         entry->group_count = group_count;
2853
2854         entry->group_ids = rocker_kcalloc(trans, flags,
2855                                           group_count, sizeof(u32));
2856         if (!entry->group_ids) {
2857                 rocker_kfree(trans, entry);
2858                 return -ENOMEM;
2859         }
2860         memcpy(entry->group_ids, group_ids, group_count * sizeof(u32));
2861
2862         return rocker_group_tbl_do(rocker_port, trans, flags, entry);
2863 }
2864
2865 static int rocker_group_l2_flood(struct rocker_port *rocker_port,
2866                                  struct switchdev_trans *trans, int flags,
2867                                  __be16 vlan_id, u8 group_count,
2868                                  const u32 *group_ids, u32 group_id)
2869 {
2870         return rocker_group_l2_fan_out(rocker_port, trans, flags,
2871                                        group_count, group_ids,
2872                                        group_id);
2873 }
2874
2875 static int rocker_group_l3_unicast(struct rocker_port *rocker_port,
2876                                    struct switchdev_trans *trans, int flags,
2877                                    u32 index, const u8 *src_mac, const u8 *dst_mac,
2878                                    __be16 vlan_id, bool ttl_check, u32 pport)
2879 {
2880         struct rocker_group_tbl_entry *entry;
2881
2882         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2883         if (!entry)
2884                 return -ENOMEM;
2885
2886         entry->group_id = ROCKER_GROUP_L3_UNICAST(index);
2887         if (src_mac)
2888                 ether_addr_copy(entry->l3_unicast.eth_src, src_mac);
2889         if (dst_mac)
2890                 ether_addr_copy(entry->l3_unicast.eth_dst, dst_mac);
2891         entry->l3_unicast.vlan_id = vlan_id;
2892         entry->l3_unicast.ttl_check = ttl_check;
2893         entry->l3_unicast.group_id = ROCKER_GROUP_L2_INTERFACE(vlan_id, pport);
2894
2895         return rocker_group_tbl_do(rocker_port, trans, flags, entry);
2896 }
2897
2898 static struct rocker_neigh_tbl_entry *
2899 rocker_neigh_tbl_find(const struct rocker *rocker, __be32 ip_addr)
2900 {
2901         struct rocker_neigh_tbl_entry *found;
2902
2903         hash_for_each_possible(rocker->neigh_tbl, found,
2904                                entry, be32_to_cpu(ip_addr))
2905                 if (found->ip_addr == ip_addr)
2906                         return found;
2907
2908         return NULL;
2909 }
2910
2911 static void _rocker_neigh_add(struct rocker *rocker,
2912                               struct switchdev_trans *trans,
2913                               struct rocker_neigh_tbl_entry *entry)
2914 {
2915         if (!switchdev_trans_ph_commit(trans))
2916                 entry->index = rocker->neigh_tbl_next_index++;
2917         if (switchdev_trans_ph_prepare(trans))
2918                 return;
2919         entry->ref_count++;
2920         hash_add(rocker->neigh_tbl, &entry->entry,
2921                  be32_to_cpu(entry->ip_addr));
2922 }
2923
2924 static void _rocker_neigh_del(struct switchdev_trans *trans,
2925                               struct rocker_neigh_tbl_entry *entry)
2926 {
2927         if (switchdev_trans_ph_prepare(trans))
2928                 return;
2929         if (--entry->ref_count == 0) {
2930                 hash_del(&entry->entry);
2931                 rocker_kfree(trans, entry);
2932         }
2933 }
2934
2935 static void _rocker_neigh_update(struct rocker_neigh_tbl_entry *entry,
2936                                  struct switchdev_trans *trans,
2937                                  const u8 *eth_dst, bool ttl_check)
2938 {
2939         if (eth_dst) {
2940                 ether_addr_copy(entry->eth_dst, eth_dst);
2941                 entry->ttl_check = ttl_check;
2942         } else if (!switchdev_trans_ph_prepare(trans)) {
2943                 entry->ref_count++;
2944         }
2945 }
2946
2947 static int rocker_port_ipv4_neigh(struct rocker_port *rocker_port,
2948                                   struct switchdev_trans *trans,
2949                                   int flags, __be32 ip_addr, const u8 *eth_dst)
2950 {
2951         struct rocker *rocker = rocker_port->rocker;
2952         struct rocker_neigh_tbl_entry *entry;
2953         struct rocker_neigh_tbl_entry *found;
2954         unsigned long lock_flags;
2955         __be16 eth_type = htons(ETH_P_IP);
2956         enum rocker_of_dpa_table_id goto_tbl =
2957                 ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
2958         u32 group_id;
2959         u32 priority = 0;
2960         bool adding = !(flags & ROCKER_OP_FLAG_REMOVE);
2961         bool updating;
2962         bool removing;
2963         int err = 0;
2964
2965         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
2966         if (!entry)
2967                 return -ENOMEM;
2968
2969         spin_lock_irqsave(&rocker->neigh_tbl_lock, lock_flags);
2970
2971         found = rocker_neigh_tbl_find(rocker, ip_addr);
2972
2973         updating = found && adding;
2974         removing = found && !adding;
2975         adding = !found && adding;
2976
2977         if (adding) {
2978                 entry->ip_addr = ip_addr;
2979                 entry->dev = rocker_port->dev;
2980                 ether_addr_copy(entry->eth_dst, eth_dst);
2981                 entry->ttl_check = true;
2982                 _rocker_neigh_add(rocker, trans, entry);
2983         } else if (removing) {
2984                 memcpy(entry, found, sizeof(*entry));
2985                 _rocker_neigh_del(trans, found);
2986         } else if (updating) {
2987                 _rocker_neigh_update(found, trans, eth_dst, true);
2988                 memcpy(entry, found, sizeof(*entry));
2989         } else {
2990                 err = -ENOENT;
2991         }
2992
2993         spin_unlock_irqrestore(&rocker->neigh_tbl_lock, lock_flags);
2994
2995         if (err)
2996                 goto err_out;
2997
2998         /* For each active neighbor, we have an L3 unicast group and
2999          * a /32 route to the neighbor, which uses the L3 unicast
3000          * group.  The L3 unicast group can also be referred to by
3001          * other routes' nexthops.
3002          */
3003
3004         err = rocker_group_l3_unicast(rocker_port, trans, flags,
3005                                       entry->index,
3006                                       rocker_port->dev->dev_addr,
3007                                       entry->eth_dst,
3008                                       rocker_port->internal_vlan_id,
3009                                       entry->ttl_check,
3010                                       rocker_port->pport);
3011         if (err) {
3012                 netdev_err(rocker_port->dev,
3013                            "Error (%d) L3 unicast group index %d\n",
3014                            err, entry->index);
3015                 goto err_out;
3016         }
3017
3018         if (adding || removing) {
3019                 group_id = ROCKER_GROUP_L3_UNICAST(entry->index);
3020                 err = rocker_flow_tbl_ucast4_routing(rocker_port, trans,
3021                                                      eth_type, ip_addr,
3022                                                      inet_make_mask(32),
3023                                                      priority, goto_tbl,
3024                                                      group_id, flags);
3025
3026                 if (err)
3027                         netdev_err(rocker_port->dev,
3028                                    "Error (%d) /32 unicast route %pI4 group 0x%08x\n",
3029                                    err, &entry->ip_addr, group_id);
3030         }
3031
3032 err_out:
3033         if (!adding)
3034                 rocker_kfree(trans, entry);
3035
3036         return err;
3037 }
3038
3039 static int rocker_port_ipv4_resolve(struct rocker_port *rocker_port,
3040                                     struct switchdev_trans *trans,
3041                                     __be32 ip_addr)
3042 {
3043         struct net_device *dev = rocker_port->dev;
3044         struct neighbour *n = __ipv4_neigh_lookup(dev, (__force u32)ip_addr);
3045         int err = 0;
3046
3047         if (!n) {
3048                 n = neigh_create(&arp_tbl, &ip_addr, dev);
3049                 if (IS_ERR(n))
3050                         return IS_ERR(n);
3051         }
3052
3053         /* If the neigh is already resolved, then go ahead and
3054          * install the entry, otherwise start the ARP process to
3055          * resolve the neigh.
3056          */
3057
3058         if (n->nud_state & NUD_VALID)
3059                 err = rocker_port_ipv4_neigh(rocker_port, trans, 0,
3060                                              ip_addr, n->ha);
3061         else
3062                 neigh_event_send(n, NULL);
3063
3064         neigh_release(n);
3065         return err;
3066 }
3067
3068 static int rocker_port_ipv4_nh(struct rocker_port *rocker_port,
3069                                struct switchdev_trans *trans, int flags,
3070                                __be32 ip_addr, u32 *index)
3071 {
3072         struct rocker *rocker = rocker_port->rocker;
3073         struct rocker_neigh_tbl_entry *entry;
3074         struct rocker_neigh_tbl_entry *found;
3075         unsigned long lock_flags;
3076         bool adding = !(flags & ROCKER_OP_FLAG_REMOVE);
3077         bool updating;
3078         bool removing;
3079         bool resolved = true;
3080         int err = 0;
3081
3082         entry = rocker_kzalloc(trans, flags, sizeof(*entry));
3083         if (!entry)
3084                 return -ENOMEM;
3085
3086         spin_lock_irqsave(&rocker->neigh_tbl_lock, lock_flags);
3087
3088         found = rocker_neigh_tbl_find(rocker, ip_addr);
3089         if (found)
3090                 *index = found->index;
3091
3092         updating = found && adding;
3093         removing = found && !adding;
3094         adding = !found && adding;
3095
3096         if (adding) {
3097                 entry->ip_addr = ip_addr;
3098                 entry->dev = rocker_port->dev;
3099                 _rocker_neigh_add(rocker, trans, entry);
3100                 *index = entry->index;
3101                 resolved = false;
3102         } else if (removing) {
3103                 _rocker_neigh_del(trans, found);
3104         } else if (updating) {
3105                 _rocker_neigh_update(found, trans, NULL, false);
3106                 resolved = !is_zero_ether_addr(found->eth_dst);
3107         } else {
3108                 err = -ENOENT;
3109         }
3110
3111         spin_unlock_irqrestore(&rocker->neigh_tbl_lock, lock_flags);
3112
3113         if (!adding)
3114                 rocker_kfree(trans, entry);
3115
3116         if (err)
3117                 return err;
3118
3119         /* Resolved means neigh ip_addr is resolved to neigh mac. */
3120
3121         if (!resolved)
3122                 err = rocker_port_ipv4_resolve(rocker_port, trans, ip_addr);
3123
3124         return err;
3125 }
3126
3127 static int rocker_port_vlan_flood_group(struct rocker_port *rocker_port,
3128                                         struct switchdev_trans *trans,
3129                                         int flags, __be16 vlan_id)
3130 {
3131         struct rocker_port *p;
3132         const struct rocker *rocker = rocker_port->rocker;
3133         u32 group_id = ROCKER_GROUP_L2_FLOOD(vlan_id, 0);
3134         u32 *group_ids;
3135         u8 group_count = 0;
3136         int err = 0;
3137         int i;
3138
3139         group_ids = rocker_kcalloc(trans, flags,
3140                                    rocker->port_count, sizeof(u32));
3141         if (!group_ids)
3142                 return -ENOMEM;
3143
3144         /* Adjust the flood group for this VLAN.  The flood group
3145          * references an L2 interface group for each port in this
3146          * VLAN.
3147          */
3148
3149         for (i = 0; i < rocker->port_count; i++) {
3150                 p = rocker->ports[i];
3151                 if (!p)
3152                         continue;
3153                 if (!rocker_port_is_bridged(p))
3154                         continue;
3155                 if (test_bit(ntohs(vlan_id), p->vlan_bitmap)) {
3156                         group_ids[group_count++] =
3157                                 ROCKER_GROUP_L2_INTERFACE(vlan_id, p->pport);
3158                 }
3159         }
3160
3161         /* If there are no bridged ports in this VLAN, we're done */
3162         if (group_count == 0)
3163                 goto no_ports_in_vlan;
3164
3165         err = rocker_group_l2_flood(rocker_port, trans, flags, vlan_id,
3166                                     group_count, group_ids, group_id);
3167         if (err)
3168                 netdev_err(rocker_port->dev,
3169                            "Error (%d) port VLAN l2 flood group\n", err);
3170
3171 no_ports_in_vlan:
3172         rocker_kfree(trans, group_ids);
3173         return err;
3174 }
3175
3176 static int rocker_port_vlan_l2_groups(struct rocker_port *rocker_port,
3177                                       struct switchdev_trans *trans, int flags,
3178                                       __be16 vlan_id, bool pop_vlan)
3179 {
3180         const struct rocker *rocker = rocker_port->rocker;
3181         struct rocker_port *p;
3182         bool adding = !(flags & ROCKER_OP_FLAG_REMOVE);
3183         u32 out_pport;
3184         int ref = 0;
3185         int err;
3186         int i;
3187
3188         /* An L2 interface group for this port in this VLAN, but
3189          * only when port STP state is LEARNING|FORWARDING.
3190          */
3191
3192         if (rocker_port->stp_state == BR_STATE_LEARNING ||
3193             rocker_port->stp_state == BR_STATE_FORWARDING) {
3194                 out_pport = rocker_port->pport;
3195                 err = rocker_group_l2_interface(rocker_port, trans, flags,
3196                                                 vlan_id, out_pport, pop_vlan);
3197                 if (err) {
3198                         netdev_err(rocker_port->dev,
3199                                    "Error (%d) port VLAN l2 group for pport %d\n",
3200                                    err, out_pport);
3201                         return err;
3202                 }
3203         }
3204
3205         /* An L2 interface group for this VLAN to CPU port.
3206          * Add when first port joins this VLAN and destroy when
3207          * last port leaves this VLAN.
3208          */
3209
3210         for (i = 0; i < rocker->port_count; i++) {
3211                 p = rocker->ports[i];
3212                 if (p && test_bit(ntohs(vlan_id), p->vlan_bitmap))
3213                         ref++;
3214         }
3215
3216         if ((!adding || ref != 1) && (adding || ref != 0))
3217                 return 0;
3218
3219         out_pport = 0;
3220         err = rocker_group_l2_interface(rocker_port, trans, flags,
3221                                         vlan_id, out_pport, pop_vlan);
3222         if (err) {
3223                 netdev_err(rocker_port->dev,
3224                            "Error (%d) port VLAN l2 group for CPU port\n", err);
3225                 return err;
3226         }
3227
3228         return 0;
3229 }
3230
3231 static struct rocker_ctrl {
3232         const u8 *eth_dst;
3233         const u8 *eth_dst_mask;
3234         __be16 eth_type;
3235         bool acl;
3236         bool bridge;
3237         bool term;
3238         bool copy_to_cpu;
3239 } rocker_ctrls[] = {
3240         [ROCKER_CTRL_LINK_LOCAL_MCAST] = {
3241                 /* pass link local multicast pkts up to CPU for filtering */
3242                 .eth_dst = ll_mac,
3243                 .eth_dst_mask = ll_mask,
3244                 .acl = true,
3245         },
3246         [ROCKER_CTRL_LOCAL_ARP] = {
3247                 /* pass local ARP pkts up to CPU */
3248                 .eth_dst = zero_mac,
3249                 .eth_dst_mask = zero_mac,
3250                 .eth_type = htons(ETH_P_ARP),
3251                 .acl = true,
3252         },
3253         [ROCKER_CTRL_IPV4_MCAST] = {
3254                 /* pass IPv4 mcast pkts up to CPU, RFC 1112 */
3255                 .eth_dst = ipv4_mcast,
3256                 .eth_dst_mask = ipv4_mask,
3257                 .eth_type = htons(ETH_P_IP),
3258                 .term  = true,
3259                 .copy_to_cpu = true,
3260         },
3261         [ROCKER_CTRL_IPV6_MCAST] = {
3262                 /* pass IPv6 mcast pkts up to CPU, RFC 2464 */
3263                 .eth_dst = ipv6_mcast,
3264                 .eth_dst_mask = ipv6_mask,
3265                 .eth_type = htons(ETH_P_IPV6),
3266                 .term  = true,
3267                 .copy_to_cpu = true,
3268         },
3269         [ROCKER_CTRL_DFLT_BRIDGING] = {
3270                 /* flood any pkts on vlan */
3271                 .bridge = true,
3272                 .copy_to_cpu = true,
3273         },
3274         [ROCKER_CTRL_DFLT_OVS] = {
3275                 /* pass all pkts up to CPU */
3276                 .eth_dst = zero_mac,
3277                 .eth_dst_mask = zero_mac,
3278                 .acl = true,
3279         },
3280 };
3281
3282 static int rocker_port_ctrl_vlan_acl(struct rocker_port *rocker_port,
3283                                      struct switchdev_trans *trans, int flags,
3284                                      const struct rocker_ctrl *ctrl, __be16 vlan_id)
3285 {
3286         u32 in_pport = rocker_port->pport;
3287         u32 in_pport_mask = 0xffffffff;
3288         u32 out_pport = 0;
3289         const u8 *eth_src = NULL;
3290         const u8 *eth_src_mask = NULL;
3291         __be16 vlan_id_mask = htons(0xffff);
3292         u8 ip_proto = 0;
3293         u8 ip_proto_mask = 0;
3294         u8 ip_tos = 0;
3295         u8 ip_tos_mask = 0;
3296         u32 group_id = ROCKER_GROUP_L2_INTERFACE(vlan_id, out_pport);
3297         int err;
3298
3299         err = rocker_flow_tbl_acl(rocker_port, trans, flags,
3300                                   in_pport, in_pport_mask,
3301                                   eth_src, eth_src_mask,
3302                                   ctrl->eth_dst, ctrl->eth_dst_mask,
3303                                   ctrl->eth_type,
3304                                   vlan_id, vlan_id_mask,
3305                                   ip_proto, ip_proto_mask,
3306                                   ip_tos, ip_tos_mask,
3307                                   group_id);
3308
3309         if (err)
3310                 netdev_err(rocker_port->dev, "Error (%d) ctrl ACL\n", err);
3311
3312         return err;
3313 }
3314
3315 static int rocker_port_ctrl_vlan_bridge(struct rocker_port *rocker_port,
3316                                         struct switchdev_trans *trans,
3317                                         int flags,
3318                                         const struct rocker_ctrl *ctrl,
3319                                         __be16 vlan_id)
3320 {
3321         enum rocker_of_dpa_table_id goto_tbl =
3322                 ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
3323         u32 group_id = ROCKER_GROUP_L2_FLOOD(vlan_id, 0);
3324         u32 tunnel_id = 0;
3325         int err;
3326
3327         if (!rocker_port_is_bridged(rocker_port))
3328                 return 0;
3329
3330         err = rocker_flow_tbl_bridge(rocker_port, trans, flags,
3331                                      ctrl->eth_dst, ctrl->eth_dst_mask,
3332                                      vlan_id, tunnel_id,
3333                                      goto_tbl, group_id, ctrl->copy_to_cpu);
3334
3335         if (err)
3336                 netdev_err(rocker_port->dev, "Error (%d) ctrl FLOOD\n", err);
3337
3338         return err;
3339 }
3340
3341 static int rocker_port_ctrl_vlan_term(struct rocker_port *rocker_port,
3342                                       struct switchdev_trans *trans, int flags,
3343                                       const struct rocker_ctrl *ctrl, __be16 vlan_id)
3344 {
3345         u32 in_pport_mask = 0xffffffff;
3346         __be16 vlan_id_mask = htons(0xffff);
3347         int err;
3348
3349         if (ntohs(vlan_id) == 0)
3350                 vlan_id = rocker_port->internal_vlan_id;
3351
3352         err = rocker_flow_tbl_term_mac(rocker_port, trans,
3353                                        rocker_port->pport, in_pport_mask,
3354                                        ctrl->eth_type, ctrl->eth_dst,
3355                                        ctrl->eth_dst_mask, vlan_id,
3356                                        vlan_id_mask, ctrl->copy_to_cpu,
3357                                        flags);
3358
3359         if (err)
3360                 netdev_err(rocker_port->dev, "Error (%d) ctrl term\n", err);
3361
3362         return err;
3363 }
3364
3365 static int rocker_port_ctrl_vlan(struct rocker_port *rocker_port,
3366                                  struct switchdev_trans *trans, int flags,
3367                                  const struct rocker_ctrl *ctrl, __be16 vlan_id)
3368 {
3369         if (ctrl->acl)
3370                 return rocker_port_ctrl_vlan_acl(rocker_port, trans, flags,
3371                                                  ctrl, vlan_id);
3372         if (ctrl->bridge)
3373                 return rocker_port_ctrl_vlan_bridge(rocker_port, trans, flags,
3374                                                     ctrl, vlan_id);
3375
3376         if (ctrl->term)
3377                 return rocker_port_ctrl_vlan_term(rocker_port, trans, flags,
3378                                                   ctrl, vlan_id);
3379
3380         return -EOPNOTSUPP;
3381 }
3382
3383 static int rocker_port_ctrl_vlan_add(struct rocker_port *rocker_port,
3384                                      struct switchdev_trans *trans, int flags,
3385                                      __be16 vlan_id)
3386 {
3387         int err = 0;
3388         int i;
3389
3390         for (i = 0; i < ROCKER_CTRL_MAX; i++) {
3391                 if (rocker_port->ctrls[i]) {
3392                         err = rocker_port_ctrl_vlan(rocker_port, trans, flags,
3393                                                     &rocker_ctrls[i], vlan_id);
3394                         if (err)
3395                                 return err;
3396                 }
3397         }
3398
3399         return err;
3400 }
3401
3402 static int rocker_port_ctrl(struct rocker_port *rocker_port,
3403                             struct switchdev_trans *trans, int flags,
3404                             const struct rocker_ctrl *ctrl)
3405 {
3406         u16 vid;
3407         int err = 0;
3408
3409         for (vid = 1; vid < VLAN_N_VID; vid++) {
3410                 if (!test_bit(vid, rocker_port->vlan_bitmap))
3411                         continue;
3412                 err = rocker_port_ctrl_vlan(rocker_port, trans, flags,
3413                                             ctrl, htons(vid));
3414                 if (err)
3415                         break;
3416         }
3417
3418         return err;
3419 }
3420
3421 static int rocker_port_vlan(struct rocker_port *rocker_port,
3422                             struct switchdev_trans *trans, int flags, u16 vid)
3423 {
3424         enum rocker_of_dpa_table_id goto_tbl =
3425                 ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC;
3426         u32 in_pport = rocker_port->pport;
3427         __be16 vlan_id = htons(vid);
3428         __be16 vlan_id_mask = htons(0xffff);
3429         __be16 internal_vlan_id;
3430         bool untagged;
3431         bool adding = !(flags & ROCKER_OP_FLAG_REMOVE);
3432         int err;
3433
3434         internal_vlan_id = rocker_port_vid_to_vlan(rocker_port, vid, &untagged);
3435
3436         if (adding && test_bit(ntohs(internal_vlan_id),
3437                                rocker_port->vlan_bitmap))
3438                         return 0; /* already added */
3439         else if (!adding && !test_bit(ntohs(internal_vlan_id),
3440                                       rocker_port->vlan_bitmap))
3441                         return 0; /* already removed */
3442
3443         change_bit(ntohs(internal_vlan_id), rocker_port->vlan_bitmap);
3444
3445         if (adding) {
3446                 err = rocker_port_ctrl_vlan_add(rocker_port, trans, flags,
3447                                                 internal_vlan_id);
3448                 if (err) {
3449                         netdev_err(rocker_port->dev,
3450                                    "Error (%d) port ctrl vlan add\n", err);
3451                         goto err_out;
3452                 }
3453         }
3454
3455         err = rocker_port_vlan_l2_groups(rocker_port, trans, flags,
3456                                          internal_vlan_id, untagged);
3457         if (err) {
3458                 netdev_err(rocker_port->dev,
3459                            "Error (%d) port VLAN l2 groups\n", err);
3460                 goto err_out;
3461         }
3462
3463         err = rocker_port_vlan_flood_group(rocker_port, trans, flags,
3464                                            internal_vlan_id);
3465         if (err) {
3466                 netdev_err(rocker_port->dev,
3467                            "Error (%d) port VLAN l2 flood group\n", err);
3468                 goto err_out;
3469         }
3470
3471         err = rocker_flow_tbl_vlan(rocker_port, trans, flags,
3472                                    in_pport, vlan_id, vlan_id_mask,
3473                                    goto_tbl, untagged, internal_vlan_id);
3474         if (err)
3475                 netdev_err(rocker_port->dev,
3476                            "Error (%d) port VLAN table\n", err);
3477
3478 err_out:
3479         if (switchdev_trans_ph_prepare(trans))
3480                 change_bit(ntohs(internal_vlan_id), rocker_port->vlan_bitmap);
3481
3482         return err;
3483 }
3484
3485 static int rocker_port_ig_tbl(struct rocker_port *rocker_port,
3486                               struct switchdev_trans *trans, int flags)
3487 {
3488         enum rocker_of_dpa_table_id goto_tbl;
3489         u32 in_pport;
3490         u32 in_pport_mask;
3491         int err;
3492
3493         /* Normal Ethernet Frames.  Matches pkts from any local physical
3494          * ports.  Goto VLAN tbl.
3495          */
3496
3497         in_pport = 0;
3498         in_pport_mask = 0xffff0000;
3499         goto_tbl = ROCKER_OF_DPA_TABLE_ID_VLAN;
3500
3501         err = rocker_flow_tbl_ig_port(rocker_port, trans, flags,
3502                                       in_pport, in_pport_mask,
3503                                       goto_tbl);
3504         if (err)
3505                 netdev_err(rocker_port->dev,
3506                            "Error (%d) ingress port table entry\n", err);
3507
3508         return err;
3509 }
3510
3511 struct rocker_fdb_learn_work {
3512         struct work_struct work;
3513         struct rocker_port *rocker_port;
3514         struct switchdev_trans *trans;
3515         int flags;
3516         u8 addr[ETH_ALEN];
3517         u16 vid;
3518 };
3519
3520 static void rocker_port_fdb_learn_work(struct work_struct *work)
3521 {
3522         const struct rocker_fdb_learn_work *lw =
3523                 container_of(work, struct rocker_fdb_learn_work, work);
3524         bool removing = (lw->flags & ROCKER_OP_FLAG_REMOVE);
3525         bool learned = (lw->flags & ROCKER_OP_FLAG_LEARNED);
3526         struct switchdev_notifier_fdb_info info;
3527
3528         info.addr = lw->addr;
3529         info.vid = lw->vid;
3530
3531         rtnl_lock();
3532         if (learned && removing)
3533                 call_switchdev_notifiers(SWITCHDEV_FDB_DEL,
3534                                          lw->rocker_port->dev, &info.info);
3535         else if (learned && !removing)
3536                 call_switchdev_notifiers(SWITCHDEV_FDB_ADD,
3537                                          lw->rocker_port->dev, &info.info);
3538         rtnl_unlock();
3539
3540         rocker_kfree(lw->trans, work);
3541 }
3542
3543 static int rocker_port_fdb_learn(struct rocker_port *rocker_port,
3544                                  struct switchdev_trans *trans, int flags,
3545                                  const u8 *addr, __be16 vlan_id)
3546 {
3547         struct rocker_fdb_learn_work *lw;
3548         enum rocker_of_dpa_table_id goto_tbl =
3549                 ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
3550         u32 out_pport = rocker_port->pport;
3551         u32 tunnel_id = 0;
3552         u32 group_id = ROCKER_GROUP_NONE;
3553         bool syncing = !!(rocker_port->brport_flags & BR_LEARNING_SYNC);
3554         bool copy_to_cpu = false;
3555         int err;
3556
3557         if (rocker_port_is_bridged(rocker_port))
3558                 group_id = ROCKER_GROUP_L2_INTERFACE(vlan_id, out_pport);
3559
3560         if (!(flags & ROCKER_OP_FLAG_REFRESH)) {
3561                 err = rocker_flow_tbl_bridge(rocker_port, trans, flags, addr,
3562                                              NULL, vlan_id, tunnel_id, goto_tbl,
3563                                              group_id, copy_to_cpu);
3564                 if (err)
3565                         return err;
3566         }
3567
3568         if (!syncing)
3569                 return 0;
3570
3571         if (!rocker_port_is_bridged(rocker_port))
3572                 return 0;
3573
3574         lw = rocker_kzalloc(trans, flags, sizeof(*lw));
3575         if (!lw)
3576                 return -ENOMEM;
3577
3578         INIT_WORK(&lw->work, rocker_port_fdb_learn_work);
3579
3580         lw->rocker_port = rocker_port;
3581         lw->trans = trans;
3582         lw->flags = flags;
3583         ether_addr_copy(lw->addr, addr);
3584         lw->vid = rocker_port_vlan_to_vid(rocker_port, vlan_id);
3585
3586         if (switchdev_trans_ph_prepare(trans))
3587                 rocker_kfree(trans, lw);
3588         else
3589                 schedule_work(&lw->work);
3590
3591         return 0;
3592 }
3593
3594 static struct rocker_fdb_tbl_entry *
3595 rocker_fdb_tbl_find(const struct rocker *rocker,
3596                     const struct rocker_fdb_tbl_entry *match)
3597 {
3598         struct rocker_fdb_tbl_entry *found;
3599
3600         hash_for_each_possible(rocker->fdb_tbl, found, entry, match->key_crc32)
3601                 if (memcmp(&found->key, &match->key, sizeof(found->key)) == 0)
3602                         return found;
3603
3604         return NULL;
3605 }
3606
3607 static int rocker_port_fdb(struct rocker_port *rocker_port,
3608                            struct switchdev_trans *trans,
3609                            const unsigned char *addr,
3610                            __be16 vlan_id, int flags)
3611 {
3612         struct rocker *rocker = rocker_port->rocker;
3613         struct rocker_fdb_tbl_entry *fdb;
3614         struct rocker_fdb_tbl_entry *found;
3615         bool removing = (flags & ROCKER_OP_FLAG_REMOVE);
3616         unsigned long lock_flags;
3617
3618         fdb = rocker_kzalloc(trans, flags, sizeof(*fdb));
3619         if (!fdb)
3620                 return -ENOMEM;
3621
3622         fdb->learned = (flags & ROCKER_OP_FLAG_LEARNED);
3623         fdb->touched = jiffies;
3624         fdb->key.rocker_port = rocker_port;
3625         ether_addr_copy(fdb->key.addr, addr);
3626         fdb->key.vlan_id = vlan_id;
3627         fdb->key_crc32 = crc32(~0, &fdb->key, sizeof(fdb->key));
3628
3629         spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
3630
3631         found = rocker_fdb_tbl_find(rocker, fdb);
3632
3633         if (found) {
3634                 found->touched = jiffies;
3635                 if (removing) {
3636                         rocker_kfree(trans, fdb);
3637                         if (!switchdev_trans_ph_prepare(trans))
3638                                 hash_del(&found->entry);
3639                 }
3640         } else if (!removing) {
3641                 if (!switchdev_trans_ph_prepare(trans))
3642                         hash_add(rocker->fdb_tbl, &fdb->entry,
3643                                  fdb->key_crc32);
3644         }
3645
3646         spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
3647
3648         /* Check if adding and already exists, or removing and can't find */
3649         if (!found != !removing) {
3650                 rocker_kfree(trans, fdb);
3651                 if (!found && removing)
3652                         return 0;
3653                 /* Refreshing existing to update aging timers */
3654                 flags |= ROCKER_OP_FLAG_REFRESH;
3655         }
3656
3657         return rocker_port_fdb_learn(rocker_port, trans, flags, addr, vlan_id);
3658 }
3659
3660 static int rocker_port_fdb_flush(struct rocker_port *rocker_port,
3661                                  struct switchdev_trans *trans, int flags)
3662 {
3663         struct rocker *rocker = rocker_port->rocker;
3664         struct rocker_fdb_tbl_entry *found;
3665         unsigned long lock_flags;
3666         struct hlist_node *tmp;
3667         int bkt;
3668         int err = 0;
3669
3670         if (rocker_port->stp_state == BR_STATE_LEARNING ||
3671             rocker_port->stp_state == BR_STATE_FORWARDING)
3672                 return 0;
3673
3674         flags |= ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_REMOVE;
3675
3676         spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
3677
3678         hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, found, entry) {
3679                 if (found->key.rocker_port != rocker_port)
3680                         continue;
3681                 if (!found->learned)
3682                         continue;
3683                 err = rocker_port_fdb_learn(rocker_port, trans, flags,
3684                                             found->key.addr,
3685                                             found->key.vlan_id);
3686                 if (err)
3687                         goto err_out;
3688                 if (!switchdev_trans_ph_prepare(trans))
3689                         hash_del(&found->entry);
3690         }
3691
3692 err_out:
3693         spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
3694
3695         return err;
3696 }
3697
3698 static void rocker_fdb_cleanup(unsigned long data)
3699 {
3700         struct rocker *rocker = (struct rocker *)data;
3701         struct rocker_port *rocker_port;
3702         struct rocker_fdb_tbl_entry *entry;
3703         struct hlist_node *tmp;
3704         unsigned long next_timer = jiffies + BR_MIN_AGEING_TIME;
3705         unsigned long expires;
3706         unsigned long lock_flags;
3707         int flags = ROCKER_OP_FLAG_NOWAIT | ROCKER_OP_FLAG_REMOVE |
3708                     ROCKER_OP_FLAG_LEARNED;
3709         int bkt;
3710
3711         spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
3712
3713         hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, entry, entry) {
3714                 if (!entry->learned)
3715                         continue;
3716                 rocker_port = entry->key.rocker_port;
3717                 expires = entry->touched + rocker_port->ageing_time;
3718                 if (time_before_eq(expires, jiffies)) {
3719                         rocker_port_fdb_learn(rocker_port, NULL,
3720                                               flags, entry->key.addr,
3721                                               entry->key.vlan_id);
3722                         hash_del(&entry->entry);
3723                 } else if (time_before(expires, next_timer)) {
3724                         next_timer = expires;
3725                 }
3726         }
3727
3728         spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
3729
3730         mod_timer(&rocker->fdb_cleanup_timer, round_jiffies_up(next_timer));
3731 }
3732
3733 static int rocker_port_router_mac(struct rocker_port *rocker_port,
3734                                   struct switchdev_trans *trans, int flags,
3735                                   __be16 vlan_id)
3736 {
3737         u32 in_pport_mask = 0xffffffff;
3738         __be16 eth_type;
3739         const u8 *dst_mac_mask = ff_mac;
3740         __be16 vlan_id_mask = htons(0xffff);
3741         bool copy_to_cpu = false;
3742         int err;
3743
3744         if (ntohs(vlan_id) == 0)
3745                 vlan_id = rocker_port->internal_vlan_id;
3746
3747         eth_type = htons(ETH_P_IP);
3748         err = rocker_flow_tbl_term_mac(rocker_port, trans,
3749                                        rocker_port->pport, in_pport_mask,
3750                                        eth_type, rocker_port->dev->dev_addr,
3751                                        dst_mac_mask, vlan_id, vlan_id_mask,
3752                                        copy_to_cpu, flags);
3753         if (err)
3754                 return err;
3755
3756         eth_type = htons(ETH_P_IPV6);
3757         err = rocker_flow_tbl_term_mac(rocker_port, trans,
3758                                        rocker_port->pport, in_pport_mask,
3759                                        eth_type, rocker_port->dev->dev_addr,
3760                                        dst_mac_mask, vlan_id, vlan_id_mask,
3761                                        copy_to_cpu, flags);
3762
3763         return err;
3764 }
3765
3766 static int rocker_port_fwding(struct rocker_port *rocker_port,
3767                               struct switchdev_trans *trans, int flags)
3768 {
3769         bool pop_vlan;
3770         u32 out_pport;
3771         __be16 vlan_id;
3772         u16 vid;
3773         int err;
3774
3775         /* Port will be forwarding-enabled if its STP state is LEARNING
3776          * or FORWARDING.  Traffic from CPU can still egress, regardless of
3777          * port STP state.  Use L2 interface group on port VLANs as a way
3778          * to toggle port forwarding: if forwarding is disabled, L2
3779          * interface group will not exist.
3780          */
3781
3782         if (rocker_port->stp_state != BR_STATE_LEARNING &&
3783             rocker_port->stp_state != BR_STATE_FORWARDING)
3784                 flags |= ROCKER_OP_FLAG_REMOVE;
3785
3786         out_pport = rocker_port->pport;
3787         for (vid = 1; vid < VLAN_N_VID; vid++) {
3788                 if (!test_bit(vid, rocker_port->vlan_bitmap))
3789                         continue;
3790                 vlan_id = htons(vid);
3791                 pop_vlan = rocker_vlan_id_is_internal(vlan_id);
3792                 err = rocker_group_l2_interface(rocker_port, trans, flags,
3793                                                 vlan_id, out_pport, pop_vlan);
3794                 if (err) {
3795                         netdev_err(rocker_port->dev,
3796                                    "Error (%d) port VLAN l2 group for pport %d\n",
3797                                    err, out_pport);
3798                         return err;
3799                 }
3800         }
3801
3802         return 0;
3803 }
3804
3805 static int rocker_port_stp_update(struct rocker_port *rocker_port,
3806                                   struct switchdev_trans *trans, int flags,
3807                                   u8 state)
3808 {
3809         bool want[ROCKER_CTRL_MAX] = { 0, };
3810         bool prev_ctrls[ROCKER_CTRL_MAX];
3811         u8 uninitialized_var(prev_state);
3812         int err;
3813         int i;
3814
3815         if (switchdev_trans_ph_prepare(trans)) {
3816                 memcpy(prev_ctrls, rocker_port->ctrls, sizeof(prev_ctrls));
3817                 prev_state = rocker_port->stp_state;
3818         }
3819
3820         if (rocker_port->stp_state == state)
3821                 return 0;
3822
3823         rocker_port->stp_state = state;
3824
3825         switch (state) {
3826         case BR_STATE_DISABLED:
3827                 /* port is completely disabled */
3828                 break;
3829         case BR_STATE_LISTENING:
3830         case BR_STATE_BLOCKING:
3831                 want[ROCKER_CTRL_LINK_LOCAL_MCAST] = true;
3832                 break;
3833         case BR_STATE_LEARNING:
3834         case BR_STATE_FORWARDING:
3835                 if (!rocker_port_is_ovsed(rocker_port))
3836                         want[ROCKER_CTRL_LINK_LOCAL_MCAST] = true;
3837                 want[ROCKER_CTRL_IPV4_MCAST] = true;
3838                 want[ROCKER_CTRL_IPV6_MCAST] = true;
3839                 if (rocker_port_is_bridged(rocker_port))
3840                         want[ROCKER_CTRL_DFLT_BRIDGING] = true;
3841                 else if (rocker_port_is_ovsed(rocker_port))
3842                         want[ROCKER_CTRL_DFLT_OVS] = true;
3843                 else
3844                         want[ROCKER_CTRL_LOCAL_ARP] = true;
3845                 break;
3846         }
3847
3848         for (i = 0; i < ROCKER_CTRL_MAX; i++) {
3849                 if (want[i] != rocker_port->ctrls[i]) {
3850                         int ctrl_flags = flags |
3851                                          (want[i] ? 0 : ROCKER_OP_FLAG_REMOVE);
3852                         err = rocker_port_ctrl(rocker_port, trans, ctrl_flags,
3853                                                &rocker_ctrls[i]);
3854                         if (err)
3855                                 goto err_out;
3856                         rocker_port->ctrls[i] = want[i];
3857                 }
3858         }
3859
3860         err = rocker_port_fdb_flush(rocker_port, trans, flags);
3861         if (err)
3862                 goto err_out;
3863
3864         err = rocker_port_fwding(rocker_port, trans, flags);
3865
3866 err_out:
3867         if (switchdev_trans_ph_prepare(trans)) {
3868                 memcpy(rocker_port->ctrls, prev_ctrls, sizeof(prev_ctrls));
3869                 rocker_port->stp_state = prev_state;
3870         }
3871
3872         return err;
3873 }
3874
3875 static int rocker_port_fwd_enable(struct rocker_port *rocker_port,
3876                                   struct switchdev_trans *trans, int flags)
3877 {
3878         if (rocker_port_is_bridged(rocker_port))
3879                 /* bridge STP will enable port */
3880                 return 0;
3881
3882         /* port is not bridged, so simulate going to FORWARDING state */
3883         return rocker_port_stp_update(rocker_port, trans, flags,
3884                                       BR_STATE_FORWARDING);
3885 }
3886
3887 static int rocker_port_fwd_disable(struct rocker_port *rocker_port,
3888                                    struct switchdev_trans *trans, int flags)
3889 {
3890         if (rocker_port_is_bridged(rocker_port))
3891                 /* bridge STP will disable port */
3892                 return 0;
3893
3894         /* port is not bridged, so simulate going to DISABLED state */
3895         return rocker_port_stp_update(rocker_port, trans, flags,
3896                                       BR_STATE_DISABLED);
3897 }
3898
3899 static struct rocker_internal_vlan_tbl_entry *
3900 rocker_internal_vlan_tbl_find(const struct rocker *rocker, int ifindex)
3901 {
3902         struct rocker_internal_vlan_tbl_entry *found;
3903
3904         hash_for_each_possible(rocker->internal_vlan_tbl, found,
3905                                entry, ifindex) {
3906                 if (found->ifindex == ifindex)
3907                         return found;
3908         }
3909
3910         return NULL;
3911 }
3912
3913 static __be16 rocker_port_internal_vlan_id_get(struct rocker_port *rocker_port,
3914                                                int ifindex)
3915 {
3916         struct rocker *rocker = rocker_port->rocker;
3917         struct rocker_internal_vlan_tbl_entry *entry;
3918         struct rocker_internal_vlan_tbl_entry *found;
3919         unsigned long lock_flags;
3920         int i;
3921
3922         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3923         if (!entry)
3924                 return 0;
3925
3926         entry->ifindex = ifindex;
3927
3928         spin_lock_irqsave(&rocker->internal_vlan_tbl_lock, lock_flags);
3929
3930         found = rocker_internal_vlan_tbl_find(rocker, ifindex);
3931         if (found) {
3932                 kfree(entry);
3933                 goto found;
3934         }
3935
3936         found = entry;
3937         hash_add(rocker->internal_vlan_tbl, &found->entry, found->ifindex);
3938
3939         for (i = 0; i < ROCKER_N_INTERNAL_VLANS; i++) {
3940                 if (test_and_set_bit(i, rocker->internal_vlan_bitmap))
3941                         continue;
3942                 found->vlan_id = htons(ROCKER_INTERNAL_VLAN_ID_BASE + i);
3943                 goto found;
3944         }
3945
3946         netdev_err(rocker_port->dev, "Out of internal VLAN IDs\n");
3947
3948 found:
3949         found->ref_count++;
3950         spin_unlock_irqrestore(&rocker->internal_vlan_tbl_lock, lock_flags);
3951
3952         return found->vlan_id;
3953 }
3954
3955 static void
3956 rocker_port_internal_vlan_id_put(const struct rocker_port *rocker_port,
3957                                  int ifindex)
3958 {
3959         struct rocker *rocker = rocker_port->rocker;
3960         struct rocker_internal_vlan_tbl_entry *found;
3961         unsigned long lock_flags;
3962         unsigned long bit;
3963
3964         spin_lock_irqsave(&rocker->internal_vlan_tbl_lock, lock_flags);
3965
3966         found = rocker_internal_vlan_tbl_find(rocker, ifindex);
3967         if (!found) {
3968                 netdev_err(rocker_port->dev,
3969                            "ifindex (%d) not found in internal VLAN tbl\n",
3970                            ifindex);
3971                 goto not_found;
3972         }
3973
3974         if (--found->ref_count <= 0) {
3975                 bit = ntohs(found->vlan_id) - ROCKER_INTERNAL_VLAN_ID_BASE;
3976                 clear_bit(bit, rocker->internal_vlan_bitmap);
3977                 hash_del(&found->entry);
3978                 kfree(found);
3979         }
3980
3981 not_found:
3982         spin_unlock_irqrestore(&rocker->internal_vlan_tbl_lock, lock_flags);
3983 }
3984
3985 static int rocker_port_fib_ipv4(struct rocker_port *rocker_port,
3986                                 struct switchdev_trans *trans, __be32 dst,
3987                                 int dst_len, const struct fib_info *fi,
3988                                 u32 tb_id, int flags)
3989 {
3990         const struct fib_nh *nh;
3991         __be16 eth_type = htons(ETH_P_IP);
3992         __be32 dst_mask = inet_make_mask(dst_len);
3993         __be16 internal_vlan_id = rocker_port->internal_vlan_id;
3994         u32 priority = fi->fib_priority;
3995         enum rocker_of_dpa_table_id goto_tbl =
3996                 ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
3997         u32 group_id;
3998         bool nh_on_port;
3999         bool has_gw;
4000         u32 index;
4001         int err;
4002
4003         /* XXX support ECMP */
4004
4005         nh = fi->fib_nh;
4006         nh_on_port = (fi->fib_dev == rocker_port->dev);
4007         has_gw = !!nh->nh_gw;
4008
4009         if (has_gw && nh_on_port) {
4010                 err = rocker_port_ipv4_nh(rocker_port, trans, flags,
4011                                           nh->nh_gw, &index);
4012                 if (err)
4013                         return err;
4014
4015                 group_id = ROCKER_GROUP_L3_UNICAST(index);
4016         } else {
4017                 /* Send to CPU for processing */
4018                 group_id = ROCKER_GROUP_L2_INTERFACE(internal_vlan_id, 0);
4019         }
4020
4021         err = rocker_flow_tbl_ucast4_routing(rocker_port, trans, eth_type, dst,
4022                                              dst_mask, priority, goto_tbl,
4023                                              group_id, flags);
4024         if (err)
4025                 netdev_err(rocker_port->dev, "Error (%d) IPv4 route %pI4\n",
4026                            err, &dst);
4027
4028         return err;
4029 }
4030
4031 /*****************
4032  * Net device ops
4033  *****************/
4034
4035 static int rocker_port_open(struct net_device *dev)
4036 {
4037         struct rocker_port *rocker_port = netdev_priv(dev);
4038         int err;
4039
4040         err = rocker_port_dma_rings_init(rocker_port);
4041         if (err)
4042                 return err;
4043
4044         err = request_irq(rocker_msix_tx_vector(rocker_port),
4045                           rocker_tx_irq_handler, 0,
4046                           rocker_driver_name, rocker_port);
4047         if (err) {
4048                 netdev_err(rocker_port->dev, "cannot assign tx irq\n");
4049                 goto err_request_tx_irq;
4050         }
4051
4052         err = request_irq(rocker_msix_rx_vector(rocker_port),
4053                           rocker_rx_irq_handler, 0,
4054                           rocker_driver_name, rocker_port);
4055         if (err) {
4056                 netdev_err(rocker_port->dev, "cannot assign rx irq\n");
4057                 goto err_request_rx_irq;
4058         }
4059
4060         err = rocker_port_fwd_enable(rocker_port, NULL, 0);
4061         if (err)
4062                 goto err_fwd_enable;
4063
4064         napi_enable(&rocker_port->napi_tx);
4065         napi_enable(&rocker_port->napi_rx);
4066         if (!dev->proto_down)
4067                 rocker_port_set_enable(rocker_port, true);
4068         netif_start_queue(dev);
4069         return 0;
4070
4071 err_fwd_enable:
4072         free_irq(rocker_msix_rx_vector(rocker_port), rocker_port);
4073 err_request_rx_irq:
4074         free_irq(rocker_msix_tx_vector(rocker_port), rocker_port);
4075 err_request_tx_irq:
4076         rocker_port_dma_rings_fini(rocker_port);
4077         return err;
4078 }
4079
4080 static int rocker_port_stop(struct net_device *dev)
4081 {
4082         struct rocker_port *rocker_port = netdev_priv(dev);
4083
4084         netif_stop_queue(dev);
4085         rocker_port_set_enable(rocker_port, false);
4086         napi_disable(&rocker_port->napi_rx);
4087         napi_disable(&rocker_port->napi_tx);
4088         rocker_port_fwd_disable(rocker_port, NULL,
4089                                 ROCKER_OP_FLAG_NOWAIT);
4090         free_irq(rocker_msix_rx_vector(rocker_port), rocker_port);
4091         free_irq(rocker_msix_tx_vector(rocker_port), rocker_port);
4092         rocker_port_dma_rings_fini(rocker_port);
4093
4094         return 0;
4095 }
4096
4097 static void rocker_tx_desc_frags_unmap(const struct rocker_port *rocker_port,
4098                                        const struct rocker_desc_info *desc_info)
4099 {
4100         const struct rocker *rocker = rocker_port->rocker;
4101         struct pci_dev *pdev = rocker->pdev;
4102         const struct rocker_tlv *attrs[ROCKER_TLV_TX_MAX + 1];
4103         struct rocker_tlv *attr;
4104         int rem;
4105
4106         rocker_tlv_parse_desc(attrs, ROCKER_TLV_TX_MAX, desc_info);
4107         if (!attrs[ROCKER_TLV_TX_FRAGS])
4108                 return;
4109         rocker_tlv_for_each_nested(attr, attrs[ROCKER_TLV_TX_FRAGS], rem) {
4110                 const struct rocker_tlv *frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_MAX + 1];
4111                 dma_addr_t dma_handle;
4112                 size_t len;
4113
4114                 if (rocker_tlv_type(attr) != ROCKER_TLV_TX_FRAG)
4115                         continue;
4116                 rocker_tlv_parse_nested(frag_attrs, ROCKER_TLV_TX_FRAG_ATTR_MAX,
4117                                         attr);
4118                 if (!frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_ADDR] ||
4119                     !frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_LEN])
4120                         continue;
4121                 dma_handle = rocker_tlv_get_u64(frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_ADDR]);
4122                 len = rocker_tlv_get_u16(frag_attrs[ROCKER_TLV_TX_FRAG_ATTR_LEN]);
4123                 pci_unmap_single(pdev, dma_handle, len, DMA_TO_DEVICE);
4124         }
4125 }
4126
4127 static int rocker_tx_desc_frag_map_put(const struct rocker_port *rocker_port,
4128                                        struct rocker_desc_info *desc_info,
4129                                        char *buf, size_t buf_len)
4130 {
4131         const struct rocker *rocker = rocker_port->rocker;
4132         struct pci_dev *pdev = rocker->pdev;
4133         dma_addr_t dma_handle;
4134         struct rocker_tlv *frag;
4135
4136         dma_handle = pci_map_single(pdev, buf, buf_len, DMA_TO_DEVICE);
4137         if (unlikely(pci_dma_mapping_error(pdev, dma_handle))) {
4138                 if (net_ratelimit())
4139                         netdev_err(rocker_port->dev, "failed to dma map tx frag\n");
4140                 return -EIO;
4141         }
4142         frag = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAG);
4143         if (!frag)
4144                 goto unmap_frag;
4145         if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_TX_FRAG_ATTR_ADDR,
4146                                dma_handle))
4147                 goto nest_cancel;
4148         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_TX_FRAG_ATTR_LEN,
4149                                buf_len))
4150                 goto nest_cancel;
4151         rocker_tlv_nest_end(desc_info, frag);
4152         return 0;
4153
4154 nest_cancel:
4155         rocker_tlv_nest_cancel(desc_info, frag);
4156 unmap_frag:
4157         pci_unmap_single(pdev, dma_handle, buf_len, DMA_TO_DEVICE);
4158         return -EMSGSIZE;
4159 }
4160
4161 static netdev_tx_t rocker_port_xmit(struct sk_buff *skb, struct net_device *dev)
4162 {
4163         struct rocker_port *rocker_port = netdev_priv(dev);
4164         struct rocker *rocker = rocker_port->rocker;
4165         struct rocker_desc_info *desc_info;
4166         struct rocker_tlv *frags;
4167         int i;
4168         int err;
4169
4170         desc_info = rocker_desc_head_get(&rocker_port->tx_ring);
4171         if (unlikely(!desc_info)) {
4172                 if (net_ratelimit())
4173                         netdev_err(dev, "tx ring full when queue awake\n");
4174                 return NETDEV_TX_BUSY;
4175         }
4176
4177         rocker_desc_cookie_ptr_set(desc_info, skb);
4178
4179         frags = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAGS);
4180         if (!frags)
4181                 goto out;
4182         err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,
4183                                           skb->data, skb_headlen(skb));
4184         if (err)
4185                 goto nest_cancel;
4186         if (skb_shinfo(skb)->nr_frags > ROCKER_TX_FRAGS_MAX) {
4187                 err = skb_linearize(skb);
4188                 if (err)
4189                         goto unmap_frags;
4190         }
4191
4192         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
4193                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4194
4195                 err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,
4196                                                   skb_frag_address(frag),
4197                                                   skb_frag_size(frag));
4198                 if (err)
4199                         goto unmap_frags;
4200         }
4201         rocker_tlv_nest_end(desc_info, frags);
4202
4203         rocker_desc_gen_clear(desc_info);
4204         rocker_desc_head_set(rocker, &rocker_port->tx_ring, desc_info);
4205
4206         desc_info = rocker_desc_head_get(&rocker_port->tx_ring);
4207         if (!desc_info)
4208                 netif_stop_queue(dev);
4209
4210         return NETDEV_TX_OK;
4211
4212 unmap_frags:
4213         rocker_tx_desc_frags_unmap(rocker_port, desc_info);
4214 nest_cancel:
4215         rocker_tlv_nest_cancel(desc_info, frags);
4216 out:
4217         dev_kfree_skb(skb);
4218         dev->stats.tx_dropped++;
4219
4220         return NETDEV_TX_OK;
4221 }
4222
4223 static int rocker_port_set_mac_address(struct net_device *dev, void *p)
4224 {
4225         struct sockaddr *addr = p;
4226         struct rocker_port *rocker_port = netdev_priv(dev);
4227         int err;
4228
4229         if (!is_valid_ether_addr(addr->sa_data))
4230                 return -EADDRNOTAVAIL;
4231
4232         err = rocker_cmd_set_port_settings_macaddr(rocker_port, addr->sa_data);
4233         if (err)
4234                 return err;
4235         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4236         return 0;
4237 }
4238
4239 static int rocker_port_change_mtu(struct net_device *dev, int new_mtu)
4240 {
4241         struct rocker_port *rocker_port = netdev_priv(dev);
4242         int running = netif_running(dev);
4243         int err;
4244
4245 #define ROCKER_PORT_MIN_MTU     68
4246 #define ROCKER_PORT_MAX_MTU     9000
4247
4248         if (new_mtu < ROCKER_PORT_MIN_MTU || new_mtu > ROCKER_PORT_MAX_MTU)
4249                 return -EINVAL;
4250
4251         if (running)
4252                 rocker_port_stop(dev);
4253
4254         netdev_info(dev, "MTU change from %d to %d\n", dev->mtu, new_mtu);
4255         dev->mtu = new_mtu;
4256
4257         err = rocker_cmd_set_port_settings_mtu(rocker_port, new_mtu);
4258         if (err)
4259                 return err;
4260
4261         if (running)
4262                 err = rocker_port_open(dev);
4263
4264         return err;
4265 }
4266
4267 static int rocker_port_get_phys_port_name(struct net_device *dev,
4268                                           char *buf, size_t len)
4269 {
4270         struct rocker_port *rocker_port = netdev_priv(dev);
4271         struct port_name name = { .buf = buf, .len = len };
4272         int err;
4273
4274         err = rocker_cmd_exec(rocker_port, NULL, 0,
4275                               rocker_cmd_get_port_settings_prep, NULL,
4276                               rocker_cmd_get_port_settings_phys_name_proc,
4277                               &name);
4278
4279         return err ? -EOPNOTSUPP : 0;
4280 }
4281
4282 static int rocker_port_change_proto_down(struct net_device *dev,
4283                                          bool proto_down)
4284 {
4285         struct rocker_port *rocker_port = netdev_priv(dev);
4286
4287         if (rocker_port->dev->flags & IFF_UP)
4288                 rocker_port_set_enable(rocker_port, !proto_down);
4289         rocker_port->dev->proto_down = proto_down;
4290         return 0;
4291 }
4292
4293 static void rocker_port_neigh_destroy(struct neighbour *n)
4294 {
4295         struct rocker_port *rocker_port = netdev_priv(n->dev);
4296         int flags = ROCKER_OP_FLAG_REMOVE | ROCKER_OP_FLAG_NOWAIT;
4297         __be32 ip_addr = *(__be32 *)n->primary_key;
4298
4299         rocker_port_ipv4_neigh(rocker_port, NULL,
4300                                flags, ip_addr, n->ha);
4301 }
4302
4303 static const struct net_device_ops rocker_port_netdev_ops = {
4304         .ndo_open                       = rocker_port_open,
4305         .ndo_stop                       = rocker_port_stop,
4306         .ndo_start_xmit                 = rocker_port_xmit,
4307         .ndo_set_mac_address            = rocker_port_set_mac_address,
4308         .ndo_change_mtu                 = rocker_port_change_mtu,
4309         .ndo_bridge_getlink             = switchdev_port_bridge_getlink,
4310         .ndo_bridge_setlink             = switchdev_port_bridge_setlink,
4311         .ndo_bridge_dellink             = switchdev_port_bridge_dellink,
4312         .ndo_fdb_add                    = switchdev_port_fdb_add,
4313         .ndo_fdb_del                    = switchdev_port_fdb_del,
4314         .ndo_fdb_dump                   = switchdev_port_fdb_dump,
4315         .ndo_get_phys_port_name         = rocker_port_get_phys_port_name,
4316         .ndo_change_proto_down          = rocker_port_change_proto_down,
4317         .ndo_neigh_destroy              = rocker_port_neigh_destroy,
4318 };
4319
4320 /********************
4321  * swdev interface
4322  ********************/
4323
4324 static int rocker_port_attr_get(struct net_device *dev,
4325                                 struct switchdev_attr *attr)
4326 {
4327         const struct rocker_port *rocker_port = netdev_priv(dev);
4328         const struct rocker *rocker = rocker_port->rocker;
4329
4330         switch (attr->id) {
4331         case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
4332                 attr->u.ppid.id_len = sizeof(rocker->hw.id);
4333                 memcpy(&attr->u.ppid.id, &rocker->hw.id, attr->u.ppid.id_len);
4334                 break;
4335         case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
4336                 attr->u.brport_flags = rocker_port->brport_flags;
4337                 break;
4338         default:
4339                 return -EOPNOTSUPP;
4340         }
4341
4342         return 0;
4343 }
4344
4345 static int rocker_port_brport_flags_set(struct rocker_port *rocker_port,
4346                                         struct switchdev_trans *trans,
4347                                         unsigned long brport_flags)
4348 {
4349         unsigned long orig_flags;
4350         int err = 0;
4351
4352         orig_flags = rocker_port->brport_flags;
4353         rocker_port->brport_flags = brport_flags;
4354         if ((orig_flags ^ rocker_port->brport_flags) & BR_LEARNING)
4355                 err = rocker_port_set_learning(rocker_port, trans);
4356
4357         if (switchdev_trans_ph_prepare(trans))
4358                 rocker_port->brport_flags = orig_flags;
4359
4360         return err;
4361 }
4362
4363 static int rocker_port_bridge_ageing_time(struct rocker_port *rocker_port,
4364                                           struct switchdev_trans *trans,
4365                                           u32 ageing_time)
4366 {
4367         if (!switchdev_trans_ph_prepare(trans)) {
4368                 rocker_port->ageing_time = clock_t_to_jiffies(ageing_time);
4369                 mod_timer(&rocker_port->rocker->fdb_cleanup_timer, jiffies);
4370         }
4371
4372         return 0;
4373 }
4374
4375 static int rocker_port_attr_set(struct net_device *dev,
4376                                 const struct switchdev_attr *attr,
4377                                 struct switchdev_trans *trans)
4378 {
4379         struct rocker_port *rocker_port = netdev_priv(dev);
4380         int err = 0;
4381
4382         switch (attr->id) {
4383         case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
4384                 err = rocker_port_stp_update(rocker_port, trans, 0,
4385                                              attr->u.stp_state);
4386                 break;
4387         case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
4388                 err = rocker_port_brport_flags_set(rocker_port, trans,
4389                                                    attr->u.brport_flags);
4390                 break;
4391         case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
4392                 err = rocker_port_bridge_ageing_time(rocker_port, trans,
4393                                                      attr->u.ageing_time);
4394                 break;
4395         default:
4396                 err = -EOPNOTSUPP;
4397                 break;
4398         }
4399
4400         return err;
4401 }
4402
4403 static int rocker_port_vlan_add(struct rocker_port *rocker_port,
4404                                 struct switchdev_trans *trans,
4405                                 u16 vid, u16 flags)
4406 {
4407         int err;
4408
4409         /* XXX deal with flags for PVID and untagged */
4410
4411         err = rocker_port_vlan(rocker_port, trans, 0, vid);
4412         if (err)
4413                 return err;
4414
4415         err = rocker_port_router_mac(rocker_port, trans, 0, htons(vid));
4416         if (err)
4417                 rocker_port_vlan(rocker_port, trans,
4418                                  ROCKER_OP_FLAG_REMOVE, vid);
4419
4420         return err;
4421 }
4422
4423 static int rocker_port_vlans_add(struct rocker_port *rocker_port,
4424                                  struct switchdev_trans *trans,
4425                                  const struct switchdev_obj_port_vlan *vlan)
4426 {
4427         u16 vid;
4428         int err;
4429
4430         for (vid = vlan->vid_begin; vid <= vlan->vid_end; vid++) {
4431                 err = rocker_port_vlan_add(rocker_port, trans,
4432                                            vid, vlan->flags);
4433                 if (err)
4434                         return err;
4435         }
4436
4437         return 0;
4438 }
4439
4440 static int rocker_port_fdb_add(struct rocker_port *rocker_port,
4441                                struct switchdev_trans *trans,
4442                                const struct switchdev_obj_port_fdb *fdb)
4443 {
4444         __be16 vlan_id = rocker_port_vid_to_vlan(rocker_port, fdb->vid, NULL);
4445         int flags = 0;
4446
4447         if (!rocker_port_is_bridged(rocker_port))
4448                 return -EINVAL;
4449
4450         return rocker_port_fdb(rocker_port, trans, fdb->addr, vlan_id, flags);
4451 }
4452
4453 static int rocker_port_obj_add(struct net_device *dev,
4454                                const struct switchdev_obj *obj,
4455                                struct switchdev_trans *trans)
4456 {
4457         struct rocker_port *rocker_port = netdev_priv(dev);
4458         const struct switchdev_obj_ipv4_fib *fib4;
4459         int err = 0;
4460
4461         switch (obj->id) {
4462         case SWITCHDEV_OBJ_ID_PORT_VLAN:
4463                 err = rocker_port_vlans_add(rocker_port, trans,
4464                                             SWITCHDEV_OBJ_PORT_VLAN(obj));
4465                 break;
4466         case SWITCHDEV_OBJ_ID_IPV4_FIB:
4467                 fib4 = SWITCHDEV_OBJ_IPV4_FIB(obj);
4468                 err = rocker_port_fib_ipv4(rocker_port, trans,
4469                                            htonl(fib4->dst), fib4->dst_len,
4470                                            &fib4->fi, fib4->tb_id, 0);
4471                 break;
4472         case SWITCHDEV_OBJ_ID_PORT_FDB:
4473                 err = rocker_port_fdb_add(rocker_port, trans,
4474                                           SWITCHDEV_OBJ_PORT_FDB(obj));
4475                 break;
4476         default:
4477                 err = -EOPNOTSUPP;
4478                 break;
4479         }
4480
4481         return err;
4482 }
4483
4484 static int rocker_port_vlan_del(struct rocker_port *rocker_port,
4485                                 u16 vid, u16 flags)
4486 {
4487         int err;
4488
4489         err = rocker_port_router_mac(rocker_port, NULL,
4490                                      ROCKER_OP_FLAG_REMOVE, htons(vid));
4491         if (err)
4492                 return err;
4493
4494         return rocker_port_vlan(rocker_port, NULL,
4495                                 ROCKER_OP_FLAG_REMOVE, vid);
4496 }
4497
4498 static int rocker_port_vlans_del(struct rocker_port *rocker_port,
4499                                  const struct switchdev_obj_port_vlan *vlan)
4500 {
4501         u16 vid;
4502         int err;
4503
4504         for (vid = vlan->vid_begin; vid <= vlan->vid_end; vid++) {
4505                 err = rocker_port_vlan_del(rocker_port, vid, vlan->flags);
4506                 if (err)
4507                         return err;
4508         }
4509
4510         return 0;
4511 }
4512
4513 static int rocker_port_fdb_del(struct rocker_port *rocker_port,
4514                                struct switchdev_trans *trans,
4515                                const struct switchdev_obj_port_fdb *fdb)
4516 {
4517         __be16 vlan_id = rocker_port_vid_to_vlan(rocker_port, fdb->vid, NULL);
4518         int flags = ROCKER_OP_FLAG_REMOVE;
4519
4520         if (!rocker_port_is_bridged(rocker_port))
4521                 return -EINVAL;
4522
4523         return rocker_port_fdb(rocker_port, trans, fdb->addr, vlan_id, flags);
4524 }
4525
4526 static int rocker_port_obj_del(struct net_device *dev,
4527                                const struct switchdev_obj *obj)
4528 {
4529         struct rocker_port *rocker_port = netdev_priv(dev);
4530         const struct switchdev_obj_ipv4_fib *fib4;
4531         int err = 0;
4532
4533         switch (obj->id) {
4534         case SWITCHDEV_OBJ_ID_PORT_VLAN:
4535                 err = rocker_port_vlans_del(rocker_port,
4536                                             SWITCHDEV_OBJ_PORT_VLAN(obj));
4537                 break;
4538         case SWITCHDEV_OBJ_ID_IPV4_FIB:
4539                 fib4 = SWITCHDEV_OBJ_IPV4_FIB(obj);
4540                 err = rocker_port_fib_ipv4(rocker_port, NULL,
4541                                            htonl(fib4->dst), fib4->dst_len,
4542                                            &fib4->fi, fib4->tb_id,
4543                                            ROCKER_OP_FLAG_REMOVE);
4544                 break;
4545         case SWITCHDEV_OBJ_ID_PORT_FDB:
4546                 err = rocker_port_fdb_del(rocker_port, NULL,
4547                                           SWITCHDEV_OBJ_PORT_FDB(obj));
4548                 break;
4549         default:
4550                 err = -EOPNOTSUPP;
4551                 break;
4552         }
4553
4554         return err;
4555 }
4556
4557 static int rocker_port_fdb_dump(const struct rocker_port *rocker_port,
4558                                 struct switchdev_obj_port_fdb *fdb,
4559                                 switchdev_obj_dump_cb_t *cb)
4560 {
4561         struct rocker *rocker = rocker_port->rocker;
4562         struct rocker_fdb_tbl_entry *found;
4563         struct hlist_node *tmp;
4564         unsigned long lock_flags;
4565         int bkt;
4566         int err = 0;
4567
4568         spin_lock_irqsave(&rocker->fdb_tbl_lock, lock_flags);
4569         hash_for_each_safe(rocker->fdb_tbl, bkt, tmp, found, entry) {
4570                 if (found->key.rocker_port != rocker_port)
4571                         continue;
4572                 ether_addr_copy(fdb->addr, found->key.addr);
4573                 fdb->ndm_state = NUD_REACHABLE;
4574                 fdb->vid = rocker_port_vlan_to_vid(rocker_port,
4575                                                    found->key.vlan_id);
4576                 err = cb(&fdb->obj);
4577                 if (err)
4578                         break;
4579         }
4580         spin_unlock_irqrestore(&rocker->fdb_tbl_lock, lock_flags);
4581
4582         return err;
4583 }
4584
4585 static int rocker_port_vlan_dump(const struct rocker_port *rocker_port,
4586                                  struct switchdev_obj_port_vlan *vlan,
4587                                  switchdev_obj_dump_cb_t *cb)
4588 {
4589         u16 vid;
4590         int err = 0;
4591
4592         for (vid = 1; vid < VLAN_N_VID; vid++) {
4593                 if (!test_bit(vid, rocker_port->vlan_bitmap))
4594                         continue;
4595                 vlan->flags = 0;
4596                 if (rocker_vlan_id_is_internal(htons(vid)))
4597                         vlan->flags |= BRIDGE_VLAN_INFO_PVID;
4598                 vlan->vid_begin = vlan->vid_end = vid;
4599                 err = cb(&vlan->obj);
4600                 if (err)
4601                         break;
4602         }
4603
4604         return err;
4605 }
4606
4607 static int rocker_port_obj_dump(struct net_device *dev,
4608                                 struct switchdev_obj *obj,
4609                                 switchdev_obj_dump_cb_t *cb)
4610 {
4611         const struct rocker_port *rocker_port = netdev_priv(dev);
4612         int err = 0;
4613
4614         switch (obj->id) {
4615         case SWITCHDEV_OBJ_ID_PORT_FDB:
4616                 err = rocker_port_fdb_dump(rocker_port,
4617                                            SWITCHDEV_OBJ_PORT_FDB(obj), cb);
4618                 break;
4619         case SWITCHDEV_OBJ_ID_PORT_VLAN:
4620                 err = rocker_port_vlan_dump(rocker_port,
4621                                             SWITCHDEV_OBJ_PORT_VLAN(obj), cb);
4622                 break;
4623         default:
4624                 err = -EOPNOTSUPP;
4625                 break;
4626         }
4627
4628         return err;
4629 }
4630
4631 static const struct switchdev_ops rocker_port_switchdev_ops = {
4632         .switchdev_port_attr_get        = rocker_port_attr_get,
4633         .switchdev_port_attr_set        = rocker_port_attr_set,
4634         .switchdev_port_obj_add         = rocker_port_obj_add,
4635         .switchdev_port_obj_del         = rocker_port_obj_del,
4636         .switchdev_port_obj_dump        = rocker_port_obj_dump,
4637 };
4638
4639 /********************
4640  * ethtool interface
4641  ********************/
4642
4643 static int rocker_port_get_settings(struct net_device *dev,
4644                                     struct ethtool_cmd *ecmd)
4645 {
4646         struct rocker_port *rocker_port = netdev_priv(dev);
4647
4648         return rocker_cmd_get_port_settings_ethtool(rocker_port, ecmd);
4649 }
4650
4651 static int rocker_port_set_settings(struct net_device *dev,
4652                                     struct ethtool_cmd *ecmd)
4653 {
4654         struct rocker_port *rocker_port = netdev_priv(dev);
4655
4656         return rocker_cmd_set_port_settings_ethtool(rocker_port, ecmd);
4657 }
4658
4659 static void rocker_port_get_drvinfo(struct net_device *dev,
4660                                     struct ethtool_drvinfo *drvinfo)
4661 {
4662         strlcpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver));
4663         strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
4664 }
4665
4666 static struct rocker_port_stats {
4667         char str[ETH_GSTRING_LEN];
4668         int type;
4669 } rocker_port_stats[] = {
4670         { "rx_packets", ROCKER_TLV_CMD_PORT_STATS_RX_PKTS,    },
4671         { "rx_bytes",   ROCKER_TLV_CMD_PORT_STATS_RX_BYTES,   },
4672         { "rx_dropped", ROCKER_TLV_CMD_PORT_STATS_RX_DROPPED, },
4673         { "rx_errors",  ROCKER_TLV_CMD_PORT_STATS_RX_ERRORS,  },
4674
4675         { "tx_packets", ROCKER_TLV_CMD_PORT_STATS_TX_PKTS,    },
4676         { "tx_bytes",   ROCKER_TLV_CMD_PORT_STATS_TX_BYTES,   },
4677         { "tx_dropped", ROCKER_TLV_CMD_PORT_STATS_TX_DROPPED, },
4678         { "tx_errors",  ROCKER_TLV_CMD_PORT_STATS_TX_ERRORS,  },
4679 };
4680
4681 #define ROCKER_PORT_STATS_LEN  ARRAY_SIZE(rocker_port_stats)
4682
4683 static void rocker_port_get_strings(struct net_device *netdev, u32 stringset,
4684                                     u8 *data)
4685 {
4686         u8 *p = data;
4687         int i;
4688
4689         switch (stringset) {
4690         case ETH_SS_STATS:
4691                 for (i = 0; i < ARRAY_SIZE(rocker_port_stats); i++) {
4692                         memcpy(p, rocker_port_stats[i].str, ETH_GSTRING_LEN);
4693                         p += ETH_GSTRING_LEN;
4694                 }
4695                 break;
4696         }
4697 }
4698
4699 static int
4700 rocker_cmd_get_port_stats_prep(const struct rocker_port *rocker_port,
4701                                struct rocker_desc_info *desc_info,
4702                                void *priv)
4703 {
4704         struct rocker_tlv *cmd_stats;
4705
4706         if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,
4707                                ROCKER_TLV_CMD_TYPE_GET_PORT_STATS))
4708                 return -EMSGSIZE;
4709
4710         cmd_stats = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);
4711         if (!cmd_stats)
4712                 return -EMSGSIZE;
4713
4714         if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_STATS_PPORT,
4715                                rocker_port->pport))
4716                 return -EMSGSIZE;
4717
4718         rocker_tlv_nest_end(desc_info, cmd_stats);
4719
4720         return 0;
4721 }
4722
4723 static int
4724 rocker_cmd_get_port_stats_ethtool_proc(const struct rocker_port *rocker_port,
4725                                        const struct rocker_desc_info *desc_info,
4726                                        void *priv)
4727 {
4728         const struct rocker_tlv *attrs[ROCKER_TLV_CMD_MAX + 1];
4729         const struct rocker_tlv *stats_attrs[ROCKER_TLV_CMD_PORT_STATS_MAX + 1];
4730         const struct rocker_tlv *pattr;
4731         u32 pport;
4732         u64 *data = priv;
4733         int i;
4734
4735         rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);
4736
4737         if (!attrs[ROCKER_TLV_CMD_INFO])
4738                 return -EIO;
4739
4740         rocker_tlv_parse_nested(stats_attrs, ROCKER_TLV_CMD_PORT_STATS_MAX,
4741                                 attrs[ROCKER_TLV_CMD_INFO]);
4742
4743         if (!stats_attrs[ROCKER_TLV_CMD_PORT_STATS_PPORT])
4744                 return -EIO;
4745
4746         pport = rocker_tlv_get_u32(stats_attrs[ROCKER_TLV_CMD_PORT_STATS_PPORT]);
4747         if (pport != rocker_port->pport)
4748                 return -EIO;
4749
4750         for (i = 0; i < ARRAY_SIZE(rocker_port_stats); i++) {
4751                 pattr = stats_attrs[rocker_port_stats[i].type];
4752                 if (!pattr)
4753                         continue;
4754
4755                 data[i] = rocker_tlv_get_u64(pattr);
4756         }
4757
4758         return 0;
4759 }
4760
4761 static int rocker_cmd_get_port_stats_ethtool(struct rocker_port *rocker_port,
4762                                              void *priv)
4763 {
4764         return rocker_cmd_exec(rocker_port, NULL, 0,
4765                                rocker_cmd_get_port_stats_prep, NULL,
4766                                rocker_cmd_get_port_stats_ethtool_proc,
4767                                priv);
4768 }
4769
4770 static void rocker_port_get_stats(struct net_device *dev,
4771                                   struct ethtool_stats *stats, u64 *data)
4772 {
4773         struct rocker_port *rocker_port = netdev_priv(dev);
4774
4775         if (rocker_cmd_get_port_stats_ethtool(rocker_port, data) != 0) {
4776                 int i;
4777
4778                 for (i = 0; i < ARRAY_SIZE(rocker_port_stats); ++i)
4779                         data[i] = 0;
4780         }
4781 }
4782
4783 static int rocker_port_get_sset_count(struct net_device *netdev, int sset)
4784 {
4785         switch (sset) {
4786         case ETH_SS_STATS:
4787                 return ROCKER_PORT_STATS_LEN;
4788         default:
4789                 return -EOPNOTSUPP;
4790         }
4791 }
4792
4793 static const struct ethtool_ops rocker_port_ethtool_ops = {
4794         .get_settings           = rocker_port_get_settings,
4795         .set_settings           = rocker_port_set_settings,
4796         .get_drvinfo            = rocker_port_get_drvinfo,
4797         .get_link               = ethtool_op_get_link,
4798         .get_strings            = rocker_port_get_strings,
4799         .get_ethtool_stats      = rocker_port_get_stats,
4800         .get_sset_count         = rocker_port_get_sset_count,
4801 };
4802
4803 /*****************
4804  * NAPI interface
4805  *****************/
4806
4807 static struct rocker_port *rocker_port_napi_tx_get(struct napi_struct *napi)
4808 {
4809         return container_of(napi, struct rocker_port, napi_tx);
4810 }
4811
4812 static int rocker_port_poll_tx(struct napi_struct *napi, int budget)
4813 {
4814         struct rocker_port *rocker_port = rocker_port_napi_tx_get(napi);
4815         const struct rocker *rocker = rocker_port->rocker;
4816         const struct rocker_desc_info *desc_info;
4817         u32 credits = 0;
4818         int err;
4819
4820         /* Cleanup tx descriptors */
4821         while ((desc_info = rocker_desc_tail_get(&rocker_port->tx_ring))) {
4822                 struct sk_buff *skb;
4823
4824                 err = rocker_desc_err(desc_info);
4825                 if (err && net_ratelimit())
4826                         netdev_err(rocker_port->dev, "tx desc received with err %d\n",
4827                                    err);
4828                 rocker_tx_desc_frags_unmap(rocker_port, desc_info);
4829
4830                 skb = rocker_desc_cookie_ptr_get(desc_info);
4831                 if (err == 0) {
4832                         rocker_port->dev->stats.tx_packets++;
4833                         rocker_port->dev->stats.tx_bytes += skb->len;
4834                 } else {
4835                         rocker_port->dev->stats.tx_errors++;
4836                 }
4837
4838                 dev_kfree_skb_any(skb);
4839                 credits++;
4840         }
4841
4842         if (credits && netif_queue_stopped(rocker_port->dev))
4843                 netif_wake_queue(rocker_port->dev);
4844
4845         napi_complete(napi);
4846         rocker_dma_ring_credits_set(rocker, &rocker_port->tx_ring, credits);
4847
4848         return 0;
4849 }
4850
4851 static int rocker_port_rx_proc(const struct rocker *rocker,
4852                                const struct rocker_port *rocker_port,
4853                                struct rocker_desc_info *desc_info)
4854 {
4855         const struct rocker_tlv *attrs[ROCKER_TLV_RX_MAX + 1];
4856         struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);
4857         size_t rx_len;
4858         u16 rx_flags = 0;
4859
4860         if (!skb)
4861                 return -ENOENT;
4862
4863         rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);
4864         if (!attrs[ROCKER_TLV_RX_FRAG_LEN])
4865                 return -EINVAL;
4866         if (attrs[ROCKER_TLV_RX_FLAGS])
4867                 rx_flags = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FLAGS]);
4868
4869         rocker_dma_rx_ring_skb_unmap(rocker, attrs);
4870
4871         rx_len = rocker_tlv_get_u16(attrs[ROCKER_TLV_RX_FRAG_LEN]);
4872         skb_put(skb, rx_len);
4873         skb->protocol = eth_type_trans(skb, rocker_port->dev);
4874
4875         if (rx_flags & ROCKER_RX_FLAGS_FWD_OFFLOAD)
4876                 skb->offload_fwd_mark = rocker_port->dev->offload_fwd_mark;
4877
4878         rocker_port->dev->stats.rx_packets++;
4879         rocker_port->dev->stats.rx_bytes += skb->len;
4880
4881         netif_receive_skb(skb);
4882
4883         return rocker_dma_rx_ring_skb_alloc(rocker_port, desc_info);
4884 }
4885
4886 static struct rocker_port *rocker_port_napi_rx_get(struct napi_struct *napi)
4887 {
4888         return container_of(napi, struct rocker_port, napi_rx);
4889 }
4890
4891 static int rocker_port_poll_rx(struct napi_struct *napi, int budget)
4892 {
4893         struct rocker_port *rocker_port = rocker_port_napi_rx_get(napi);
4894         const struct rocker *rocker = rocker_port->rocker;
4895         struct rocker_desc_info *desc_info;
4896         u32 credits = 0;
4897         int err;
4898
4899         /* Process rx descriptors */
4900         while (credits < budget &&
4901                (desc_info = rocker_desc_tail_get(&rocker_port->rx_ring))) {
4902                 err = rocker_desc_err(desc_info);
4903                 if (err) {
4904                         if (net_ratelimit())
4905                                 netdev_err(rocker_port->dev, "rx desc received with err %d\n",
4906                                            err);
4907                 } else {
4908                         err = rocker_port_rx_proc(rocker, rocker_port,
4909                                                   desc_info);
4910                         if (err && net_ratelimit())
4911                                 netdev_err(rocker_port->dev, "rx processing failed with err %d\n",
4912                                            err);
4913                 }
4914                 if (err)
4915                         rocker_port->dev->stats.rx_errors++;
4916
4917                 rocker_desc_gen_clear(desc_info);
4918                 rocker_desc_head_set(rocker, &rocker_port->rx_ring, desc_info);
4919                 credits++;
4920         }
4921
4922         if (credits < budget)
4923                 napi_complete(napi);
4924
4925         rocker_dma_ring_credits_set(rocker, &rocker_port->rx_ring, credits);
4926
4927         return credits;
4928 }
4929
4930 /*****************
4931  * PCI driver ops
4932  *****************/
4933
4934 static void rocker_carrier_init(const struct rocker_port *rocker_port)
4935 {
4936         const struct rocker *rocker = rocker_port->rocker;
4937         u64 link_status = rocker_read64(rocker, PORT_PHYS_LINK_STATUS);
4938         bool link_up;
4939
4940         link_up = link_status & (1 << rocker_port->pport);
4941         if (link_up)
4942                 netif_carrier_on(rocker_port->dev);
4943         else
4944                 netif_carrier_off(rocker_port->dev);
4945 }
4946
4947 static void rocker_remove_ports(const struct rocker *rocker)
4948 {
4949         struct rocker_port *rocker_port;
4950         int i;
4951
4952         for (i = 0; i < rocker->port_count; i++) {
4953                 rocker_port = rocker->ports[i];
4954                 if (!rocker_port)
4955                         continue;
4956                 rocker_port_ig_tbl(rocker_port, NULL, ROCKER_OP_FLAG_REMOVE);
4957                 unregister_netdev(rocker_port->dev);
4958                 free_netdev(rocker_port->dev);
4959         }
4960         kfree(rocker->ports);
4961 }
4962
4963 static void rocker_port_dev_addr_init(struct rocker_port *rocker_port)
4964 {
4965         const struct rocker *rocker = rocker_port->rocker;
4966         const struct pci_dev *pdev = rocker->pdev;
4967         int err;
4968
4969         err = rocker_cmd_get_port_settings_macaddr(rocker_port,
4970                                                    rocker_port->dev->dev_addr);
4971         if (err) {
4972                 dev_warn(&pdev->dev, "failed to get mac address, using random\n");
4973                 eth_hw_addr_random(rocker_port->dev);
4974         }
4975 }
4976
4977 static int rocker_probe_port(struct rocker *rocker, unsigned int port_number)
4978 {
4979         const struct pci_dev *pdev = rocker->pdev;
4980         struct rocker_port *rocker_port;
4981         struct net_device *dev;
4982         u16 untagged_vid = 0;
4983         int err;
4984
4985         dev = alloc_etherdev(sizeof(struct rocker_port));
4986         if (!dev)
4987                 return -ENOMEM;
4988         rocker_port = netdev_priv(dev);
4989         rocker_port->dev = dev;
4990         rocker_port->rocker = rocker;
4991         rocker_port->port_number = port_number;
4992         rocker_port->pport = port_number + 1;
4993         rocker_port->brport_flags = BR_LEARNING | BR_LEARNING_SYNC;
4994         rocker_port->ageing_time = BR_DEFAULT_AGEING_TIME;
4995
4996         rocker_port_dev_addr_init(rocker_port);
4997         dev->netdev_ops = &rocker_port_netdev_ops;
4998         dev->ethtool_ops = &rocker_port_ethtool_ops;
4999         dev->switchdev_ops = &rocker_port_switchdev_ops;
5000         netif_tx_napi_add(dev, &rocker_port->napi_tx, rocker_port_poll_tx,
5001                        NAPI_POLL_WEIGHT);
5002         netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx,
5003                        NAPI_POLL_WEIGHT);
5004         rocker_carrier_init(rocker_port);
5005
5006         dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG;
5007
5008         err = register_netdev(dev);
5009         if (err) {
5010                 dev_err(&pdev->dev, "register_netdev failed\n");
5011                 goto err_register_netdev;
5012         }
5013         rocker->ports[port_number] = rocker_port;
5014
5015         switchdev_port_fwd_mark_set(rocker_port->dev, NULL, false);
5016
5017         rocker_port_set_learning(rocker_port, NULL);
5018
5019         err = rocker_port_ig_tbl(rocker_port, NULL, 0);
5020         if (err) {
5021                 netdev_err(rocker_port->dev, "install ig port table failed\n");
5022                 goto err_port_ig_tbl;
5023         }
5024
5025         rocker_port->internal_vlan_id =
5026                 rocker_port_internal_vlan_id_get(rocker_port, dev->ifindex);
5027
5028         err = rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
5029         if (err) {
5030                 netdev_err(rocker_port->dev, "install untagged VLAN failed\n");
5031                 goto err_untagged_vlan;
5032         }
5033
5034         return 0;
5035
5036 err_untagged_vlan:
5037         rocker_port_ig_tbl(rocker_port, NULL, ROCKER_OP_FLAG_REMOVE);
5038 err_port_ig_tbl:
5039         rocker->ports[port_number] = NULL;
5040         unregister_netdev(dev);
5041 err_register_netdev:
5042         free_netdev(dev);
5043         return err;
5044 }
5045
5046 static int rocker_probe_ports(struct rocker *rocker)
5047 {
5048         int i;
5049         size_t alloc_size;
5050         int err;
5051
5052         alloc_size = sizeof(struct rocker_port *) * rocker->port_count;
5053         rocker->ports = kzalloc(alloc_size, GFP_KERNEL);
5054         if (!rocker->ports)
5055                 return -ENOMEM;
5056         for (i = 0; i < rocker->port_count; i++) {
5057                 err = rocker_probe_port(rocker, i);
5058                 if (err)
5059                         goto remove_ports;
5060         }
5061         return 0;
5062
5063 remove_ports:
5064         rocker_remove_ports(rocker);
5065         return err;
5066 }
5067
5068 static int rocker_msix_init(struct rocker *rocker)
5069 {
5070         struct pci_dev *pdev = rocker->pdev;
5071         int msix_entries;
5072         int i;
5073         int err;
5074
5075         msix_entries = pci_msix_vec_count(pdev);
5076         if (msix_entries < 0)
5077                 return msix_entries;
5078
5079         if (msix_entries != ROCKER_MSIX_VEC_COUNT(rocker->port_count))
5080                 return -EINVAL;
5081
5082         rocker->msix_entries = kmalloc_array(msix_entries,
5083                                              sizeof(struct msix_entry),
5084                                              GFP_KERNEL);
5085         if (!rocker->msix_entries)
5086                 return -ENOMEM;
5087
5088         for (i = 0; i < msix_entries; i++)
5089                 rocker->msix_entries[i].entry = i;
5090
5091         err = pci_enable_msix_exact(pdev, rocker->msix_entries, msix_entries);
5092         if (err < 0)
5093                 goto err_enable_msix;
5094
5095         return 0;
5096
5097 err_enable_msix:
5098         kfree(rocker->msix_entries);
5099         return err;
5100 }
5101
5102 static void rocker_msix_fini(const struct rocker *rocker)
5103 {
5104         pci_disable_msix(rocker->pdev);
5105         kfree(rocker->msix_entries);
5106 }
5107
5108 static int rocker_probe(struct pci_dev *pdev, const struct pci_device_id *id)
5109 {
5110         struct rocker *rocker;
5111         int err;
5112
5113         rocker = kzalloc(sizeof(*rocker), GFP_KERNEL);
5114         if (!rocker)
5115                 return -ENOMEM;
5116
5117         err = pci_enable_device(pdev);
5118         if (err) {
5119                 dev_err(&pdev->dev, "pci_enable_device failed\n");
5120                 goto err_pci_enable_device;
5121         }
5122
5123         err = pci_request_regions(pdev, rocker_driver_name);
5124         if (err) {
5125                 dev_err(&pdev->dev, "pci_request_regions failed\n");
5126                 goto err_pci_request_regions;
5127         }
5128
5129         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
5130         if (!err) {
5131                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5132                 if (err) {
5133                         dev_err(&pdev->dev, "pci_set_consistent_dma_mask failed\n");
5134                         goto err_pci_set_dma_mask;
5135                 }
5136         } else {
5137                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5138                 if (err) {
5139                         dev_err(&pdev->dev, "pci_set_dma_mask failed\n");
5140                         goto err_pci_set_dma_mask;
5141                 }
5142         }
5143
5144         if (pci_resource_len(pdev, 0) < ROCKER_PCI_BAR0_SIZE) {
5145                 dev_err(&pdev->dev, "invalid PCI region size\n");
5146                 err = -EINVAL;
5147                 goto err_pci_resource_len_check;
5148         }
5149
5150         rocker->hw_addr = ioremap(pci_resource_start(pdev, 0),
5151                                   pci_resource_len(pdev, 0));
5152         if (!rocker->hw_addr) {
5153                 dev_err(&pdev->dev, "ioremap failed\n");
5154                 err = -EIO;
5155                 goto err_ioremap;
5156         }
5157         pci_set_master(pdev);
5158
5159         rocker->pdev = pdev;
5160         pci_set_drvdata(pdev, rocker);
5161
5162         rocker->port_count = rocker_read32(rocker, PORT_PHYS_COUNT);
5163
5164         err = rocker_msix_init(rocker);
5165         if (err) {
5166                 dev_err(&pdev->dev, "MSI-X init failed\n");
5167                 goto err_msix_init;
5168         }
5169
5170         err = rocker_basic_hw_test(rocker);
5171         if (err) {
5172                 dev_err(&pdev->dev, "basic hw test failed\n");
5173                 goto err_basic_hw_test;
5174         }
5175
5176         rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
5177
5178         err = rocker_dma_rings_init(rocker);
5179         if (err)
5180                 goto err_dma_rings_init;
5181
5182         err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD),
5183                           rocker_cmd_irq_handler, 0,
5184                           rocker_driver_name, rocker);
5185         if (err) {
5186                 dev_err(&pdev->dev, "cannot assign cmd irq\n");
5187                 goto err_request_cmd_irq;
5188         }
5189
5190         err = request_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT),
5191                           rocker_event_irq_handler, 0,
5192                           rocker_driver_name, rocker);
5193         if (err) {
5194                 dev_err(&pdev->dev, "cannot assign event irq\n");
5195                 goto err_request_event_irq;
5196         }
5197
5198         rocker->hw.id = rocker_read64(rocker, SWITCH_ID);
5199
5200         err = rocker_init_tbls(rocker);
5201         if (err) {
5202                 dev_err(&pdev->dev, "cannot init rocker tables\n");
5203                 goto err_init_tbls;
5204         }
5205
5206         setup_timer(&rocker->fdb_cleanup_timer, rocker_fdb_cleanup,
5207                     (unsigned long) rocker);
5208         mod_timer(&rocker->fdb_cleanup_timer, jiffies);
5209
5210         err = rocker_probe_ports(rocker);
5211         if (err) {
5212                 dev_err(&pdev->dev, "failed to probe ports\n");
5213                 goto err_probe_ports;
5214         }
5215
5216         dev_info(&pdev->dev, "Rocker switch with id %*phN\n",
5217                  (int)sizeof(rocker->hw.id), &rocker->hw.id);
5218
5219         return 0;
5220
5221 err_probe_ports:
5222         del_timer_sync(&rocker->fdb_cleanup_timer);
5223         rocker_free_tbls(rocker);
5224 err_init_tbls:
5225         free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
5226 err_request_event_irq:
5227         free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD), rocker);
5228 err_request_cmd_irq:
5229         rocker_dma_rings_fini(rocker);
5230 err_dma_rings_init:
5231 err_basic_hw_test:
5232         rocker_msix_fini(rocker);
5233 err_msix_init:
5234         iounmap(rocker->hw_addr);
5235 err_ioremap:
5236 err_pci_resource_len_check:
5237 err_pci_set_dma_mask:
5238         pci_release_regions(pdev);
5239 err_pci_request_regions:
5240         pci_disable_device(pdev);
5241 err_pci_enable_device:
5242         kfree(rocker);
5243         return err;
5244 }
5245
5246 static void rocker_remove(struct pci_dev *pdev)
5247 {
5248         struct rocker *rocker = pci_get_drvdata(pdev);
5249
5250         del_timer_sync(&rocker->fdb_cleanup_timer);
5251         rocker_free_tbls(rocker);
5252         rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
5253         rocker_remove_ports(rocker);
5254         free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
5255         free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_CMD), rocker);
5256         rocker_dma_rings_fini(rocker);
5257         rocker_msix_fini(rocker);
5258         iounmap(rocker->hw_addr);
5259         pci_release_regions(rocker->pdev);
5260         pci_disable_device(rocker->pdev);
5261         kfree(rocker);
5262 }
5263
5264 static struct pci_driver rocker_pci_driver = {
5265         .name           = rocker_driver_name,
5266         .id_table       = rocker_pci_id_table,
5267         .probe          = rocker_probe,
5268         .remove         = rocker_remove,
5269 };
5270
5271 /************************************
5272  * Net device notifier event handler
5273  ************************************/
5274
5275 static bool rocker_port_dev_check(const struct net_device *dev)
5276 {
5277         return dev->netdev_ops == &rocker_port_netdev_ops;
5278 }
5279
5280 static int rocker_port_bridge_join(struct rocker_port *rocker_port,
5281                                    struct net_device *bridge)
5282 {
5283         u16 untagged_vid = 0;
5284         int err;
5285
5286         /* Port is joining bridge, so the internal VLAN for the
5287          * port is going to change to the bridge internal VLAN.
5288          * Let's remove untagged VLAN (vid=0) from port and
5289          * re-add once internal VLAN has changed.
5290          */
5291
5292         err = rocker_port_vlan_del(rocker_port, untagged_vid, 0);
5293         if (err)
5294                 return err;
5295
5296         rocker_port_internal_vlan_id_put(rocker_port,
5297                                          rocker_port->dev->ifindex);
5298         rocker_port->internal_vlan_id =
5299                 rocker_port_internal_vlan_id_get(rocker_port, bridge->ifindex);
5300
5301         rocker_port->bridge_dev = bridge;
5302         switchdev_port_fwd_mark_set(rocker_port->dev, bridge, true);
5303
5304         return rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
5305 }
5306
5307 static int rocker_port_bridge_leave(struct rocker_port *rocker_port)
5308 {
5309         u16 untagged_vid = 0;
5310         int err;
5311
5312         err = rocker_port_vlan_del(rocker_port, untagged_vid, 0);
5313         if (err)
5314                 return err;
5315
5316         rocker_port_internal_vlan_id_put(rocker_port,
5317                                          rocker_port->bridge_dev->ifindex);
5318         rocker_port->internal_vlan_id =
5319                 rocker_port_internal_vlan_id_get(rocker_port,
5320                                                  rocker_port->dev->ifindex);
5321
5322         switchdev_port_fwd_mark_set(rocker_port->dev, rocker_port->bridge_dev,
5323                                     false);
5324         rocker_port->bridge_dev = NULL;
5325
5326         err = rocker_port_vlan_add(rocker_port, NULL, untagged_vid, 0);
5327         if (err)
5328                 return err;
5329
5330         if (rocker_port->dev->flags & IFF_UP)
5331                 err = rocker_port_fwd_enable(rocker_port, NULL, 0);
5332
5333         return err;
5334 }
5335
5336
5337 static int rocker_port_ovs_changed(struct rocker_port *rocker_port,
5338                                    struct net_device *master)
5339 {
5340         int err;
5341
5342         rocker_port->bridge_dev = master;
5343
5344         err = rocker_port_fwd_disable(rocker_port, NULL, 0);
5345         if (err)
5346                 return err;
5347         err = rocker_port_fwd_enable(rocker_port, NULL, 0);
5348
5349         return err;
5350 }
5351
5352 static int rocker_port_master_linked(struct rocker_port *rocker_port,
5353                                      struct net_device *master)
5354 {
5355         int err = 0;
5356
5357         if (netif_is_bridge_master(master))
5358                 err = rocker_port_bridge_join(rocker_port, master);
5359         else if (netif_is_ovs_master(master))
5360                 err = rocker_port_ovs_changed(rocker_port, master);
5361         return err;
5362 }
5363
5364 static int rocker_port_master_unlinked(struct rocker_port *rocker_port)
5365 {
5366         int err = 0;
5367
5368         if (rocker_port_is_bridged(rocker_port))
5369                 err = rocker_port_bridge_leave(rocker_port);
5370         else if (rocker_port_is_ovsed(rocker_port))
5371                 err = rocker_port_ovs_changed(rocker_port, NULL);
5372         return err;
5373 }
5374
5375 static int rocker_netdevice_event(struct notifier_block *unused,
5376                                   unsigned long event, void *ptr)
5377 {
5378         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
5379         struct netdev_notifier_changeupper_info *info;
5380         struct rocker_port *rocker_port;
5381         int err;
5382
5383         if (!rocker_port_dev_check(dev))
5384                 return NOTIFY_DONE;
5385
5386         switch (event) {
5387         case NETDEV_CHANGEUPPER:
5388                 info = ptr;
5389                 if (!info->master)
5390                         goto out;
5391                 rocker_port = netdev_priv(dev);
5392                 if (info->linking) {
5393                         err = rocker_port_master_linked(rocker_port,
5394                                                         info->upper_dev);
5395                         if (err)
5396                                 netdev_warn(dev, "failed to reflect master linked (err %d)\n",
5397                                             err);
5398                 } else {
5399                         err = rocker_port_master_unlinked(rocker_port);
5400                         if (err)
5401                                 netdev_warn(dev, "failed to reflect master unlinked (err %d)\n",
5402                                             err);
5403                 }
5404                 break;
5405         }
5406 out:
5407         return NOTIFY_DONE;
5408 }
5409
5410 static struct notifier_block rocker_netdevice_nb __read_mostly = {
5411         .notifier_call = rocker_netdevice_event,
5412 };
5413
5414 /************************************
5415  * Net event notifier event handler
5416  ************************************/
5417
5418 static int rocker_neigh_update(struct net_device *dev, struct neighbour *n)
5419 {
5420         struct rocker_port *rocker_port = netdev_priv(dev);
5421         int flags = (n->nud_state & NUD_VALID ? 0 : ROCKER_OP_FLAG_REMOVE) |
5422                     ROCKER_OP_FLAG_NOWAIT;
5423         __be32 ip_addr = *(__be32 *)n->primary_key;
5424
5425         return rocker_port_ipv4_neigh(rocker_port, NULL, flags, ip_addr, n->ha);
5426 }
5427
5428 static int rocker_netevent_event(struct notifier_block *unused,
5429                                  unsigned long event, void *ptr)
5430 {
5431         struct net_device *dev;
5432         struct neighbour *n = ptr;
5433         int err;
5434
5435         switch (event) {
5436         case NETEVENT_NEIGH_UPDATE:
5437                 if (n->tbl != &arp_tbl)
5438                         return NOTIFY_DONE;
5439                 dev = n->dev;
5440                 if (!rocker_port_dev_check(dev))
5441                         return NOTIFY_DONE;
5442                 err = rocker_neigh_update(dev, n);
5443                 if (err)
5444                         netdev_warn(dev,
5445                                     "failed to handle neigh update (err %d)\n",
5446                                     err);
5447                 break;
5448         }
5449
5450         return NOTIFY_DONE;
5451 }
5452
5453 static struct notifier_block rocker_netevent_nb __read_mostly = {
5454         .notifier_call = rocker_netevent_event,
5455 };
5456
5457 /***********************
5458  * Module init and exit
5459  ***********************/
5460
5461 static int __init rocker_module_init(void)
5462 {
5463         int err;
5464
5465         register_netdevice_notifier(&rocker_netdevice_nb);
5466         register_netevent_notifier(&rocker_netevent_nb);
5467         err = pci_register_driver(&rocker_pci_driver);
5468         if (err)
5469                 goto err_pci_register_driver;
5470         return 0;
5471
5472 err_pci_register_driver:
5473         unregister_netevent_notifier(&rocker_netevent_nb);
5474         unregister_netdevice_notifier(&rocker_netdevice_nb);
5475         return err;
5476 }
5477
5478 static void __exit rocker_module_exit(void)
5479 {
5480         unregister_netevent_notifier(&rocker_netevent_nb);
5481         unregister_netdevice_notifier(&rocker_netdevice_nb);
5482         pci_unregister_driver(&rocker_pci_driver);
5483 }
5484
5485 module_init(rocker_module_init);
5486 module_exit(rocker_module_exit);
5487
5488 MODULE_LICENSE("GPL v2");
5489 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
5490 MODULE_AUTHOR("Scott Feldman <sfeldma@gmail.com>");
5491 MODULE_DESCRIPTION("Rocker switch device driver");
5492 MODULE_DEVICE_TABLE(pci, rocker_pci_id_table);