drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static DEFINE_PCI_DEVICE_TABLE(be_dev_ids) = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190         val |= qid & DB_RQ_RING_ID_MASK;
 191         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 192
 193         wmb();
 194         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 195 }
 196
 197 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 198                           u16 posted)
 199 {
 200         u32 val = 0;
 201         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 202         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 203
 204         wmb();
 205         iowrite32(val, adapter->db + txo->db_offset);
 206 }
 207
 208 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 209                          bool arm, bool clear_int, u16 num_popped)
 210 {
 211         u32 val = 0;
 212         val |= qid & DB_EQ_RING_ID_MASK;
 213         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 214
 215         if (adapter->eeh_error)
 216                 return;
 217
 218         if (arm)
 219                 val |= 1 << DB_EQ_REARM_SHIFT;
 220         if (clear_int)
 221                 val |= 1 << DB_EQ_CLR_SHIFT;
 222         val |= 1 << DB_EQ_EVNT_SHIFT;
 223         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 224         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 225 }
 226
 227 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 228 {
 229         u32 val = 0;
 230         val |= qid & DB_CQ_RING_ID_MASK;
 231         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 232                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 233
 234         if (adapter->eeh_error)
 235                 return;
 236
 237         if (arm)
 238                 val |= 1 << DB_CQ_REARM_SHIFT;
 239         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 240         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 241 }
 242
 243 static int be_mac_addr_set(struct net_device *netdev, void *p)
 244 {
 245         struct be_adapter *adapter = netdev_priv(netdev);
 246         struct device *dev = &adapter->pdev->dev;
 247         struct sockaddr *addr = p;
 248         int status;
 249         u8 mac[ETH_ALEN];
 250         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 251
 252         if (!is_valid_ether_addr(addr->sa_data))
 253                 return -EADDRNOTAVAIL;
 254
 255         /* Proceed further only if, User provided MAC is different
 256          * from active MAC
 257          */
 258         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 259                 return 0;
 260
 261         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 262          * privilege or if PF did not provision the new MAC address.
 263          * On BE3, this cmd will always fail if the VF doesn't have the
 264          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 265          * the MAC for the VF.
 266          */
 267         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 268                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 269         if (!status) {
 270                 curr_pmac_id = adapter->pmac_id[0];
 271
 272                 /* Delete the old programmed MAC. This call may fail if the
 273                  * old MAC was already deleted by the PF driver.
 274                  */
 275                 if (adapter->pmac_id[0] != old_pmac_id)
 276                         be_cmd_pmac_del(adapter, adapter->if_handle,
 277                                         old_pmac_id, 0);
 278         }
 279
 280         /* Decide if the new MAC is successfully activated only after
 281          * querying the FW
 282          */
 283         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 284                                        adapter->if_handle, true, 0);
 285         if (status)
 286                 goto err;
 287
 288         /* The MAC change did not happen, either due to lack of privilege
 289          * or PF didn't pre-provision.
 290          */
 291         if (!ether_addr_equal(addr->sa_data, mac)) {
 292                 status = -EPERM;
 293                 goto err;
 294         }
 295
 296         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 297         dev_info(dev, "MAC address changed to %pM\n", mac);
 298         return 0;
 299 err:
 300         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 301         return status;
 302 }
 303
 304 /* BE2 supports only v0 cmd */
 305 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 306 {
 307         if (BE2_chip(adapter)) {
 308                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 309
 310                 return &cmd->hw_stats;
 311         } else if (BE3_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else {
 316                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         }
 320 }
 321
 322 /* BE2 supports only v0 cmd */
 323 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 324 {
 325         if (BE2_chip(adapter)) {
 326                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 327
 328                 return &hw_stats->erx;
 329         } else if (BE3_chip(adapter)) {
 330                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else {
 334                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         }
 338 }
 339
 340 static void populate_be_v0_stats(struct be_adapter *adapter)
 341 {
 342         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 343         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 344         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 345         struct be_port_rxf_stats_v0 *port_stats =
 346                                         &rxf_stats->port[adapter->port_num];
 347         struct be_drv_stats *drvs = &adapter->drv_stats;
 348
 349         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 350         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 351         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 352         drvs->rx_control_frames = port_stats->rx_control_frames;
 353         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 354         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 355         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 356         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 357         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 358         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 359         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 360         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 361         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 362         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 363         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 364         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 365         drvs->rx_dropped_header_too_small =
 366                 port_stats->rx_dropped_header_too_small;
 367         drvs->rx_address_filtered =
 368                                         port_stats->rx_address_filtered +
 369                                         port_stats->rx_vlan_filtered;
 370         drvs->rx_alignment_symbol_errors =
 371                 port_stats->rx_alignment_symbol_errors;
 372
 373         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 374         drvs->tx_controlframes = port_stats->tx_controlframes;
 375
 376         if (adapter->port_num)
 377                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 378         else
 379                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 380         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 381         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 382         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 383         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 384         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 385         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 386         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 387 }
 388
 389 static void populate_be_v1_stats(struct be_adapter *adapter)
 390 {
 391         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 392         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 393         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 394         struct be_port_rxf_stats_v1 *port_stats =
 395                                         &rxf_stats->port[adapter->port_num];
 396         struct be_drv_stats *drvs = &adapter->drv_stats;
 397
 398         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 399         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 400         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 401         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 402         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 403         drvs->rx_control_frames = port_stats->rx_control_frames;
 404         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 405         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 406         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 407         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 408         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 409         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 410         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 411         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 412         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 413         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 414         drvs->rx_dropped_header_too_small =
 415                 port_stats->rx_dropped_header_too_small;
 416         drvs->rx_input_fifo_overflow_drop =
 417                 port_stats->rx_input_fifo_overflow_drop;
 418         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 419         drvs->rx_alignment_symbol_errors =
 420                 port_stats->rx_alignment_symbol_errors;
 421         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 422         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 423         drvs->tx_controlframes = port_stats->tx_controlframes;
 424         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 425         drvs->jabber_events = port_stats->jabber_events;
 426         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 427         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 428         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 429         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 430         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 431         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 432         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 433 }
 434
 435 static void populate_be_v2_stats(struct be_adapter *adapter)
 436 {
 437         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 438         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 439         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 440         struct be_port_rxf_stats_v2 *port_stats =
 441                                         &rxf_stats->port[adapter->port_num];
 442         struct be_drv_stats *drvs = &adapter->drv_stats;
 443
 444         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 445         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 446         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 447         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 448         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 449         drvs->rx_control_frames = port_stats->rx_control_frames;
 450         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 451         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 452         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 453         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 454         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 455         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 456         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 457         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 458         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 459         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 460         drvs->rx_dropped_header_too_small =
 461                 port_stats->rx_dropped_header_too_small;
 462         drvs->rx_input_fifo_overflow_drop =
 463                 port_stats->rx_input_fifo_overflow_drop;
 464         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 465         drvs->rx_alignment_symbol_errors =
 466                 port_stats->rx_alignment_symbol_errors;
 467         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 468         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 469         drvs->tx_controlframes = port_stats->tx_controlframes;
 470         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 471         drvs->jabber_events = port_stats->jabber_events;
 472         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 473         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 474         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 475         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 476         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 477         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 478         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 479         if (be_roce_supported(adapter)) {
 480                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 481                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 482                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 483                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 484                 drvs->roce_drops_payload_len =
 485                         port_stats->roce_drops_payload_len;
 486         }
 487 }
 488
 489 static void populate_lancer_stats(struct be_adapter *adapter)
 490 {
 491
 492         struct be_drv_stats *drvs = &adapter->drv_stats;
 493         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 494
 495         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 496         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 497         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 498         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 499         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 500         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 501         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 502         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 503         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 504         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 505         drvs->rx_dropped_tcp_length =
 506                                 pport_stats->rx_dropped_invalid_tcp_length;
 507         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 508         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 509         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 510         drvs->rx_dropped_header_too_small =
 511                                 pport_stats->rx_dropped_header_too_small;
 512         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 513         drvs->rx_address_filtered =
 514                                         pport_stats->rx_address_filtered +
 515                                         pport_stats->rx_vlan_filtered;
 516         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 517         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 518         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 519         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 520         drvs->jabber_events = pport_stats->rx_jabbers;
 521         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 522         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 523         drvs->rx_drops_too_many_frags =
 524                                 pport_stats->rx_drops_too_many_frags_lo;
 525 }
 526
 527 static void accumulate_16bit_val(u32 *acc, u16 val)
 528 {
 529 #define lo(x)                   (x & 0xFFFF)
 530 #define hi(x)                   (x & 0xFFFF0000)
 531         bool wrapped = val < lo(*acc);
 532         u32 newacc = hi(*acc) + val;
 533
 534         if (wrapped)
 535                 newacc += 65536;
 536         ACCESS_ONCE(*acc) = newacc;
 537 }
 538
 539 static void populate_erx_stats(struct be_adapter *adapter,
 540                                struct be_rx_obj *rxo, u32 erx_stat)
 541 {
 542         if (!BEx_chip(adapter))
 543                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 544         else
 545                 /* below erx HW counter can actually wrap around after
 546                  * 65535. Driver accumulates a 32-bit value
 547                  */
 548                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 549                                      (u16)erx_stat);
 550 }
 551
 552 void be_parse_stats(struct be_adapter *adapter)
 553 {
 554         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 555         struct be_rx_obj *rxo;
 556         int i;
 557         u32 erx_stat;
 558
 559         if (lancer_chip(adapter)) {
 560                 populate_lancer_stats(adapter);
 561         } else {
 562                 if (BE2_chip(adapter))
 563                         populate_be_v0_stats(adapter);
 564                 else if (BE3_chip(adapter))
 565                         /* for BE3 */
 566                         populate_be_v1_stats(adapter);
 567                 else
 568                         populate_be_v2_stats(adapter);
 569
 570                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 571                 for_all_rx_queues(adapter, rxo, i) {
 572                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 573                         populate_erx_stats(adapter, rxo, erx_stat);
 574                 }
 575         }
 576 }
 577
 578 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 579                                                 struct rtnl_link_stats64 *stats)
 580 {
 581         struct be_adapter *adapter = netdev_priv(netdev);
 582         struct be_drv_stats *drvs = &adapter->drv_stats;
 583         struct be_rx_obj *rxo;
 584         struct be_tx_obj *txo;
 585         u64 pkts, bytes;
 586         unsigned int start;
 587         int i;
 588
 589         for_all_rx_queues(adapter, rxo, i) {
 590                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 591                 do {
 592                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 593                         pkts = rx_stats(rxo)->rx_pkts;
 594                         bytes = rx_stats(rxo)->rx_bytes;
 595                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 596                 stats->rx_packets += pkts;
 597                 stats->rx_bytes += bytes;
 598                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 599                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 600                                         rx_stats(rxo)->rx_drops_no_frags;
 601         }
 602
 603         for_all_tx_queues(adapter, txo, i) {
 604                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 605                 do {
 606                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 607                         pkts = tx_stats(txo)->tx_pkts;
 608                         bytes = tx_stats(txo)->tx_bytes;
 609                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 610                 stats->tx_packets += pkts;
 611                 stats->tx_bytes += bytes;
 612         }
 613
 614         /* bad pkts received */
 615         stats->rx_errors = drvs->rx_crc_errors +
 616                 drvs->rx_alignment_symbol_errors +
 617                 drvs->rx_in_range_errors +
 618                 drvs->rx_out_range_errors +
 619                 drvs->rx_frame_too_long +
 620                 drvs->rx_dropped_too_small +
 621                 drvs->rx_dropped_too_short +
 622                 drvs->rx_dropped_header_too_small +
 623                 drvs->rx_dropped_tcp_length +
 624                 drvs->rx_dropped_runt;
 625
 626         /* detailed rx errors */
 627         stats->rx_length_errors = drvs->rx_in_range_errors +
 628                 drvs->rx_out_range_errors +
 629                 drvs->rx_frame_too_long;
 630
 631         stats->rx_crc_errors = drvs->rx_crc_errors;
 632
 633         /* frame alignment errors */
 634         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 635
 636         /* receiver fifo overrun */
 637         /* drops_no_pbuf is no per i/f, it's per BE card */
 638         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 639                                 drvs->rx_input_fifo_overflow_drop +
 640                                 drvs->rx_drops_no_pbuf;
 641         return stats;
 642 }
 643
 644 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 645 {
 646         struct net_device *netdev = adapter->netdev;
 647
 648         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 649                 netif_carrier_off(netdev);
 650                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 651         }
 652
 653         if (link_status)
 654                 netif_carrier_on(netdev);
 655         else
 656                 netif_carrier_off(netdev);
 657 }
 658
 659 static void be_tx_stats_update(struct be_tx_obj *txo,
 660                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 661                                bool stopped)
 662 {
 663         struct be_tx_stats *stats = tx_stats(txo);
 664
 665         u64_stats_update_begin(&stats->sync);
 666         stats->tx_reqs++;
 667         stats->tx_wrbs += wrb_cnt;
 668         stats->tx_bytes += copied;
 669         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 670         if (stopped)
 671                 stats->tx_stops++;
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Determine number of WRB entries needed to xmit data in an skb */
 676 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 677                            bool *dummy)
 678 {
 679         int cnt = (skb->len > skb->data_len);
 680
 681         cnt += skb_shinfo(skb)->nr_frags;
 682
 683         /* to account for hdr wrb */
 684         cnt++;
 685         if (lancer_chip(adapter) || !(cnt & 1)) {
 686                 *dummy = false;
 687         } else {
 688                 /* add a dummy to make it an even num */
 689                 cnt++;
 690                 *dummy = true;
 691         }
 692         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 693         return cnt;
 694 }
 695
 696 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 697 {
 698         wrb->frag_pa_hi = upper_32_bits(addr);
 699         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 700         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 701         wrb->rsvd0 = 0;
 702 }
 703
 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 705                                      struct sk_buff *skb)
 706 {
 707         u8 vlan_prio;
 708         u16 vlan_tag;
 709
 710         vlan_tag = vlan_tx_tag_get(skb);
 711         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 712         /* If vlan priority provided by OS is NOT in available bmap */
 713         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 714                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 715                                 adapter->recommended_prio;
 716
 717         return vlan_tag;
 718 }
 719
 720 /* Used only for IP tunnel packets */
 721 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 722 {
 723         return (inner_ip_hdr(skb)->version == 4) ?
 724                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 725 }
 726
 727 static u16 skb_ip_proto(struct sk_buff *skb)
 728 {
 729         return (ip_hdr(skb)->version == 4) ?
 730                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 734                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 735                          bool skip_hw_vlan)
 736 {
 737         u16 vlan_tag, proto;
 738
 739         memset(hdr, 0, sizeof(*hdr));
 740
 741         AMAP_SET_BITS(struct amap_eth_hdr_wrb, crc, hdr, 1);
 742
 743         if (skb_is_gso(skb)) {
 744                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso, hdr, 1);
 745                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso_mss,
 746                         hdr, skb_shinfo(skb)->gso_size);
 747                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 748                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso6, hdr, 1);
 749         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 750                 if (skb->encapsulation) {
 751                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, ipcs, hdr, 1);
 752                         proto = skb_inner_ip_proto(skb);
 753                 } else {
 754                         proto = skb_ip_proto(skb);
 755                 }
 756                 if (proto == IPPROTO_TCP)
 757                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, tcpcs, hdr, 1);
 758                 else if (proto == IPPROTO_UDP)
 759                         AMAP_SET_BITS(struct amap_eth_hdr_wrb, udpcs, hdr, 1);
 760         }
 761
 762         if (vlan_tx_tag_present(skb)) {
 763                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan, hdr, 1);
 764                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 765                 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan_tag, hdr, vlan_tag);
 766         }
 767
 768         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 769         AMAP_SET_BITS(struct amap_eth_hdr_wrb, complete, hdr, !skip_hw_vlan);
 770         AMAP_SET_BITS(struct amap_eth_hdr_wrb, event, hdr, 1);
 771         AMAP_SET_BITS(struct amap_eth_hdr_wrb, num_wrb, hdr, wrb_cnt);
 772         AMAP_SET_BITS(struct amap_eth_hdr_wrb, len, hdr, len);
 773 }
 774
 775 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 776                           bool unmap_single)
 777 {
 778         dma_addr_t dma;
 779
 780         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 781
 782         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 783         if (wrb->frag_len) {
 784                 if (unmap_single)
 785                         dma_unmap_single(dev, dma, wrb->frag_len,
 786                                          DMA_TO_DEVICE);
 787                 else
 788                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 789         }
 790 }
 791
 792 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 793                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 794                         bool skip_hw_vlan)
 795 {
 796         dma_addr_t busaddr;
 797         int i, copied = 0;
 798         struct device *dev = &adapter->pdev->dev;
 799         struct sk_buff *first_skb = skb;
 800         struct be_eth_wrb *wrb;
 801         struct be_eth_hdr_wrb *hdr;
 802         bool map_single = false;
 803         u16 map_head;
 804
 805         hdr = queue_head_node(txq);
 806         queue_head_inc(txq);
 807         map_head = txq->head;
 808
 809         if (skb->len > skb->data_len) {
 810                 int len = skb_headlen(skb);
 811                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 812                 if (dma_mapping_error(dev, busaddr))
 813                         goto dma_err;
 814                 map_single = true;
 815                 wrb = queue_head_node(txq);
 816                 wrb_fill(wrb, busaddr, len);
 817                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 818                 queue_head_inc(txq);
 819                 copied += len;
 820         }
 821
 822         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 823                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 824                 busaddr = skb_frag_dma_map(dev, frag, 0,
 825                                            skb_frag_size(frag), DMA_TO_DEVICE);
 826                 if (dma_mapping_error(dev, busaddr))
 827                         goto dma_err;
 828                 wrb = queue_head_node(txq);
 829                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 830                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 831                 queue_head_inc(txq);
 832                 copied += skb_frag_size(frag);
 833         }
 834
 835         if (dummy_wrb) {
 836                 wrb = queue_head_node(txq);
 837                 wrb_fill(wrb, 0, 0);
 838                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 839                 queue_head_inc(txq);
 840         }
 841
 842         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 843         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 844
 845         return copied;
 846 dma_err:
 847         txq->head = map_head;
 848         while (copied) {
 849                 wrb = queue_head_node(txq);
 850                 unmap_tx_frag(dev, wrb, map_single);
 851                 map_single = false;
 852                 copied -= wrb->frag_len;
 853                 queue_head_inc(txq);
 854         }
 855         return 0;
 856 }
 857
 858 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 859                                              struct sk_buff *skb,
 860                                              bool *skip_hw_vlan)
 861 {
 862         u16 vlan_tag = 0;
 863
 864         skb = skb_share_check(skb, GFP_ATOMIC);
 865         if (unlikely(!skb))
 866                 return skb;
 867
 868         if (vlan_tx_tag_present(skb))
 869                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 870
 871         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 872                 if (!vlan_tag)
 873                         vlan_tag = adapter->pvid;
 874                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 875                  * skip VLAN insertion
 876                  */
 877                 if (skip_hw_vlan)
 878                         *skip_hw_vlan = true;
 879         }
 880
 881         if (vlan_tag) {
 882                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 if (skip_hw_vlan)
 895                         *skip_hw_vlan = true;
 896         }
 897
 898         return skb;
 899 }
 900
 901 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 902 {
 903         struct ethhdr *eh = (struct ethhdr *)skb->data;
 904         u16 offset = ETH_HLEN;
 905
 906         if (eh->h_proto == htons(ETH_P_IPV6)) {
 907                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 908
 909                 offset += sizeof(struct ipv6hdr);
 910                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 911                     ip6h->nexthdr != NEXTHDR_UDP) {
 912                         struct ipv6_opt_hdr *ehdr =
 913                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 914
 915                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 916                         if (ehdr->hdrlen == 0xff)
 917                                 return true;
 918                 }
 919         }
 920         return false;
 921 }
 922
 923 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 924 {
 925         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 926 }
 927
 928 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 929 {
 930         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 931 }
 932
 933 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 934                                                   struct sk_buff *skb,
 935                                                   bool *skip_hw_vlan)
 936 {
 937         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 938         unsigned int eth_hdr_len;
 939         struct iphdr *ip;
 940
 941         /* For padded packets, BE HW modifies tot_len field in IP header
 942          * incorrecly when VLAN tag is inserted by HW.
 943          * For padded packets, Lancer computes incorrect checksum.
 944          */
 945         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 946                                                 VLAN_ETH_HLEN : ETH_HLEN;
 947         if (skb->len <= 60 &&
 948             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 949             is_ipv4_pkt(skb)) {
 950                 ip = (struct iphdr *)ip_hdr(skb);
 951                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 952         }
 953
 954         /* If vlan tag is already inlined in the packet, skip HW VLAN
 955          * tagging in pvid-tagging mode
 956          */
 957         if (be_pvid_tagging_enabled(adapter) &&
 958             veh->h_vlan_proto == htons(ETH_P_8021Q))
 959                 *skip_hw_vlan = true;
 960
 961         /* HW has a bug wherein it will calculate CSUM for VLAN
 962          * pkts even though it is disabled.
 963          * Manually insert VLAN in pkt.
 964          */
 965         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 966             vlan_tx_tag_present(skb)) {
 967                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 968                 if (unlikely(!skb))
 969                         goto err;
 970         }
 971
 972         /* HW may lockup when VLAN HW tagging is requested on
 973          * certain ipv6 packets. Drop such pkts if the HW workaround to
 974          * skip HW tagging is not enabled by FW.
 975          */
 976         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 977             (adapter->pvid || adapter->qnq_vid) &&
 978             !qnq_async_evt_rcvd(adapter)))
 979                 goto tx_drop;
 980
 981         /* Manual VLAN tag insertion to prevent:
 982          * ASIC lockup when the ASIC inserts VLAN tag into
 983          * certain ipv6 packets. Insert VLAN tags in driver,
 984          * and set event, completion, vlan bits accordingly
 985          * in the Tx WRB.
 986          */
 987         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 988             be_vlan_tag_tx_chk(adapter, skb)) {
 989                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 990                 if (unlikely(!skb))
 991                         goto err;
 992         }
 993
 994         return skb;
 995 tx_drop:
 996         dev_kfree_skb_any(skb);
 997 err:
 998         return NULL;
 999 }
1000
1001 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1002                                            struct sk_buff *skb,
1003                                            bool *skip_hw_vlan)
1004 {
1005         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1006          * less may cause a transmit stall on that port. So the work-around is
1007          * to pad short packets (<= 32 bytes) to a 36-byte length.
1008          */
1009         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1010                 if (skb_padto(skb, 36))
1011                         return NULL;
1012                 skb->len = 36;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1025 {
1026         struct be_adapter *adapter = netdev_priv(netdev);
1027         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1028         struct be_queue_info *txq = &txo->q;
1029         bool dummy_wrb, stopped = false;
1030         u32 wrb_cnt = 0, copied = 0;
1031         bool skip_hw_vlan = false;
1032         u32 start = txq->head;
1033
1034         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1035         if (!skb) {
1036                 tx_stats(txo)->tx_drv_drops++;
1037                 return NETDEV_TX_OK;
1038         }
1039
1040         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1041
1042         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1043                               skip_hw_vlan);
1044         if (copied) {
1045                 int gso_segs = skb_shinfo(skb)->gso_segs;
1046
1047                 /* record the sent skb in the sent_skb table */
1048                 BUG_ON(txo->sent_skb_list[start]);
1049                 txo->sent_skb_list[start] = skb;
1050
1051                 /* Ensure txq has space for the next skb; Else stop the queue
1052                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1053                  * tx compls of the current transmit which'll wake up the queue
1054                  */
1055                 atomic_add(wrb_cnt, &txq->used);
1056                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1057                                                                 txq->len) {
1058                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1059                         stopped = true;
1060                 }
1061
1062                 be_txq_notify(adapter, txo, wrb_cnt);
1063
1064                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1065         } else {
1066                 txq->head = start;
1067                 tx_stats(txo)->tx_drv_drops++;
1068                 dev_kfree_skb_any(skb);
1069         }
1070         return NETDEV_TX_OK;
1071 }
1072
1073 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1074 {
1075         struct be_adapter *adapter = netdev_priv(netdev);
1076         if (new_mtu < BE_MIN_MTU ||
1077             new_mtu > (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN))) {
1078                 dev_info(&adapter->pdev->dev,
1079                          "MTU must be between %d and %d bytes\n",
1080                          BE_MIN_MTU,
1081                          (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN)));
1082                 return -EINVAL;
1083         }
1084         dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n",
1085                  netdev->mtu, new_mtu);
1086         netdev->mtu = new_mtu;
1087         return 0;
1088 }
1089
1090 /*
1091  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1092  * If the user configures more, place BE in vlan promiscuous mode.
1093  */
1094 static int be_vid_config(struct be_adapter *adapter)
1095 {
1096         u16 vids[BE_NUM_VLANS_SUPPORTED];
1097         u16 num = 0, i = 0;
1098         int status = 0;
1099
1100         /* No need to further configure vids if in promiscuous mode */
1101         if (adapter->promiscuous)
1102                 return 0;
1103
1104         if (adapter->vlans_added > be_max_vlans(adapter))
1105                 goto set_vlan_promisc;
1106
1107         /* Construct VLAN Table to give to HW */
1108         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1109                 vids[num++] = cpu_to_le16(i);
1110
1111         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1112         if (status) {
1113                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1114                 if (addl_status(status) ==
1115                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1116                         goto set_vlan_promisc;
1117                 dev_err(&adapter->pdev->dev,
1118                         "Setting HW VLAN filtering failed.\n");
1119         } else {
1120                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1121                         /* hw VLAN filtering re-enabled. */
1122                         status = be_cmd_rx_filter(adapter,
1123                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1124                         if (!status) {
1125                                 dev_info(&adapter->pdev->dev,
1126                                          "Disabling VLAN Promiscuous mode.\n");
1127                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1128                         }
1129                 }
1130         }
1131
1132         return status;
1133
1134 set_vlan_promisc:
1135         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1136                 return 0;
1137
1138         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1139         if (!status) {
1140                 dev_info(&adapter->pdev->dev, "Enable VLAN Promiscuous mode\n");
1141                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1142         } else
1143                 dev_err(&adapter->pdev->dev,
1144                         "Failed to enable VLAN Promiscuous mode.\n");
1145         return status;
1146 }
1147
1148 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1149 {
1150         struct be_adapter *adapter = netdev_priv(netdev);
1151         int status = 0;
1152
1153         /* Packets with VID 0 are always received by Lancer by default */
1154         if (lancer_chip(adapter) && vid == 0)
1155                 return status;
1156
1157         if (test_bit(vid, adapter->vids))
1158                 return status;
1159
1160         set_bit(vid, adapter->vids);
1161         adapter->vlans_added++;
1162
1163         status = be_vid_config(adapter);
1164         if (status) {
1165                 adapter->vlans_added--;
1166                 clear_bit(vid, adapter->vids);
1167         }
1168
1169         return status;
1170 }
1171
1172 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1173 {
1174         struct be_adapter *adapter = netdev_priv(netdev);
1175
1176         /* Packets with VID 0 are always received by Lancer by default */
1177         if (lancer_chip(adapter) && vid == 0)
1178                 return 0;
1179
1180         clear_bit(vid, adapter->vids);
1181         adapter->vlans_added--;
1182
1183         return be_vid_config(adapter);
1184 }
1185
1186 static void be_clear_promisc(struct be_adapter *adapter)
1187 {
1188         adapter->promiscuous = false;
1189         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1190
1191         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1192 }
1193
1194 static void be_set_rx_mode(struct net_device *netdev)
1195 {
1196         struct be_adapter *adapter = netdev_priv(netdev);
1197         int status;
1198
1199         if (netdev->flags & IFF_PROMISC) {
1200                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1201                 adapter->promiscuous = true;
1202                 goto done;
1203         }
1204
1205         /* BE was previously in promiscuous mode; disable it */
1206         if (adapter->promiscuous) {
1207                 be_clear_promisc(adapter);
1208                 if (adapter->vlans_added)
1209                         be_vid_config(adapter);
1210         }
1211
1212         /* Enable multicast promisc if num configured exceeds what we support */
1213         if (netdev->flags & IFF_ALLMULTI ||
1214             netdev_mc_count(netdev) > be_max_mc(adapter))
1215                 goto set_mcast_promisc;
1216
1217         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1218                 struct netdev_hw_addr *ha;
1219                 int i = 1; /* First slot is claimed by the Primary MAC */
1220
1221                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1222                         be_cmd_pmac_del(adapter, adapter->if_handle,
1223                                         adapter->pmac_id[i], 0);
1224                 }
1225
1226                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1227                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1228                         adapter->promiscuous = true;
1229                         goto done;
1230                 }
1231
1232                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1233                         adapter->uc_macs++; /* First slot is for Primary MAC */
1234                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1235                                         adapter->if_handle,
1236                                         &adapter->pmac_id[adapter->uc_macs], 0);
1237                 }
1238         }
1239
1240         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1241         if (!status) {
1242                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1243                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1244                 goto done;
1245         }
1246
1247 set_mcast_promisc:
1248         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1249                 return;
1250
1251         /* Set to MCAST promisc mode if setting MULTICAST address fails
1252          * or if num configured exceeds what we support
1253          */
1254         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1255         if (!status)
1256                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1257 done:
1258         return;
1259 }
1260
1261 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1262 {
1263         struct be_adapter *adapter = netdev_priv(netdev);
1264         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1265         int status;
1266
1267         if (!sriov_enabled(adapter))
1268                 return -EPERM;
1269
1270         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1271                 return -EINVAL;
1272
1273         if (BEx_chip(adapter)) {
1274                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1275                                 vf + 1);
1276
1277                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1278                                          &vf_cfg->pmac_id, vf + 1);
1279         } else {
1280                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1281                                         vf + 1);
1282         }
1283
1284         if (status) {
1285                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1286                         mac, vf, status);
1287                 return be_cmd_status(status);
1288         }
1289
1290         ether_addr_copy(vf_cfg->mac_addr, mac);
1291
1292         return 0;
1293 }
1294
1295 static int be_get_vf_config(struct net_device *netdev, int vf,
1296                             struct ifla_vf_info *vi)
1297 {
1298         struct be_adapter *adapter = netdev_priv(netdev);
1299         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1300
1301         if (!sriov_enabled(adapter))
1302                 return -EPERM;
1303
1304         if (vf >= adapter->num_vfs)
1305                 return -EINVAL;
1306
1307         vi->vf = vf;
1308         vi->max_tx_rate = vf_cfg->tx_rate;
1309         vi->min_tx_rate = 0;
1310         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1311         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1312         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1313         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1314
1315         return 0;
1316 }
1317
1318 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1319 {
1320         struct be_adapter *adapter = netdev_priv(netdev);
1321         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1322         int status = 0;
1323
1324         if (!sriov_enabled(adapter))
1325                 return -EPERM;
1326
1327         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1328                 return -EINVAL;
1329
1330         if (vlan || qos) {
1331                 vlan |= qos << VLAN_PRIO_SHIFT;
1332                 if (vf_cfg->vlan_tag != vlan)
1333                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1334                                                        vf_cfg->if_handle, 0);
1335         } else {
1336                 /* Reset Transparent Vlan Tagging. */
1337                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1338                                                vf + 1, vf_cfg->if_handle, 0);
1339         }
1340
1341         if (status) {
1342                 dev_err(&adapter->pdev->dev,
1343                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1344                         vf, status);
1345                 return be_cmd_status(status);
1346         }
1347
1348         vf_cfg->vlan_tag = vlan;
1349
1350         return 0;
1351 }
1352
1353 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1354                              int min_tx_rate, int max_tx_rate)
1355 {
1356         struct be_adapter *adapter = netdev_priv(netdev);
1357         struct device *dev = &adapter->pdev->dev;
1358         int percent_rate, status = 0;
1359         u16 link_speed = 0;
1360         u8 link_status;
1361
1362         if (!sriov_enabled(adapter))
1363                 return -EPERM;
1364
1365         if (vf >= adapter->num_vfs)
1366                 return -EINVAL;
1367
1368         if (min_tx_rate)
1369                 return -EINVAL;
1370
1371         if (!max_tx_rate)
1372                 goto config_qos;
1373
1374         status = be_cmd_link_status_query(adapter, &link_speed,
1375                                           &link_status, 0);
1376         if (status)
1377                 goto err;
1378
1379         if (!link_status) {
1380                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1381                 status = -ENETDOWN;
1382                 goto err;
1383         }
1384
1385         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1386                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1387                         link_speed);
1388                 status = -EINVAL;
1389                 goto err;
1390         }
1391
1392         /* On Skyhawk the QOS setting must be done only as a % value */
1393         percent_rate = link_speed / 100;
1394         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1395                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1396                         percent_rate);
1397                 status = -EINVAL;
1398                 goto err;
1399         }
1400
1401 config_qos:
1402         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1403         if (status)
1404                 goto err;
1405
1406         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1407         return 0;
1408
1409 err:
1410         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1411                 max_tx_rate, vf);
1412         return be_cmd_status(status);
1413 }
1414 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1415                                 int link_state)
1416 {
1417         struct be_adapter *adapter = netdev_priv(netdev);
1418         int status;
1419
1420         if (!sriov_enabled(adapter))
1421                 return -EPERM;
1422
1423         if (vf >= adapter->num_vfs)
1424                 return -EINVAL;
1425
1426         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1427         if (status) {
1428                 dev_err(&adapter->pdev->dev,
1429                         "Link state change on VF %d failed: %#x\n", vf, status);
1430                 return be_cmd_status(status);
1431         }
1432
1433         adapter->vf_cfg[vf].plink_tracking = link_state;
1434
1435         return 0;
1436 }
1437
1438 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1439                           ulong now)
1440 {
1441         aic->rx_pkts_prev = rx_pkts;
1442         aic->tx_reqs_prev = tx_pkts;
1443         aic->jiffies = now;
1444 }
1445
1446 static void be_eqd_update(struct be_adapter *adapter)
1447 {
1448         struct be_set_eqd set_eqd[MAX_EVT_QS];
1449         int eqd, i, num = 0, start;
1450         struct be_aic_obj *aic;
1451         struct be_eq_obj *eqo;
1452         struct be_rx_obj *rxo;
1453         struct be_tx_obj *txo;
1454         u64 rx_pkts, tx_pkts;
1455         ulong now;
1456         u32 pps, delta;
1457
1458         for_all_evt_queues(adapter, eqo, i) {
1459                 aic = &adapter->aic_obj[eqo->idx];
1460                 if (!aic->enable) {
1461                         if (aic->jiffies)
1462                                 aic->jiffies = 0;
1463                         eqd = aic->et_eqd;
1464                         goto modify_eqd;
1465                 }
1466
1467                 rxo = &adapter->rx_obj[eqo->idx];
1468                 do {
1469                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1470                         rx_pkts = rxo->stats.rx_pkts;
1471                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1472
1473                 txo = &adapter->tx_obj[eqo->idx];
1474                 do {
1475                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1476                         tx_pkts = txo->stats.tx_reqs;
1477                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1478
1479
1480                 /* Skip, if wrapped around or first calculation */
1481                 now = jiffies;
1482                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1483                     rx_pkts < aic->rx_pkts_prev ||
1484                     tx_pkts < aic->tx_reqs_prev) {
1485                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1486                         continue;
1487                 }
1488
1489                 delta = jiffies_to_msecs(now - aic->jiffies);
1490                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1491                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1492                 eqd = (pps / 15000) << 2;
1493
1494                 if (eqd < 8)
1495                         eqd = 0;
1496                 eqd = min_t(u32, eqd, aic->max_eqd);
1497                 eqd = max_t(u32, eqd, aic->min_eqd);
1498
1499                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1500 modify_eqd:
1501                 if (eqd != aic->prev_eqd) {
1502                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1503                         set_eqd[num].eq_id = eqo->q.id;
1504                         aic->prev_eqd = eqd;
1505                         num++;
1506                 }
1507         }
1508
1509         if (num)
1510                 be_cmd_modify_eqd(adapter, set_eqd, num);
1511 }
1512
1513 static void be_rx_stats_update(struct be_rx_obj *rxo,
1514                                struct be_rx_compl_info *rxcp)
1515 {
1516         struct be_rx_stats *stats = rx_stats(rxo);
1517
1518         u64_stats_update_begin(&stats->sync);
1519         stats->rx_compl++;
1520         stats->rx_bytes += rxcp->pkt_size;
1521         stats->rx_pkts++;
1522         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1523                 stats->rx_mcast_pkts++;
1524         if (rxcp->err)
1525                 stats->rx_compl_err++;
1526         u64_stats_update_end(&stats->sync);
1527 }
1528
1529 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1530 {
1531         /* L4 checksum is not reliable for non TCP/UDP packets.
1532          * Also ignore ipcksm for ipv6 pkts
1533          */
1534         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1535                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1536 }
1537
1538 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1539 {
1540         struct be_adapter *adapter = rxo->adapter;
1541         struct be_rx_page_info *rx_page_info;
1542         struct be_queue_info *rxq = &rxo->q;
1543         u16 frag_idx = rxq->tail;
1544
1545         rx_page_info = &rxo->page_info_tbl[frag_idx];
1546         BUG_ON(!rx_page_info->page);
1547
1548         if (rx_page_info->last_frag) {
1549                 dma_unmap_page(&adapter->pdev->dev,
1550                                dma_unmap_addr(rx_page_info, bus),
1551                                adapter->big_page_size, DMA_FROM_DEVICE);
1552                 rx_page_info->last_frag = false;
1553         } else {
1554                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1555                                         dma_unmap_addr(rx_page_info, bus),
1556                                         rx_frag_size, DMA_FROM_DEVICE);
1557         }
1558
1559         queue_tail_inc(rxq);
1560         atomic_dec(&rxq->used);
1561         return rx_page_info;
1562 }
1563
1564 /* Throwaway the data in the Rx completion */
1565 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1566                                 struct be_rx_compl_info *rxcp)
1567 {
1568         struct be_rx_page_info *page_info;
1569         u16 i, num_rcvd = rxcp->num_rcvd;
1570
1571         for (i = 0; i < num_rcvd; i++) {
1572                 page_info = get_rx_page_info(rxo);
1573                 put_page(page_info->page);
1574                 memset(page_info, 0, sizeof(*page_info));
1575         }
1576 }
1577
1578 /*
1579  * skb_fill_rx_data forms a complete skb for an ether frame
1580  * indicated by rxcp.
1581  */
1582 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1583                              struct be_rx_compl_info *rxcp)
1584 {
1585         struct be_rx_page_info *page_info;
1586         u16 i, j;
1587         u16 hdr_len, curr_frag_len, remaining;
1588         u8 *start;
1589
1590         page_info = get_rx_page_info(rxo);
1591         start = page_address(page_info->page) + page_info->page_offset;
1592         prefetch(start);
1593
1594         /* Copy data in the first descriptor of this completion */
1595         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1596
1597         skb->len = curr_frag_len;
1598         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1599                 memcpy(skb->data, start, curr_frag_len);
1600                 /* Complete packet has now been moved to data */
1601                 put_page(page_info->page);
1602                 skb->data_len = 0;
1603                 skb->tail += curr_frag_len;
1604         } else {
1605                 hdr_len = ETH_HLEN;
1606                 memcpy(skb->data, start, hdr_len);
1607                 skb_shinfo(skb)->nr_frags = 1;
1608                 skb_frag_set_page(skb, 0, page_info->page);
1609                 skb_shinfo(skb)->frags[0].page_offset =
1610                                         page_info->page_offset + hdr_len;
1611                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1612                                   curr_frag_len - hdr_len);
1613                 skb->data_len = curr_frag_len - hdr_len;
1614                 skb->truesize += rx_frag_size;
1615                 skb->tail += hdr_len;
1616         }
1617         page_info->page = NULL;
1618
1619         if (rxcp->pkt_size <= rx_frag_size) {
1620                 BUG_ON(rxcp->num_rcvd != 1);
1621                 return;
1622         }
1623
1624         /* More frags present for this completion */
1625         remaining = rxcp->pkt_size - curr_frag_len;
1626         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1627                 page_info = get_rx_page_info(rxo);
1628                 curr_frag_len = min(remaining, rx_frag_size);
1629
1630                 /* Coalesce all frags from the same physical page in one slot */
1631                 if (page_info->page_offset == 0) {
1632                         /* Fresh page */
1633                         j++;
1634                         skb_frag_set_page(skb, j, page_info->page);
1635                         skb_shinfo(skb)->frags[j].page_offset =
1636                                                         page_info->page_offset;
1637                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1638                         skb_shinfo(skb)->nr_frags++;
1639                 } else {
1640                         put_page(page_info->page);
1641                 }
1642
1643                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1644                 skb->len += curr_frag_len;
1645                 skb->data_len += curr_frag_len;
1646                 skb->truesize += rx_frag_size;
1647                 remaining -= curr_frag_len;
1648                 page_info->page = NULL;
1649         }
1650         BUG_ON(j > MAX_SKB_FRAGS);
1651 }
1652
1653 /* Process the RX completion indicated by rxcp when GRO is disabled */
1654 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1655                                 struct be_rx_compl_info *rxcp)
1656 {
1657         struct be_adapter *adapter = rxo->adapter;
1658         struct net_device *netdev = adapter->netdev;
1659         struct sk_buff *skb;
1660
1661         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1662         if (unlikely(!skb)) {
1663                 rx_stats(rxo)->rx_drops_no_skbs++;
1664                 be_rx_compl_discard(rxo, rxcp);
1665                 return;
1666         }
1667
1668         skb_fill_rx_data(rxo, skb, rxcp);
1669
1670         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1671                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1672         else
1673                 skb_checksum_none_assert(skb);
1674
1675         skb->protocol = eth_type_trans(skb, netdev);
1676         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1677         if (netdev->features & NETIF_F_RXHASH)
1678                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1679
1680         skb->encapsulation = rxcp->tunneled;
1681         skb_mark_napi_id(skb, napi);
1682
1683         if (rxcp->vlanf)
1684                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1685
1686         netif_receive_skb(skb);
1687 }
1688
1689 /* Process the RX completion indicated by rxcp when GRO is enabled */
1690 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1691                                     struct napi_struct *napi,
1692                                     struct be_rx_compl_info *rxcp)
1693 {
1694         struct be_adapter *adapter = rxo->adapter;
1695         struct be_rx_page_info *page_info;
1696         struct sk_buff *skb = NULL;
1697         u16 remaining, curr_frag_len;
1698         u16 i, j;
1699
1700         skb = napi_get_frags(napi);
1701         if (!skb) {
1702                 be_rx_compl_discard(rxo, rxcp);
1703                 return;
1704         }
1705
1706         remaining = rxcp->pkt_size;
1707         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1708                 page_info = get_rx_page_info(rxo);
1709
1710                 curr_frag_len = min(remaining, rx_frag_size);
1711
1712                 /* Coalesce all frags from the same physical page in one slot */
1713                 if (i == 0 || page_info->page_offset == 0) {
1714                         /* First frag or Fresh page */
1715                         j++;
1716                         skb_frag_set_page(skb, j, page_info->page);
1717                         skb_shinfo(skb)->frags[j].page_offset =
1718                                                         page_info->page_offset;
1719                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1720                 } else {
1721                         put_page(page_info->page);
1722                 }
1723                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1724                 skb->truesize += rx_frag_size;
1725                 remaining -= curr_frag_len;
1726                 memset(page_info, 0, sizeof(*page_info));
1727         }
1728         BUG_ON(j > MAX_SKB_FRAGS);
1729
1730         skb_shinfo(skb)->nr_frags = j + 1;
1731         skb->len = rxcp->pkt_size;
1732         skb->data_len = rxcp->pkt_size;
1733         skb->ip_summed = CHECKSUM_UNNECESSARY;
1734         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1735         if (adapter->netdev->features & NETIF_F_RXHASH)
1736                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1737
1738         skb->encapsulation = rxcp->tunneled;
1739         skb_mark_napi_id(skb, napi);
1740
1741         if (rxcp->vlanf)
1742                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1743
1744         napi_gro_frags(napi);
1745 }
1746
1747 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1748                                  struct be_rx_compl_info *rxcp)
1749 {
1750         rxcp->pkt_size =
1751                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, pktsize, compl);
1752         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtp, compl);
1753         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, err, compl);
1754         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tcpf, compl);
1755         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, udpf, compl);
1756         rxcp->ip_csum =
1757                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ipcksm, compl);
1758         rxcp->l4_csum =
1759                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, l4_cksm, compl);
1760         rxcp->ipv6 =
1761                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ip_version, compl);
1762         rxcp->num_rcvd =
1763                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, numfrags, compl);
1764         rxcp->pkt_type =
1765                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, cast_enc, compl);
1766         rxcp->rss_hash =
1767                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, rsshash, compl);
1768         if (rxcp->vlanf) {
1769                 rxcp->qnq = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, qnq,
1770                                           compl);
1771                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v1,
1772                                                vlan_tag, compl);
1773         }
1774         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, port, compl);
1775         rxcp->tunneled =
1776                 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tunneled, compl);
1777 }
1778
1779 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1780                                  struct be_rx_compl_info *rxcp)
1781 {
1782         rxcp->pkt_size =
1783                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, pktsize, compl);
1784         rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtp, compl);
1785         rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, err, compl);
1786         rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, tcpf, compl);
1787         rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, udpf, compl);
1788         rxcp->ip_csum =
1789                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ipcksm, compl);
1790         rxcp->l4_csum =
1791                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, l4_cksm, compl);
1792         rxcp->ipv6 =
1793                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ip_version, compl);
1794         rxcp->num_rcvd =
1795                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, numfrags, compl);
1796         rxcp->pkt_type =
1797                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, cast_enc, compl);
1798         rxcp->rss_hash =
1799                 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, rsshash, compl);
1800         if (rxcp->vlanf) {
1801                 rxcp->qnq = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, qnq,
1802                                           compl);
1803                 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0,
1804                                                vlan_tag, compl);
1805         }
1806         rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, port, compl);
1807         rxcp->ip_frag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0,
1808                                       ip_frag, compl);
1809 }
1810
1811 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1812 {
1813         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1814         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1815         struct be_adapter *adapter = rxo->adapter;
1816
1817         /* For checking the valid bit it is Ok to use either definition as the
1818          * valid bit is at the same position in both v0 and v1 Rx compl */
1819         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1820                 return NULL;
1821
1822         rmb();
1823         be_dws_le_to_cpu(compl, sizeof(*compl));
1824
1825         if (adapter->be3_native)
1826                 be_parse_rx_compl_v1(compl, rxcp);
1827         else
1828                 be_parse_rx_compl_v0(compl, rxcp);
1829
1830         if (rxcp->ip_frag)
1831                 rxcp->l4_csum = 0;
1832
1833         if (rxcp->vlanf) {
1834                 /* In QNQ modes, if qnq bit is not set, then the packet was
1835                  * tagged only with the transparent outer vlan-tag and must
1836                  * not be treated as a vlan packet by host
1837                  */
1838                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1839                         rxcp->vlanf = 0;
1840
1841                 if (!lancer_chip(adapter))
1842                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1843
1844                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1845                     !test_bit(rxcp->vlan_tag, adapter->vids))
1846                         rxcp->vlanf = 0;
1847         }
1848
1849         /* As the compl has been parsed, reset it; we wont touch it again */
1850         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1851
1852         queue_tail_inc(&rxo->cq);
1853         return rxcp;
1854 }
1855
1856 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1857 {
1858         u32 order = get_order(size);
1859
1860         if (order > 0)
1861                 gfp |= __GFP_COMP;
1862         return  alloc_pages(gfp, order);
1863 }
1864
1865 /*
1866  * Allocate a page, split it to fragments of size rx_frag_size and post as
1867  * receive buffers to BE
1868  */
1869 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
1870 {
1871         struct be_adapter *adapter = rxo->adapter;
1872         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1873         struct be_queue_info *rxq = &rxo->q;
1874         struct page *pagep = NULL;
1875         struct device *dev = &adapter->pdev->dev;
1876         struct be_eth_rx_d *rxd;
1877         u64 page_dmaaddr = 0, frag_dmaaddr;
1878         u32 posted, page_offset = 0;
1879
1880         page_info = &rxo->page_info_tbl[rxq->head];
1881         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1882                 if (!pagep) {
1883                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1884                         if (unlikely(!pagep)) {
1885                                 rx_stats(rxo)->rx_post_fail++;
1886                                 break;
1887                         }
1888                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1889                                                     adapter->big_page_size,
1890                                                     DMA_FROM_DEVICE);
1891                         if (dma_mapping_error(dev, page_dmaaddr)) {
1892                                 put_page(pagep);
1893                                 pagep = NULL;
1894                                 rx_stats(rxo)->rx_post_fail++;
1895                                 break;
1896                         }
1897                         page_offset = 0;
1898                 } else {
1899                         get_page(pagep);
1900                         page_offset += rx_frag_size;
1901                 }
1902                 page_info->page_offset = page_offset;
1903                 page_info->page = pagep;
1904
1905                 rxd = queue_head_node(rxq);
1906                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1907                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1908                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1909
1910                 /* Any space left in the current big page for another frag? */
1911                 if ((page_offset + rx_frag_size + rx_frag_size) >
1912                                         adapter->big_page_size) {
1913                         pagep = NULL;
1914                         page_info->last_frag = true;
1915                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1916                 } else {
1917                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1918                 }
1919
1920                 prev_page_info = page_info;
1921                 queue_head_inc(rxq);
1922                 page_info = &rxo->page_info_tbl[rxq->head];
1923         }
1924
1925         /* Mark the last frag of a page when we break out of the above loop
1926          * with no more slots available in the RXQ
1927          */
1928         if (pagep) {
1929                 prev_page_info->last_frag = true;
1930                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1931         }
1932
1933         if (posted) {
1934                 atomic_add(posted, &rxq->used);
1935                 if (rxo->rx_post_starved)
1936                         rxo->rx_post_starved = false;
1937                 be_rxq_notify(adapter, rxq->id, posted);
1938         } else if (atomic_read(&rxq->used) == 0) {
1939                 /* Let be_worker replenish when memory is available */
1940                 rxo->rx_post_starved = true;
1941         }
1942 }
1943
1944 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1945 {
1946         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1947
1948         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1949                 return NULL;
1950
1951         rmb();
1952         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1953
1954         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1955
1956         queue_tail_inc(tx_cq);
1957         return txcp;
1958 }
1959
1960 static u16 be_tx_compl_process(struct be_adapter *adapter,
1961                                struct be_tx_obj *txo, u16 last_index)
1962 {
1963         struct be_queue_info *txq = &txo->q;
1964         struct be_eth_wrb *wrb;
1965         struct sk_buff **sent_skbs = txo->sent_skb_list;
1966         struct sk_buff *sent_skb;
1967         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1968         bool unmap_skb_hdr = true;
1969
1970         sent_skb = sent_skbs[txq->tail];
1971         BUG_ON(!sent_skb);
1972         sent_skbs[txq->tail] = NULL;
1973
1974         /* skip header wrb */
1975         queue_tail_inc(txq);
1976
1977         do {
1978                 cur_index = txq->tail;
1979                 wrb = queue_tail_node(txq);
1980                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1981                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1982                 unmap_skb_hdr = false;
1983
1984                 num_wrbs++;
1985                 queue_tail_inc(txq);
1986         } while (cur_index != last_index);
1987
1988         dev_kfree_skb_any(sent_skb);
1989         return num_wrbs;
1990 }
1991
1992 /* Return the number of events in the event queue */
1993 static inline int events_get(struct be_eq_obj *eqo)
1994 {
1995         struct be_eq_entry *eqe;
1996         int num = 0;
1997
1998         do {
1999                 eqe = queue_tail_node(&eqo->q);
2000                 if (eqe->evt == 0)
2001                         break;
2002
2003                 rmb();
2004                 eqe->evt = 0;
2005                 num++;
2006                 queue_tail_inc(&eqo->q);
2007         } while (true);
2008
2009         return num;
2010 }
2011
2012 /* Leaves the EQ is disarmed state */
2013 static void be_eq_clean(struct be_eq_obj *eqo)
2014 {
2015         int num = events_get(eqo);
2016
2017         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2018 }
2019
2020 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2021 {
2022         struct be_rx_page_info *page_info;
2023         struct be_queue_info *rxq = &rxo->q;
2024         struct be_queue_info *rx_cq = &rxo->cq;
2025         struct be_rx_compl_info *rxcp;
2026         struct be_adapter *adapter = rxo->adapter;
2027         int flush_wait = 0;
2028
2029         /* Consume pending rx completions.
2030          * Wait for the flush completion (identified by zero num_rcvd)
2031          * to arrive. Notify CQ even when there are no more CQ entries
2032          * for HW to flush partially coalesced CQ entries.
2033          * In Lancer, there is no need to wait for flush compl.
2034          */
2035         for (;;) {
2036                 rxcp = be_rx_compl_get(rxo);
2037                 if (!rxcp) {
2038                         if (lancer_chip(adapter))
2039                                 break;
2040
2041                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2042                                 dev_warn(&adapter->pdev->dev,
2043                                          "did not receive flush compl\n");
2044                                 break;
2045                         }
2046                         be_cq_notify(adapter, rx_cq->id, true, 0);
2047                         mdelay(1);
2048                 } else {
2049                         be_rx_compl_discard(rxo, rxcp);
2050                         be_cq_notify(adapter, rx_cq->id, false, 1);
2051                         if (rxcp->num_rcvd == 0)
2052                                 break;
2053                 }
2054         }
2055
2056         /* After cleanup, leave the CQ in unarmed state */
2057         be_cq_notify(adapter, rx_cq->id, false, 0);
2058
2059         /* Then free posted rx buffers that were not used */
2060         while (atomic_read(&rxq->used) > 0) {
2061                 page_info = get_rx_page_info(rxo);
2062                 put_page(page_info->page);
2063                 memset(page_info, 0, sizeof(*page_info));
2064         }
2065         BUG_ON(atomic_read(&rxq->used));
2066         rxq->tail = rxq->head = 0;
2067 }
2068
2069 static void be_tx_compl_clean(struct be_adapter *adapter)
2070 {
2071         struct be_tx_obj *txo;
2072         struct be_queue_info *txq;
2073         struct be_eth_tx_compl *txcp;
2074         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2075         struct sk_buff *sent_skb;
2076         bool dummy_wrb;
2077         int i, pending_txqs;
2078
2079         /* Stop polling for compls when HW has been silent for 10ms */
2080         do {
2081                 pending_txqs = adapter->num_tx_qs;
2082
2083                 for_all_tx_queues(adapter, txo, i) {
2084                         cmpl = 0;
2085                         num_wrbs = 0;
2086                         txq = &txo->q;
2087                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2088                                 end_idx =
2089                                         AMAP_GET_BITS(struct amap_eth_tx_compl,
2090                                                       wrb_index, txcp);
2091                                 num_wrbs += be_tx_compl_process(adapter, txo,
2092                                                                 end_idx);
2093                                 cmpl++;
2094                         }
2095                         if (cmpl) {
2096                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2097                                 atomic_sub(num_wrbs, &txq->used);
2098                                 timeo = 0;
2099                         }
2100                         if (atomic_read(&txq->used) == 0)
2101                                 pending_txqs--;
2102                 }
2103
2104                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2105                         break;
2106
2107                 mdelay(1);
2108         } while (true);
2109
2110         for_all_tx_queues(adapter, txo, i) {
2111                 txq = &txo->q;
2112                 if (atomic_read(&txq->used))
2113                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2114                                 atomic_read(&txq->used));
2115
2116                 /* free posted tx for which compls will never arrive */
2117                 while (atomic_read(&txq->used)) {
2118                         sent_skb = txo->sent_skb_list[txq->tail];
2119                         end_idx = txq->tail;
2120                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2121                                                    &dummy_wrb);
2122                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2123                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2124                         atomic_sub(num_wrbs, &txq->used);
2125                 }
2126         }
2127 }
2128
2129 static void be_evt_queues_destroy(struct be_adapter *adapter)
2130 {
2131         struct be_eq_obj *eqo;
2132         int i;
2133
2134         for_all_evt_queues(adapter, eqo, i) {
2135                 if (eqo->q.created) {
2136                         be_eq_clean(eqo);
2137                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2138                         napi_hash_del(&eqo->napi);
2139                         netif_napi_del(&eqo->napi);
2140                 }
2141                 be_queue_free(adapter, &eqo->q);
2142         }
2143 }
2144
2145 static int be_evt_queues_create(struct be_adapter *adapter)
2146 {
2147         struct be_queue_info *eq;
2148         struct be_eq_obj *eqo;
2149         struct be_aic_obj *aic;
2150         int i, rc;
2151
2152         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2153                                     adapter->cfg_num_qs);
2154
2155         for_all_evt_queues(adapter, eqo, i) {
2156                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2157                                BE_NAPI_WEIGHT);
2158                 napi_hash_add(&eqo->napi);
2159                 aic = &adapter->aic_obj[i];
2160                 eqo->adapter = adapter;
2161                 eqo->tx_budget = BE_TX_BUDGET;
2162                 eqo->idx = i;
2163                 aic->max_eqd = BE_MAX_EQD;
2164                 aic->enable = true;
2165
2166                 eq = &eqo->q;
2167                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2168                                     sizeof(struct be_eq_entry));
2169                 if (rc)
2170                         return rc;
2171
2172                 rc = be_cmd_eq_create(adapter, eqo);
2173                 if (rc)
2174                         return rc;
2175         }
2176         return 0;
2177 }
2178
2179 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2180 {
2181         struct be_queue_info *q;
2182
2183         q = &adapter->mcc_obj.q;
2184         if (q->created)
2185                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2186         be_queue_free(adapter, q);
2187
2188         q = &adapter->mcc_obj.cq;
2189         if (q->created)
2190                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2191         be_queue_free(adapter, q);
2192 }
2193
2194 /* Must be called only after TX qs are created as MCC shares TX EQ */
2195 static int be_mcc_queues_create(struct be_adapter *adapter)
2196 {
2197         struct be_queue_info *q, *cq;
2198
2199         cq = &adapter->mcc_obj.cq;
2200         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2201                            sizeof(struct be_mcc_compl)))
2202                 goto err;
2203
2204         /* Use the default EQ for MCC completions */
2205         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2206                 goto mcc_cq_free;
2207
2208         q = &adapter->mcc_obj.q;
2209         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2210                 goto mcc_cq_destroy;
2211
2212         if (be_cmd_mccq_create(adapter, q, cq))
2213                 goto mcc_q_free;
2214
2215         return 0;
2216
2217 mcc_q_free:
2218         be_queue_free(adapter, q);
2219 mcc_cq_destroy:
2220         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2221 mcc_cq_free:
2222         be_queue_free(adapter, cq);
2223 err:
2224         return -1;
2225 }
2226
2227 static void be_tx_queues_destroy(struct be_adapter *adapter)
2228 {
2229         struct be_queue_info *q;
2230         struct be_tx_obj *txo;
2231         u8 i;
2232
2233         for_all_tx_queues(adapter, txo, i) {
2234                 q = &txo->q;
2235                 if (q->created)
2236                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2237                 be_queue_free(adapter, q);
2238
2239                 q = &txo->cq;
2240                 if (q->created)
2241                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2242                 be_queue_free(adapter, q);
2243         }
2244 }
2245
2246 static int be_tx_qs_create(struct be_adapter *adapter)
2247 {
2248         struct be_queue_info *cq, *eq;
2249         struct be_tx_obj *txo;
2250         int status, i;
2251
2252         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2253
2254         for_all_tx_queues(adapter, txo, i) {
2255                 cq = &txo->cq;
2256                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2257                                         sizeof(struct be_eth_tx_compl));
2258                 if (status)
2259                         return status;
2260
2261                 u64_stats_init(&txo->stats.sync);
2262                 u64_stats_init(&txo->stats.sync_compl);
2263
2264                 /* If num_evt_qs is less than num_tx_qs, then more than
2265                  * one txq share an eq
2266                  */
2267                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2268                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2269                 if (status)
2270                         return status;
2271
2272                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2273                                         sizeof(struct be_eth_wrb));
2274                 if (status)
2275                         return status;
2276
2277                 status = be_cmd_txq_create(adapter, txo);
2278                 if (status)
2279                         return status;
2280         }
2281
2282         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2283                  adapter->num_tx_qs);
2284         return 0;
2285 }
2286
2287 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2288 {
2289         struct be_queue_info *q;
2290         struct be_rx_obj *rxo;
2291         int i;
2292
2293         for_all_rx_queues(adapter, rxo, i) {
2294                 q = &rxo->cq;
2295                 if (q->created)
2296                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2297                 be_queue_free(adapter, q);
2298         }
2299 }
2300
2301 static int be_rx_cqs_create(struct be_adapter *adapter)
2302 {
2303         struct be_queue_info *eq, *cq;
2304         struct be_rx_obj *rxo;
2305         int rc, i;
2306
2307         /* We can create as many RSS rings as there are EQs. */
2308         adapter->num_rx_qs = adapter->num_evt_qs;
2309
2310         /* We'll use RSS only if atleast 2 RSS rings are supported.
2311          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2312          */
2313         if (adapter->num_rx_qs > 1)
2314                 adapter->num_rx_qs++;
2315
2316         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2317         for_all_rx_queues(adapter, rxo, i) {
2318                 rxo->adapter = adapter;
2319                 cq = &rxo->cq;
2320                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2321                                     sizeof(struct be_eth_rx_compl));
2322                 if (rc)
2323                         return rc;
2324
2325                 u64_stats_init(&rxo->stats.sync);
2326                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2327                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2328                 if (rc)
2329                         return rc;
2330         }
2331
2332         dev_info(&adapter->pdev->dev,
2333                  "created %d RSS queue(s) and 1 default RX queue\n",
2334                  adapter->num_rx_qs - 1);
2335         return 0;
2336 }
2337
2338 static irqreturn_t be_intx(int irq, void *dev)
2339 {
2340         struct be_eq_obj *eqo = dev;
2341         struct be_adapter *adapter = eqo->adapter;
2342         int num_evts = 0;
2343
2344         /* IRQ is not expected when NAPI is scheduled as the EQ
2345          * will not be armed.
2346          * But, this can happen on Lancer INTx where it takes
2347          * a while to de-assert INTx or in BE2 where occasionaly
2348          * an interrupt may be raised even when EQ is unarmed.
2349          * If NAPI is already scheduled, then counting & notifying
2350          * events will orphan them.
2351          */
2352         if (napi_schedule_prep(&eqo->napi)) {
2353                 num_evts = events_get(eqo);
2354                 __napi_schedule(&eqo->napi);
2355                 if (num_evts)
2356                         eqo->spurious_intr = 0;
2357         }
2358         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2359
2360         /* Return IRQ_HANDLED only for the the first spurious intr
2361          * after a valid intr to stop the kernel from branding
2362          * this irq as a bad one!
2363          */
2364         if (num_evts || eqo->spurious_intr++ == 0)
2365                 return IRQ_HANDLED;
2366         else
2367                 return IRQ_NONE;
2368 }
2369
2370 static irqreturn_t be_msix(int irq, void *dev)
2371 {
2372         struct be_eq_obj *eqo = dev;
2373
2374         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2375         napi_schedule(&eqo->napi);
2376         return IRQ_HANDLED;
2377 }
2378
2379 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2380 {
2381         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2382 }
2383
2384 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2385                          int budget, int polling)
2386 {
2387         struct be_adapter *adapter = rxo->adapter;
2388         struct be_queue_info *rx_cq = &rxo->cq;
2389         struct be_rx_compl_info *rxcp;
2390         u32 work_done;
2391
2392         for (work_done = 0; work_done < budget; work_done++) {
2393                 rxcp = be_rx_compl_get(rxo);
2394                 if (!rxcp)
2395                         break;
2396
2397                 /* Is it a flush compl that has no data */
2398                 if (unlikely(rxcp->num_rcvd == 0))
2399                         goto loop_continue;
2400
2401                 /* Discard compl with partial DMA Lancer B0 */
2402                 if (unlikely(!rxcp->pkt_size)) {
2403                         be_rx_compl_discard(rxo, rxcp);
2404                         goto loop_continue;
2405                 }
2406
2407                 /* On BE drop pkts that arrive due to imperfect filtering in
2408                  * promiscuous mode on some skews
2409                  */
2410                 if (unlikely(rxcp->port != adapter->port_num &&
2411                              !lancer_chip(adapter))) {
2412                         be_rx_compl_discard(rxo, rxcp);
2413                         goto loop_continue;
2414                 }
2415
2416                 /* Don't do gro when we're busy_polling */
2417                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2418                         be_rx_compl_process_gro(rxo, napi, rxcp);
2419                 else
2420                         be_rx_compl_process(rxo, napi, rxcp);
2421
2422 loop_continue:
2423                 be_rx_stats_update(rxo, rxcp);
2424         }
2425
2426         if (work_done) {
2427                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2428
2429                 /* When an rx-obj gets into post_starved state, just
2430                  * let be_worker do the posting.
2431                  */
2432                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2433                     !rxo->rx_post_starved)
2434                         be_post_rx_frags(rxo, GFP_ATOMIC);
2435         }
2436
2437         return work_done;
2438 }
2439
2440 static bool be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2441                           int budget, int idx)
2442 {
2443         struct be_eth_tx_compl *txcp;
2444         int num_wrbs = 0, work_done;
2445
2446         for (work_done = 0; work_done < budget; work_done++) {
2447                 txcp = be_tx_compl_get(&txo->cq);
2448                 if (!txcp)
2449                         break;
2450                 num_wrbs += be_tx_compl_process(adapter, txo,
2451                                                 AMAP_GET_BITS(struct
2452                                                               amap_eth_tx_compl,
2453                                                               wrb_index, txcp));
2454         }
2455
2456         if (work_done) {
2457                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2458                 atomic_sub(num_wrbs, &txo->q.used);
2459
2460                 /* As Tx wrbs have been freed up, wake up netdev queue
2461                  * if it was stopped due to lack of tx wrbs.  */
2462                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2463                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2464                         netif_wake_subqueue(adapter->netdev, idx);
2465                 }
2466
2467                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2468                 tx_stats(txo)->tx_compl += work_done;
2469                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2470         }
2471         return (work_done < budget); /* Done */
2472 }
2473
2474 int be_poll(struct napi_struct *napi, int budget)
2475 {
2476         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2477         struct be_adapter *adapter = eqo->adapter;
2478         int max_work = 0, work, i, num_evts;
2479         struct be_rx_obj *rxo;
2480         bool tx_done;
2481
2482         num_evts = events_get(eqo);
2483
2484         /* Process all TXQs serviced by this EQ */
2485         for (i = eqo->idx; i < adapter->num_tx_qs; i += adapter->num_evt_qs) {
2486                 tx_done = be_process_tx(adapter, &adapter->tx_obj[i],
2487                                         eqo->tx_budget, i);
2488                 if (!tx_done)
2489                         max_work = budget;
2490         }
2491
2492         if (be_lock_napi(eqo)) {
2493                 /* This loop will iterate twice for EQ0 in which
2494                  * completions of the last RXQ (default one) are also processed
2495                  * For other EQs the loop iterates only once
2496                  */
2497                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2498                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2499                         max_work = max(work, max_work);
2500                 }
2501                 be_unlock_napi(eqo);
2502         } else {
2503                 max_work = budget;
2504         }
2505
2506         if (is_mcc_eqo(eqo))
2507                 be_process_mcc(adapter);
2508
2509         if (max_work < budget) {
2510                 napi_complete(napi);
2511                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2512         } else {
2513                 /* As we'll continue in polling mode, count and clear events */
2514                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2515         }
2516         return max_work;
2517 }
2518
2519 #ifdef CONFIG_NET_RX_BUSY_POLL
2520 static int be_busy_poll(struct napi_struct *napi)
2521 {
2522         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2523         struct be_adapter *adapter = eqo->adapter;
2524         struct be_rx_obj *rxo;
2525         int i, work = 0;
2526
2527         if (!be_lock_busy_poll(eqo))
2528                 return LL_FLUSH_BUSY;
2529
2530         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2531                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2532                 if (work)
2533                         break;
2534         }
2535
2536         be_unlock_busy_poll(eqo);
2537         return work;
2538 }
2539 #endif
2540
2541 void be_detect_error(struct be_adapter *adapter)
2542 {
2543         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2544         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2545         u32 i;
2546         bool error_detected = false;
2547         struct device *dev = &adapter->pdev->dev;
2548         struct net_device *netdev = adapter->netdev;
2549
2550         if (be_hw_error(adapter))
2551                 return;
2552
2553         if (lancer_chip(adapter)) {
2554                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2555                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2556                         sliport_err1 = ioread32(adapter->db +
2557                                                 SLIPORT_ERROR1_OFFSET);
2558                         sliport_err2 = ioread32(adapter->db +
2559                                                 SLIPORT_ERROR2_OFFSET);
2560                         adapter->hw_error = true;
2561                         /* Do not log error messages if its a FW reset */
2562                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2563                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2564                                 dev_info(dev, "Firmware update in progress\n");
2565                         } else {
2566                                 error_detected = true;
2567                                 dev_err(dev, "Error detected in the card\n");
2568                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2569                                         sliport_status);
2570                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2571                                         sliport_err1);
2572                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2573                                         sliport_err2);
2574                         }
2575                 }
2576         } else {
2577                 pci_read_config_dword(adapter->pdev,
2578                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2579                 pci_read_config_dword(adapter->pdev,
2580                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2581                 pci_read_config_dword(adapter->pdev,
2582                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2583                 pci_read_config_dword(adapter->pdev,
2584                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2585
2586                 ue_lo = (ue_lo & ~ue_lo_mask);
2587                 ue_hi = (ue_hi & ~ue_hi_mask);
2588
2589                 /* On certain platforms BE hardware can indicate spurious UEs.
2590                  * Allow HW to stop working completely in case of a real UE.
2591                  * Hence not setting the hw_error for UE detection.
2592                  */
2593
2594                 if (ue_lo || ue_hi) {
2595                         error_detected = true;
2596                         dev_err(dev,
2597                                 "Unrecoverable Error detected in the adapter");
2598                         dev_err(dev, "Please reboot server to recover");
2599                         if (skyhawk_chip(adapter))
2600                                 adapter->hw_error = true;
2601                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2602                                 if (ue_lo & 1)
2603                                         dev_err(dev, "UE: %s bit set\n",
2604                                                 ue_status_low_desc[i]);
2605                         }
2606                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2607                                 if (ue_hi & 1)
2608                                         dev_err(dev, "UE: %s bit set\n",
2609                                                 ue_status_hi_desc[i]);
2610                         }
2611                 }
2612         }
2613         if (error_detected)
2614                 netif_carrier_off(netdev);
2615 }
2616
2617 static void be_msix_disable(struct be_adapter *adapter)
2618 {
2619         if (msix_enabled(adapter)) {
2620                 pci_disable_msix(adapter->pdev);
2621                 adapter->num_msix_vec = 0;
2622                 adapter->num_msix_roce_vec = 0;
2623         }
2624 }
2625
2626 static int be_msix_enable(struct be_adapter *adapter)
2627 {
2628         int i, num_vec;
2629         struct device *dev = &adapter->pdev->dev;
2630
2631         /* If RoCE is supported, program the max number of NIC vectors that
2632          * may be configured via set-channels, along with vectors needed for
2633          * RoCe. Else, just program the number we'll use initially.
2634          */
2635         if (be_roce_supported(adapter))
2636                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2637                                 2 * num_online_cpus());
2638         else
2639                 num_vec = adapter->cfg_num_qs;
2640
2641         for (i = 0; i < num_vec; i++)
2642                 adapter->msix_entries[i].entry = i;
2643
2644         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2645                                         MIN_MSIX_VECTORS, num_vec);
2646         if (num_vec < 0)
2647                 goto fail;
2648
2649         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2650                 adapter->num_msix_roce_vec = num_vec / 2;
2651                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2652                          adapter->num_msix_roce_vec);
2653         }
2654
2655         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2656
2657         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2658                  adapter->num_msix_vec);
2659         return 0;
2660
2661 fail:
2662         dev_warn(dev, "MSIx enable failed\n");
2663
2664         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2665         if (!be_physfn(adapter))
2666                 return num_vec;
2667         return 0;
2668 }
2669
2670 static inline int be_msix_vec_get(struct be_adapter *adapter,
2671                                   struct be_eq_obj *eqo)
2672 {
2673         return adapter->msix_entries[eqo->msix_idx].vector;
2674 }
2675
2676 static int be_msix_register(struct be_adapter *adapter)
2677 {
2678         struct net_device *netdev = adapter->netdev;
2679         struct be_eq_obj *eqo;
2680         int status, i, vec;
2681
2682         for_all_evt_queues(adapter, eqo, i) {
2683                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2684                 vec = be_msix_vec_get(adapter, eqo);
2685                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2686                 if (status)
2687                         goto err_msix;
2688         }
2689
2690         return 0;
2691 err_msix:
2692         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2693                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2694         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2695                  status);
2696         be_msix_disable(adapter);
2697         return status;
2698 }
2699
2700 static int be_irq_register(struct be_adapter *adapter)
2701 {
2702         struct net_device *netdev = adapter->netdev;
2703         int status;
2704
2705         if (msix_enabled(adapter)) {
2706                 status = be_msix_register(adapter);
2707                 if (status == 0)
2708                         goto done;
2709                 /* INTx is not supported for VF */
2710                 if (!be_physfn(adapter))
2711                         return status;
2712         }
2713
2714         /* INTx: only the first EQ is used */
2715         netdev->irq = adapter->pdev->irq;
2716         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2717                              &adapter->eq_obj[0]);
2718         if (status) {
2719                 dev_err(&adapter->pdev->dev,
2720                         "INTx request IRQ failed - err %d\n", status);
2721                 return status;
2722         }
2723 done:
2724         adapter->isr_registered = true;
2725         return 0;
2726 }
2727
2728 static void be_irq_unregister(struct be_adapter *adapter)
2729 {
2730         struct net_device *netdev = adapter->netdev;
2731         struct be_eq_obj *eqo;
2732         int i;
2733
2734         if (!adapter->isr_registered)
2735                 return;
2736
2737         /* INTx */
2738         if (!msix_enabled(adapter)) {
2739                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2740                 goto done;
2741         }
2742
2743         /* MSIx */
2744         for_all_evt_queues(adapter, eqo, i)
2745                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2746
2747 done:
2748         adapter->isr_registered = false;
2749 }
2750
2751 static void be_rx_qs_destroy(struct be_adapter *adapter)
2752 {
2753         struct be_queue_info *q;
2754         struct be_rx_obj *rxo;
2755         int i;
2756
2757         for_all_rx_queues(adapter, rxo, i) {
2758                 q = &rxo->q;
2759                 if (q->created) {
2760                         be_cmd_rxq_destroy(adapter, q);
2761                         be_rx_cq_clean(rxo);
2762                 }
2763                 be_queue_free(adapter, q);
2764         }
2765 }
2766
2767 static int be_close(struct net_device *netdev)
2768 {
2769         struct be_adapter *adapter = netdev_priv(netdev);
2770         struct be_eq_obj *eqo;
2771         int i;
2772
2773         /* This protection is needed as be_close() may be called even when the
2774          * adapter is in cleared state (after eeh perm failure)
2775          */
2776         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2777                 return 0;
2778
2779         be_roce_dev_close(adapter);
2780
2781         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2782                 for_all_evt_queues(adapter, eqo, i) {
2783                         napi_disable(&eqo->napi);
2784                         be_disable_busy_poll(eqo);
2785                 }
2786                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2787         }
2788
2789         be_async_mcc_disable(adapter);
2790
2791         /* Wait for all pending tx completions to arrive so that
2792          * all tx skbs are freed.
2793          */
2794         netif_tx_disable(netdev);
2795         be_tx_compl_clean(adapter);
2796
2797         be_rx_qs_destroy(adapter);
2798
2799         for (i = 1; i < (adapter->uc_macs + 1); i++)
2800                 be_cmd_pmac_del(adapter, adapter->if_handle,
2801                                 adapter->pmac_id[i], 0);
2802         adapter->uc_macs = 0;
2803
2804         for_all_evt_queues(adapter, eqo, i) {
2805                 if (msix_enabled(adapter))
2806                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2807                 else
2808                         synchronize_irq(netdev->irq);
2809                 be_eq_clean(eqo);
2810         }
2811
2812         be_irq_unregister(adapter);
2813
2814         return 0;
2815 }
2816
2817 static int be_rx_qs_create(struct be_adapter *adapter)
2818 {
2819         struct be_rx_obj *rxo;
2820         int rc, i, j;
2821         u8 rss_hkey[RSS_HASH_KEY_LEN];
2822         struct rss_info *rss = &adapter->rss_info;
2823
2824         for_all_rx_queues(adapter, rxo, i) {
2825                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2826                                     sizeof(struct be_eth_rx_d));
2827                 if (rc)
2828                         return rc;
2829         }
2830
2831         /* The FW would like the default RXQ to be created first */
2832         rxo = default_rxo(adapter);
2833         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2834                                adapter->if_handle, false, &rxo->rss_id);
2835         if (rc)
2836                 return rc;
2837
2838         for_all_rss_queues(adapter, rxo, i) {
2839                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2840                                        rx_frag_size, adapter->if_handle,
2841                                        true, &rxo->rss_id);
2842                 if (rc)
2843                         return rc;
2844         }
2845
2846         if (be_multi_rxq(adapter)) {
2847                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2848                         j += adapter->num_rx_qs - 1) {
2849                         for_all_rss_queues(adapter, rxo, i) {
2850                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2851                                         break;
2852                                 rss->rsstable[j + i] = rxo->rss_id;
2853                                 rss->rss_queue[j + i] = i;
2854                         }
2855                 }
2856                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2857                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2858
2859                 if (!BEx_chip(adapter))
2860                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2861                                 RSS_ENABLE_UDP_IPV6;
2862         } else {
2863                 /* Disable RSS, if only default RX Q is created */
2864                 rss->rss_flags = RSS_ENABLE_NONE;
2865         }
2866
2867         get_random_bytes(rss_hkey, RSS_HASH_KEY_LEN);
2868         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2869                                128, rss_hkey);
2870         if (rc) {
2871                 rss->rss_flags = RSS_ENABLE_NONE;
2872                 return rc;
2873         }
2874
2875         memcpy(rss->rss_hkey, rss_hkey, RSS_HASH_KEY_LEN);
2876
2877         /* First time posting */
2878         for_all_rx_queues(adapter, rxo, i)
2879                 be_post_rx_frags(rxo, GFP_KERNEL);
2880         return 0;
2881 }
2882
2883 static int be_open(struct net_device *netdev)
2884 {
2885         struct be_adapter *adapter = netdev_priv(netdev);
2886         struct be_eq_obj *eqo;
2887         struct be_rx_obj *rxo;
2888         struct be_tx_obj *txo;
2889         u8 link_status;
2890         int status, i;
2891
2892         status = be_rx_qs_create(adapter);
2893         if (status)
2894                 goto err;
2895
2896         status = be_irq_register(adapter);
2897         if (status)
2898                 goto err;
2899
2900         for_all_rx_queues(adapter, rxo, i)
2901                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2902
2903         for_all_tx_queues(adapter, txo, i)
2904                 be_cq_notify(adapter, txo->cq.id, true, 0);
2905
2906         be_async_mcc_enable(adapter);
2907
2908         for_all_evt_queues(adapter, eqo, i) {
2909                 napi_enable(&eqo->napi);
2910                 be_enable_busy_poll(eqo);
2911                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2912         }
2913         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2914
2915         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2916         if (!status)
2917                 be_link_status_update(adapter, link_status);
2918
2919         netif_tx_start_all_queues(netdev);
2920         be_roce_dev_open(adapter);
2921
2922 #ifdef CONFIG_BE2NET_VXLAN
2923         if (skyhawk_chip(adapter))
2924                 vxlan_get_rx_port(netdev);
2925 #endif
2926
2927         return 0;
2928 err:
2929         be_close(adapter->netdev);
2930         return -EIO;
2931 }
2932
2933 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2934 {
2935         struct be_dma_mem cmd;
2936         int status = 0;
2937         u8 mac[ETH_ALEN];
2938
2939         memset(mac, 0, ETH_ALEN);
2940
2941         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2942         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2943                                      GFP_KERNEL);
2944         if (!cmd.va)
2945                 return -ENOMEM;
2946
2947         if (enable) {
2948                 status = pci_write_config_dword(adapter->pdev,
2949                                                 PCICFG_PM_CONTROL_OFFSET,
2950                                                 PCICFG_PM_CONTROL_MASK);
2951                 if (status) {
2952                         dev_err(&adapter->pdev->dev,
2953                                 "Could not enable Wake-on-lan\n");
2954                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2955                                           cmd.dma);
2956                         return status;
2957                 }
2958                 status = be_cmd_enable_magic_wol(adapter,
2959                                                  adapter->netdev->dev_addr,
2960                                                  &cmd);
2961                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2962                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2963         } else {
2964                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2965                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2966                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2967         }
2968
2969         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
2970         return status;
2971 }
2972
2973 /*
2974  * Generate a seed MAC address from the PF MAC Address using jhash.
2975  * MAC Address for VFs are assigned incrementally starting from the seed.
2976  * These addresses are programmed in the ASIC by the PF and the VF driver
2977  * queries for the MAC address during its probe.
2978  */
2979 static int be_vf_eth_addr_config(struct be_adapter *adapter)
2980 {
2981         u32 vf;
2982         int status = 0;
2983         u8 mac[ETH_ALEN];
2984         struct be_vf_cfg *vf_cfg;
2985
2986         be_vf_eth_addr_generate(adapter, mac);
2987
2988         for_all_vfs(adapter, vf_cfg, vf) {
2989                 if (BEx_chip(adapter))
2990                         status = be_cmd_pmac_add(adapter, mac,
2991                                                  vf_cfg->if_handle,
2992                                                  &vf_cfg->pmac_id, vf + 1);
2993                 else
2994                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
2995                                                 vf + 1);
2996
2997                 if (status)
2998                         dev_err(&adapter->pdev->dev,
2999                                 "Mac address assignment failed for VF %d\n",
3000                                 vf);
3001                 else
3002                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3003
3004                 mac[5] += 1;
3005         }
3006         return status;
3007 }
3008
3009 static int be_vfs_mac_query(struct be_adapter *adapter)
3010 {
3011         int status, vf;
3012         u8 mac[ETH_ALEN];
3013         struct be_vf_cfg *vf_cfg;
3014
3015         for_all_vfs(adapter, vf_cfg, vf) {
3016                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3017                                                mac, vf_cfg->if_handle,
3018                                                false, vf+1);
3019                 if (status)
3020                         return status;
3021                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3022         }
3023         return 0;
3024 }
3025
3026 static void be_vf_clear(struct be_adapter *adapter)
3027 {
3028         struct be_vf_cfg *vf_cfg;
3029         u32 vf;
3030
3031         if (pci_vfs_assigned(adapter->pdev)) {
3032                 dev_warn(&adapter->pdev->dev,
3033                          "VFs are assigned to VMs: not disabling VFs\n");
3034                 goto done;
3035         }
3036
3037         pci_disable_sriov(adapter->pdev);
3038
3039         for_all_vfs(adapter, vf_cfg, vf) {
3040                 if (BEx_chip(adapter))
3041                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3042                                         vf_cfg->pmac_id, vf + 1);
3043                 else
3044                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3045                                        vf + 1);
3046
3047                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3048         }
3049 done:
3050         kfree(adapter->vf_cfg);
3051         adapter->num_vfs = 0;
3052 }
3053
3054 static void be_clear_queues(struct be_adapter *adapter)
3055 {
3056         be_mcc_queues_destroy(adapter);
3057         be_rx_cqs_destroy(adapter);
3058         be_tx_queues_destroy(adapter);
3059         be_evt_queues_destroy(adapter);
3060 }
3061
3062 static void be_cancel_worker(struct be_adapter *adapter)
3063 {
3064         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3065                 cancel_delayed_work_sync(&adapter->work);
3066                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3067         }
3068 }
3069
3070 static void be_mac_clear(struct be_adapter *adapter)
3071 {
3072         int i;
3073
3074         if (adapter->pmac_id) {
3075                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3076                         be_cmd_pmac_del(adapter, adapter->if_handle,
3077                                         adapter->pmac_id[i], 0);
3078                 adapter->uc_macs = 0;
3079
3080                 kfree(adapter->pmac_id);
3081                 adapter->pmac_id = NULL;
3082         }
3083 }
3084
3085 #ifdef CONFIG_BE2NET_VXLAN
3086 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3087 {
3088         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3089                 be_cmd_manage_iface(adapter, adapter->if_handle,
3090                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3091
3092         if (adapter->vxlan_port)
3093                 be_cmd_set_vxlan_port(adapter, 0);
3094
3095         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3096         adapter->vxlan_port = 0;
3097 }
3098 #endif
3099
3100 static int be_clear(struct be_adapter *adapter)
3101 {
3102         be_cancel_worker(adapter);
3103
3104         if (sriov_enabled(adapter))
3105                 be_vf_clear(adapter);
3106
3107         /* Re-configure FW to distribute resources evenly across max-supported
3108          * number of VFs, only when VFs are not already enabled.
3109          */
3110         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3111                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3112                                         pci_sriov_get_totalvfs(adapter->pdev));
3113
3114 #ifdef CONFIG_BE2NET_VXLAN
3115         be_disable_vxlan_offloads(adapter);
3116 #endif
3117         /* delete the primary mac along with the uc-mac list */
3118         be_mac_clear(adapter);
3119
3120         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3121
3122         be_clear_queues(adapter);
3123
3124         be_msix_disable(adapter);
3125         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3126         return 0;
3127 }
3128
3129 static int be_vfs_if_create(struct be_adapter *adapter)
3130 {
3131         struct be_resources res = {0};
3132         struct be_vf_cfg *vf_cfg;
3133         u32 cap_flags, en_flags, vf;
3134         int status = 0;
3135
3136         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3137                     BE_IF_FLAGS_MULTICAST;
3138
3139         for_all_vfs(adapter, vf_cfg, vf) {
3140                 if (!BE3_chip(adapter)) {
3141                         status = be_cmd_get_profile_config(adapter, &res,
3142                                                            vf + 1);
3143                         if (!status)
3144                                 cap_flags = res.if_cap_flags;
3145                 }
3146
3147                 /* If a FW profile exists, then cap_flags are updated */
3148                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3149                                         BE_IF_FLAGS_BROADCAST |
3150                                         BE_IF_FLAGS_MULTICAST);
3151                 status =
3152                     be_cmd_if_create(adapter, cap_flags, en_flags,
3153                                      &vf_cfg->if_handle, vf + 1);
3154                 if (status)
3155                         goto err;
3156         }
3157 err:
3158         return status;
3159 }
3160
3161 static int be_vf_setup_init(struct be_adapter *adapter)
3162 {
3163         struct be_vf_cfg *vf_cfg;
3164         int vf;
3165
3166         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3167                                   GFP_KERNEL);
3168         if (!adapter->vf_cfg)
3169                 return -ENOMEM;
3170
3171         for_all_vfs(adapter, vf_cfg, vf) {
3172                 vf_cfg->if_handle = -1;
3173                 vf_cfg->pmac_id = -1;
3174         }
3175         return 0;
3176 }
3177
3178 static int be_vf_setup(struct be_adapter *adapter)
3179 {
3180         struct device *dev = &adapter->pdev->dev;
3181         struct be_vf_cfg *vf_cfg;
3182         int status, old_vfs, vf;
3183         u32 privileges;
3184
3185         old_vfs = pci_num_vf(adapter->pdev);
3186
3187         status = be_vf_setup_init(adapter);
3188         if (status)
3189                 goto err;
3190
3191         if (old_vfs) {
3192                 for_all_vfs(adapter, vf_cfg, vf) {
3193                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3194                         if (status)
3195                                 goto err;
3196                 }
3197
3198                 status = be_vfs_mac_query(adapter);
3199                 if (status)
3200                         goto err;
3201         } else {
3202                 status = be_vfs_if_create(adapter);
3203                 if (status)
3204                         goto err;
3205
3206                 status = be_vf_eth_addr_config(adapter);
3207                 if (status)
3208                         goto err;
3209         }
3210
3211         for_all_vfs(adapter, vf_cfg, vf) {
3212                 /* Allow VFs to programs MAC/VLAN filters */
3213                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3214                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3215                         status = be_cmd_set_fn_privileges(adapter,
3216                                                           privileges |
3217                                                           BE_PRIV_FILTMGMT,
3218                                                           vf + 1);
3219                         if (!status)
3220                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3221                                          vf);
3222                 }
3223
3224                 /* Allow full available bandwidth */
3225                 if (!old_vfs)
3226                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3227
3228                 if (!old_vfs) {
3229                         be_cmd_enable_vf(adapter, vf + 1);
3230                         be_cmd_set_logical_link_config(adapter,
3231                                                        IFLA_VF_LINK_STATE_AUTO,
3232                                                        vf+1);
3233                 }
3234         }
3235
3236         if (!old_vfs) {
3237                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3238                 if (status) {
3239                         dev_err(dev, "SRIOV enable failed\n");
3240                         adapter->num_vfs = 0;
3241                         goto err;
3242                 }
3243         }
3244         return 0;
3245 err:
3246         dev_err(dev, "VF setup failed\n");
3247         be_vf_clear(adapter);
3248         return status;
3249 }
3250
3251 /* Converting function_mode bits on BE3 to SH mc_type enums */
3252
3253 static u8 be_convert_mc_type(u32 function_mode)
3254 {
3255         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3256                 return vNIC1;
3257         else if (function_mode & QNQ_MODE)
3258                 return FLEX10;
3259         else if (function_mode & VNIC_MODE)
3260                 return vNIC2;
3261         else if (function_mode & UMC_ENABLED)
3262                 return UMC;
3263         else
3264                 return MC_NONE;
3265 }
3266
3267 /* On BE2/BE3 FW does not suggest the supported limits */
3268 static void BEx_get_resources(struct be_adapter *adapter,
3269                               struct be_resources *res)
3270 {
3271         bool use_sriov = adapter->num_vfs ? 1 : 0;
3272
3273         if (be_physfn(adapter))
3274                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3275         else
3276                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3277
3278         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3279
3280         if (be_is_mc(adapter)) {
3281                 /* Assuming that there are 4 channels per port,
3282                  * when multi-channel is enabled
3283                  */
3284                 if (be_is_qnq_mode(adapter))
3285                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3286                 else
3287                         /* In a non-qnq multichannel mode, the pvid
3288                          * takes up one vlan entry
3289                          */
3290                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3291         } else {
3292                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3293         }
3294
3295         res->max_mcast_mac = BE_MAX_MC;
3296
3297         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3298          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3299          *    *only* if it is RSS-capable.
3300          */
3301         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3302             !be_physfn(adapter) || (be_is_mc(adapter) &&
3303             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS)))
3304                 res->max_tx_qs = 1;
3305         else
3306                 res->max_tx_qs = BE3_MAX_TX_QS;
3307
3308         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3309             !use_sriov && be_physfn(adapter))
3310                 res->max_rss_qs = (adapter->be3_native) ?
3311                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3312         res->max_rx_qs = res->max_rss_qs + 1;
3313
3314         if (be_physfn(adapter))
3315                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3316                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3317         else
3318                 res->max_evt_qs = 1;
3319
3320         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3321         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3322                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3323 }
3324
3325 static void be_setup_init(struct be_adapter *adapter)
3326 {
3327         adapter->vlan_prio_bmap = 0xff;
3328         adapter->phy.link_speed = -1;
3329         adapter->if_handle = -1;
3330         adapter->be3_native = false;
3331         adapter->promiscuous = false;
3332         if (be_physfn(adapter))
3333                 adapter->cmd_privileges = MAX_PRIVILEGES;
3334         else
3335                 adapter->cmd_privileges = MIN_PRIVILEGES;
3336 }
3337
3338 static int be_get_sriov_config(struct be_adapter *adapter)
3339 {
3340         struct device *dev = &adapter->pdev->dev;
3341         struct be_resources res = {0};
3342         int status, max_vfs, old_vfs;
3343
3344         status = be_cmd_get_profile_config(adapter, &res, 0);
3345         if (status)
3346                 return status;
3347
3348         adapter->pool_res = res;
3349
3350         /* Some old versions of BE3 FW don't report max_vfs value */
3351         if (BE3_chip(adapter) && !res.max_vfs) {
3352                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3353                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3354         }
3355
3356         adapter->pool_res.max_vfs = res.max_vfs;
3357         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3358
3359         if (!be_max_vfs(adapter)) {
3360                 if (num_vfs)
3361                         dev_warn(dev, "device doesn't support SRIOV\n");
3362                 adapter->num_vfs = 0;
3363                 return 0;
3364         }
3365
3366         /* validate num_vfs module param */
3367         old_vfs = pci_num_vf(adapter->pdev);
3368         if (old_vfs) {
3369                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3370                 if (old_vfs != num_vfs)
3371                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3372                 adapter->num_vfs = old_vfs;
3373         } else {
3374                 if (num_vfs > be_max_vfs(adapter)) {
3375                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3376                                  num_vfs);
3377                         dev_info(dev, "Limiting to %d VFs\n",
3378                                  be_max_vfs(adapter));
3379                 }
3380                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3381         }
3382
3383         return 0;
3384 }
3385
3386 static int be_get_resources(struct be_adapter *adapter)
3387 {
3388         struct device *dev = &adapter->pdev->dev;
3389         struct be_resources res = {0};
3390         int status;
3391
3392         if (BEx_chip(adapter)) {
3393                 BEx_get_resources(adapter, &res);
3394                 adapter->res = res;
3395         }
3396
3397         /* For Lancer, SH etc read per-function resource limits from FW.
3398          * GET_FUNC_CONFIG returns per function guaranteed limits.
3399          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3400          */
3401         if (!BEx_chip(adapter)) {
3402                 status = be_cmd_get_func_config(adapter, &res);
3403                 if (status)
3404                         return status;
3405
3406                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3407                 if (be_roce_supported(adapter))
3408                         res.max_evt_qs /= 2;
3409                 adapter->res = res;
3410
3411                 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3412                          be_max_txqs(adapter), be_max_rxqs(adapter),
3413                          be_max_rss(adapter), be_max_eqs(adapter),
3414                          be_max_vfs(adapter));
3415                 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3416                          be_max_uc(adapter), be_max_mc(adapter),
3417                          be_max_vlans(adapter));
3418         }
3419
3420         return 0;
3421 }
3422
3423 static int be_get_config(struct be_adapter *adapter)
3424 {
3425         u16 profile_id;
3426         int status;
3427
3428         status = be_cmd_query_fw_cfg(adapter);
3429         if (status)
3430                 return status;
3431
3432          if (be_physfn(adapter)) {
3433                 status = be_cmd_get_active_profile(adapter, &profile_id);
3434                 if (!status)
3435                         dev_info(&adapter->pdev->dev,
3436                                  "Using profile 0x%x\n", profile_id);
3437         }
3438
3439         if (!BE2_chip(adapter) && be_physfn(adapter)) {
3440                 status = be_get_sriov_config(adapter);
3441                 if (status)
3442                         return status;
3443
3444                 /* When the HW is in SRIOV capable configuration, the PF-pool
3445                  * resources are equally distributed across the max-number of
3446                  * VFs. The user may request only a subset of the max-vfs to be
3447                  * enabled. Based on num_vfs, redistribute the resources across
3448                  * num_vfs so that each VF will have access to more number of
3449                  * resources. This facility is not available in BE3 FW.
3450                  * Also, this is done by FW in Lancer chip.
3451                  */
3452                 if (!pci_num_vf(adapter->pdev)) {
3453                         status = be_cmd_set_sriov_config(adapter,
3454                                                          adapter->pool_res,
3455                                                          adapter->num_vfs);
3456                         if (status)
3457                                 return status;
3458                 }
3459         }
3460
3461         status = be_get_resources(adapter);
3462         if (status)
3463                 return status;
3464
3465         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3466                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3467         if (!adapter->pmac_id)
3468                 return -ENOMEM;
3469
3470         /* Sanitize cfg_num_qs based on HW and platform limits */
3471         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3472
3473         return 0;
3474 }
3475
3476 static int be_mac_setup(struct be_adapter *adapter)
3477 {
3478         u8 mac[ETH_ALEN];
3479         int status;
3480
3481         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3482                 status = be_cmd_get_perm_mac(adapter, mac);
3483                 if (status)
3484                         return status;
3485
3486                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3487                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3488         } else {
3489                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3490                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3491         }
3492
3493         /* For BE3-R VFs, the PF programs the initial MAC address */
3494         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3495                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3496                                 &adapter->pmac_id[0], 0);
3497         return 0;
3498 }
3499
3500 static void be_schedule_worker(struct be_adapter *adapter)
3501 {
3502         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3503         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3504 }
3505
3506 static int be_setup_queues(struct be_adapter *adapter)
3507 {
3508         struct net_device *netdev = adapter->netdev;
3509         int status;
3510
3511         status = be_evt_queues_create(adapter);
3512         if (status)
3513                 goto err;
3514
3515         status = be_tx_qs_create(adapter);
3516         if (status)
3517                 goto err;
3518
3519         status = be_rx_cqs_create(adapter);
3520         if (status)
3521                 goto err;
3522
3523         status = be_mcc_queues_create(adapter);
3524         if (status)
3525                 goto err;
3526
3527         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3528         if (status)
3529                 goto err;
3530
3531         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3532         if (status)
3533                 goto err;
3534
3535         return 0;
3536 err:
3537         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3538         return status;
3539 }
3540
3541 int be_update_queues(struct be_adapter *adapter)
3542 {
3543         struct net_device *netdev = adapter->netdev;
3544         int status;
3545
3546         if (netif_running(netdev))
3547                 be_close(netdev);
3548
3549         be_cancel_worker(adapter);
3550
3551         /* If any vectors have been shared with RoCE we cannot re-program
3552          * the MSIx table.
3553          */
3554         if (!adapter->num_msix_roce_vec)
3555                 be_msix_disable(adapter);
3556
3557         be_clear_queues(adapter);
3558
3559         if (!msix_enabled(adapter)) {
3560                 status = be_msix_enable(adapter);
3561                 if (status)
3562                         return status;
3563         }
3564
3565         status = be_setup_queues(adapter);
3566         if (status)
3567                 return status;
3568
3569         be_schedule_worker(adapter);
3570
3571         if (netif_running(netdev))
3572                 status = be_open(netdev);
3573
3574         return status;
3575 }
3576
3577 static int be_setup(struct be_adapter *adapter)
3578 {
3579         struct device *dev = &adapter->pdev->dev;
3580         u32 tx_fc, rx_fc, en_flags;
3581         int status;
3582
3583         be_setup_init(adapter);
3584
3585         if (!lancer_chip(adapter))
3586                 be_cmd_req_native_mode(adapter);
3587
3588         status = be_get_config(adapter);
3589         if (status)
3590                 goto err;
3591
3592         status = be_msix_enable(adapter);
3593         if (status)
3594                 goto err;
3595
3596         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3597                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3598         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3599                 en_flags |= BE_IF_FLAGS_RSS;
3600         en_flags = en_flags & be_if_cap_flags(adapter);
3601         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3602                                   &adapter->if_handle, 0);
3603         if (status)
3604                 goto err;
3605
3606         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3607         rtnl_lock();
3608         status = be_setup_queues(adapter);
3609         rtnl_unlock();
3610         if (status)
3611                 goto err;
3612
3613         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3614
3615         status = be_mac_setup(adapter);
3616         if (status)
3617                 goto err;
3618
3619         be_cmd_get_fw_ver(adapter);
3620
3621         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3622                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work.",
3623                         adapter->fw_ver);
3624                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3625         }
3626
3627         if (adapter->vlans_added)
3628                 be_vid_config(adapter);
3629
3630         be_set_rx_mode(adapter->netdev);
3631
3632         be_cmd_get_acpi_wol_cap(adapter);
3633
3634         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3635
3636         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3637                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3638                                         adapter->rx_fc);
3639
3640         if (be_physfn(adapter))
3641                 be_cmd_set_logical_link_config(adapter,
3642                                                IFLA_VF_LINK_STATE_AUTO, 0);
3643
3644         if (adapter->num_vfs)
3645                 be_vf_setup(adapter);
3646
3647         status = be_cmd_get_phy_info(adapter);
3648         if (!status && be_pause_supported(adapter))
3649                 adapter->phy.fc_autoneg = 1;
3650
3651         be_schedule_worker(adapter);
3652         adapter->flags |= BE_FLAGS_SETUP_DONE;
3653         return 0;
3654 err:
3655         be_clear(adapter);
3656         return status;
3657 }
3658
3659 #ifdef CONFIG_NET_POLL_CONTROLLER
3660 static void be_netpoll(struct net_device *netdev)
3661 {
3662         struct be_adapter *adapter = netdev_priv(netdev);
3663         struct be_eq_obj *eqo;
3664         int i;
3665
3666         for_all_evt_queues(adapter, eqo, i) {
3667                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3668                 napi_schedule(&eqo->napi);
3669         }
3670
3671         return;
3672 }
3673 #endif
3674
3675 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3676
3677 static bool phy_flashing_required(struct be_adapter *adapter)
3678 {
3679         return (adapter->phy.phy_type == TN_8022 &&
3680                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3681 }
3682
3683 static bool is_comp_in_ufi(struct be_adapter *adapter,
3684                            struct flash_section_info *fsec, int type)
3685 {
3686         int i = 0, img_type = 0;
3687         struct flash_section_info_g2 *fsec_g2 = NULL;
3688
3689         if (BE2_chip(adapter))
3690                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3691
3692         for (i = 0; i < MAX_FLASH_COMP; i++) {
3693                 if (fsec_g2)
3694                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3695                 else
3696                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3697
3698                 if (img_type == type)
3699                         return true;
3700         }
3701         return false;
3702
3703 }
3704
3705 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3706                                                 int header_size,
3707                                                 const struct firmware *fw)
3708 {
3709         struct flash_section_info *fsec = NULL;
3710         const u8 *p = fw->data;
3711
3712         p += header_size;
3713         while (p < (fw->data + fw->size)) {
3714                 fsec = (struct flash_section_info *)p;
3715                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3716                         return fsec;
3717                 p += 32;
3718         }
3719         return NULL;
3720 }
3721
3722 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3723                               u32 img_offset, u32 img_size, int hdr_size,
3724                               u16 img_optype, bool *crc_match)
3725 {
3726         u32 crc_offset;
3727         int status;
3728         u8 crc[4];
3729
3730         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3731         if (status)
3732                 return status;
3733
3734         crc_offset = hdr_size + img_offset + img_size - 4;
3735
3736         /* Skip flashing, if crc of flashed region matches */
3737         if (!memcmp(crc, p + crc_offset, 4))
3738                 *crc_match = true;
3739         else
3740                 *crc_match = false;
3741
3742         return status;
3743 }
3744
3745 static int be_flash(struct be_adapter *adapter, const u8 *img,
3746                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3747 {
3748         struct be_cmd_write_flashrom *req = flash_cmd->va;
3749         u32 total_bytes, flash_op, num_bytes;
3750         int status;
3751
3752         total_bytes = img_size;
3753         while (total_bytes) {
3754                 num_bytes = min_t(u32, 32*1024, total_bytes);
3755
3756                 total_bytes -= num_bytes;
3757
3758                 if (!total_bytes) {
3759                         if (optype == OPTYPE_PHY_FW)
3760                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3761                         else
3762                                 flash_op = FLASHROM_OPER_FLASH;
3763                 } else {
3764                         if (optype == OPTYPE_PHY_FW)
3765                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3766                         else
3767                                 flash_op = FLASHROM_OPER_SAVE;
3768                 }
3769
3770                 memcpy(req->data_buf, img, num_bytes);
3771                 img += num_bytes;
3772                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3773                                                flash_op, num_bytes);
3774                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3775                     optype == OPTYPE_PHY_FW)
3776                         break;
3777                 else if (status)
3778                         return status;
3779         }
3780         return 0;
3781 }
3782
3783 /* For BE2, BE3 and BE3-R */
3784 static int be_flash_BEx(struct be_adapter *adapter,
3785                         const struct firmware *fw,
3786                         struct be_dma_mem *flash_cmd, int num_of_images)
3787 {
3788         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3789         struct device *dev = &adapter->pdev->dev;
3790         struct flash_section_info *fsec = NULL;
3791         int status, i, filehdr_size, num_comp;
3792         const struct flash_comp *pflashcomp;
3793         bool crc_match;
3794         const u8 *p;
3795
3796         struct flash_comp gen3_flash_types[] = {
3797                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3798                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3799                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3800                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3801                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3802                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3803                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3804                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3805                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3806                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3807                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3808                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3809                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3810                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3811                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3812                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3813                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3814                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3815                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3816                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3817         };
3818
3819         struct flash_comp gen2_flash_types[] = {
3820                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3821                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3822                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3823                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3824                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3825                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3826                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3827                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3828                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3829                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3830                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3831                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3832                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3833                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3834                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3835                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3836         };
3837
3838         if (BE3_chip(adapter)) {
3839                 pflashcomp = gen3_flash_types;
3840                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3841                 num_comp = ARRAY_SIZE(gen3_flash_types);
3842         } else {
3843                 pflashcomp = gen2_flash_types;
3844                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3845                 num_comp = ARRAY_SIZE(gen2_flash_types);
3846         }
3847
3848         /* Get flash section info*/
3849         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3850         if (!fsec) {
3851                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3852                 return -1;
3853         }
3854         for (i = 0; i < num_comp; i++) {
3855                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3856                         continue;
3857
3858                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3859                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3860                         continue;
3861
3862                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3863                     !phy_flashing_required(adapter))
3864                                 continue;
3865
3866                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3867                         status = be_check_flash_crc(adapter, fw->data,
3868                                                     pflashcomp[i].offset,
3869                                                     pflashcomp[i].size,
3870                                                     filehdr_size +
3871                                                     img_hdrs_size,
3872                                                     OPTYPE_REDBOOT, &crc_match);
3873                         if (status) {
3874                                 dev_err(dev,
3875                                         "Could not get CRC for 0x%x region\n",
3876                                         pflashcomp[i].optype);
3877                                 continue;
3878                         }
3879
3880                         if (crc_match)
3881                                 continue;
3882                 }
3883
3884                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3885                         img_hdrs_size;
3886                 if (p + pflashcomp[i].size > fw->data + fw->size)
3887                         return -1;
3888
3889                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3890                                   pflashcomp[i].size);
3891                 if (status) {
3892                         dev_err(dev, "Flashing section type 0x%x failed\n",
3893                                 pflashcomp[i].img_type);
3894                         return status;
3895                 }
3896         }
3897         return 0;
3898 }
3899
3900 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3901 {
3902         u32 img_type = le32_to_cpu(fsec_entry.type);
3903         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3904
3905         if (img_optype != 0xFFFF)
3906                 return img_optype;
3907
3908         switch (img_type) {
3909         case IMAGE_FIRMWARE_iSCSI:
3910                 img_optype = OPTYPE_ISCSI_ACTIVE;
3911                 break;
3912         case IMAGE_BOOT_CODE:
3913                 img_optype = OPTYPE_REDBOOT;
3914                 break;
3915         case IMAGE_OPTION_ROM_ISCSI:
3916                 img_optype = OPTYPE_BIOS;
3917                 break;
3918         case IMAGE_OPTION_ROM_PXE:
3919                 img_optype = OPTYPE_PXE_BIOS;
3920                 break;
3921         case IMAGE_OPTION_ROM_FCoE:
3922                 img_optype = OPTYPE_FCOE_BIOS;
3923                 break;
3924         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3925                 img_optype = OPTYPE_ISCSI_BACKUP;
3926                 break;
3927         case IMAGE_NCSI:
3928                 img_optype = OPTYPE_NCSI_FW;
3929                 break;
3930         case IMAGE_FLASHISM_JUMPVECTOR:
3931                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3932                 break;
3933         case IMAGE_FIRMWARE_PHY:
3934                 img_optype = OPTYPE_SH_PHY_FW;
3935                 break;
3936         case IMAGE_REDBOOT_DIR:
3937                 img_optype = OPTYPE_REDBOOT_DIR;
3938                 break;
3939         case IMAGE_REDBOOT_CONFIG:
3940                 img_optype = OPTYPE_REDBOOT_CONFIG;
3941                 break;
3942         case IMAGE_UFI_DIR:
3943                 img_optype = OPTYPE_UFI_DIR;
3944                 break;
3945         default:
3946                 break;
3947         }
3948
3949         return img_optype;
3950 }
3951
3952 static int be_flash_skyhawk(struct be_adapter *adapter,
3953                             const struct firmware *fw,
3954                             struct be_dma_mem *flash_cmd, int num_of_images)
3955 {
3956         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
3957         struct device *dev = &adapter->pdev->dev;
3958         struct flash_section_info *fsec = NULL;
3959         u32 img_offset, img_size, img_type;
3960         int status, i, filehdr_size;
3961         bool crc_match, old_fw_img;
3962         u16 img_optype;
3963         const u8 *p;
3964
3965         filehdr_size = sizeof(struct flash_file_hdr_g3);
3966         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3967         if (!fsec) {
3968                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3969                 return -EINVAL;
3970         }
3971
3972         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
3973                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
3974                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
3975                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
3976                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
3977                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
3978
3979                 if (img_optype == 0xFFFF)
3980                         continue;
3981                 /* Don't bother verifying CRC if an old FW image is being
3982                  * flashed
3983                  */
3984                 if (old_fw_img)
3985                         goto flash;
3986
3987                 status = be_check_flash_crc(adapter, fw->data, img_offset,
3988                                             img_size, filehdr_size +
3989                                             img_hdrs_size, img_optype,
3990                                             &crc_match);
3991                 /* The current FW image on the card does not recognize the new
3992                  * FLASH op_type. The FW download is partially complete.
3993                  * Reboot the server now to enable FW image to recognize the
3994                  * new FLASH op_type. To complete the remaining process,
3995                  * download the same FW again after the reboot.
3996                  */
3997                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
3998                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
3999                         dev_err(dev, "Flash incomplete. Reset the server\n");
4000                         dev_err(dev, "Download FW image again after reset\n");
4001                         return -EAGAIN;
4002                 } else if (status) {
4003                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4004                                 img_optype);
4005                         return -EFAULT;
4006                 }
4007
4008                 if (crc_match)
4009                         continue;
4010
4011 flash:
4012                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4013                 if (p + img_size > fw->data + fw->size)
4014                         return -1;
4015
4016                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4017                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4018                  * UFI_DIR region
4019                  */
4020                 if (old_fw_img &&
4021                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4022                      (img_optype == OPTYPE_UFI_DIR &&
4023                       base_status(status) == MCC_STATUS_FAILED))) {
4024                         continue;
4025                 } else if (status) {
4026                         dev_err(dev, "Flashing section type 0x%x failed\n",
4027                                 img_type);
4028                         return -EFAULT;
4029                 }
4030         }
4031         return 0;
4032 }
4033
4034 static int lancer_fw_download(struct be_adapter *adapter,
4035                               const struct firmware *fw)
4036 {
4037 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4038 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4039         struct be_dma_mem flash_cmd;
4040         const u8 *data_ptr = NULL;
4041         u8 *dest_image_ptr = NULL;
4042         size_t image_size = 0;
4043         u32 chunk_size = 0;
4044         u32 data_written = 0;
4045         u32 offset = 0;
4046         int status = 0;
4047         u8 add_status = 0;
4048         u8 change_status;
4049
4050         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4051                 dev_err(&adapter->pdev->dev,
4052                         "FW Image not properly aligned. "
4053                         "Length must be 4 byte aligned.\n");
4054                 status = -EINVAL;
4055                 goto lancer_fw_exit;
4056         }
4057
4058         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4059                                 + LANCER_FW_DOWNLOAD_CHUNK;
4060         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4061                                           &flash_cmd.dma, GFP_KERNEL);
4062         if (!flash_cmd.va) {
4063                 status = -ENOMEM;
4064                 goto lancer_fw_exit;
4065         }
4066
4067         dest_image_ptr = flash_cmd.va +
4068                                 sizeof(struct lancer_cmd_req_write_object);
4069         image_size = fw->size;
4070         data_ptr = fw->data;
4071
4072         while (image_size) {
4073                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4074
4075                 /* Copy the image chunk content. */
4076                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4077
4078                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4079                                                  chunk_size, offset,
4080                                                  LANCER_FW_DOWNLOAD_LOCATION,
4081                                                  &data_written, &change_status,
4082                                                  &add_status);
4083                 if (status)
4084                         break;
4085
4086                 offset += data_written;
4087                 data_ptr += data_written;
4088                 image_size -= data_written;
4089         }
4090
4091         if (!status) {
4092                 /* Commit the FW written */
4093                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4094                                                  0, offset,
4095                                                  LANCER_FW_DOWNLOAD_LOCATION,
4096                                                  &data_written, &change_status,
4097                                                  &add_status);
4098         }
4099
4100         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4101                           flash_cmd.dma);
4102         if (status) {
4103                 dev_err(&adapter->pdev->dev,
4104                         "Firmware load error. "
4105                         "Status code: 0x%x Additional Status: 0x%x\n",
4106                         status, add_status);
4107                 goto lancer_fw_exit;
4108         }
4109
4110         if (change_status == LANCER_FW_RESET_NEEDED) {
4111                 dev_info(&adapter->pdev->dev,
4112                          "Resetting adapter to activate new FW\n");
4113                 status = lancer_physdev_ctrl(adapter,
4114                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4115                 if (status) {
4116                         dev_err(&adapter->pdev->dev,
4117                                 "Adapter busy for FW reset.\n"
4118                                 "New FW will not be active.\n");
4119                         goto lancer_fw_exit;
4120                 }
4121         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4122                 dev_err(&adapter->pdev->dev,
4123                         "System reboot required for new FW to be active\n");
4124         }
4125
4126         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4127 lancer_fw_exit:
4128         return status;
4129 }
4130
4131 #define UFI_TYPE2               2
4132 #define UFI_TYPE3               3
4133 #define UFI_TYPE3R              10
4134 #define UFI_TYPE4               4
4135 static int be_get_ufi_type(struct be_adapter *adapter,
4136                            struct flash_file_hdr_g3 *fhdr)
4137 {
4138         if (!fhdr)
4139                 goto be_get_ufi_exit;
4140
4141         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4142                 return UFI_TYPE4;
4143         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4144                 if (fhdr->asic_type_rev == 0x10)
4145                         return UFI_TYPE3R;
4146                 else
4147                         return UFI_TYPE3;
4148         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4149                 return UFI_TYPE2;
4150
4151 be_get_ufi_exit:
4152         dev_err(&adapter->pdev->dev,
4153                 "UFI and Interface are not compatible for flashing\n");
4154         return -1;
4155 }
4156
4157 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4158 {
4159         struct flash_file_hdr_g3 *fhdr3;
4160         struct image_hdr *img_hdr_ptr = NULL;
4161         struct be_dma_mem flash_cmd;
4162         const u8 *p;
4163         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4164
4165         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4166         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4167                                           &flash_cmd.dma, GFP_KERNEL);
4168         if (!flash_cmd.va) {
4169                 status = -ENOMEM;
4170                 goto be_fw_exit;
4171         }
4172
4173         p = fw->data;
4174         fhdr3 = (struct flash_file_hdr_g3 *)p;
4175
4176         ufi_type = be_get_ufi_type(adapter, fhdr3);
4177
4178         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4179         for (i = 0; i < num_imgs; i++) {
4180                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4181                                 (sizeof(struct flash_file_hdr_g3) +
4182                                  i * sizeof(struct image_hdr)));
4183                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4184                         switch (ufi_type) {
4185                         case UFI_TYPE4:
4186                                 status = be_flash_skyhawk(adapter, fw,
4187                                                           &flash_cmd, num_imgs);
4188                                 break;
4189                         case UFI_TYPE3R:
4190                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4191                                                       num_imgs);
4192                                 break;
4193                         case UFI_TYPE3:
4194                                 /* Do not flash this ufi on BE3-R cards */
4195                                 if (adapter->asic_rev < 0x10)
4196                                         status = be_flash_BEx(adapter, fw,
4197                                                               &flash_cmd,
4198                                                               num_imgs);
4199                                 else {
4200                                         status = -EINVAL;
4201                                         dev_err(&adapter->pdev->dev,
4202                                                 "Can't load BE3 UFI on BE3R\n");
4203                                 }
4204                         }
4205                 }
4206         }
4207
4208         if (ufi_type == UFI_TYPE2)
4209                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4210         else if (ufi_type == -1)
4211                 status = -EINVAL;
4212
4213         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4214                           flash_cmd.dma);
4215         if (status) {
4216                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4217                 goto be_fw_exit;
4218         }
4219
4220         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4221
4222 be_fw_exit:
4223         return status;
4224 }
4225
4226 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4227 {
4228         const struct firmware *fw;
4229         int status;
4230
4231         if (!netif_running(adapter->netdev)) {
4232                 dev_err(&adapter->pdev->dev,
4233                         "Firmware load not allowed (interface is down)\n");
4234                 return -ENETDOWN;
4235         }
4236
4237         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4238         if (status)
4239                 goto fw_exit;
4240
4241         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4242
4243         if (lancer_chip(adapter))
4244                 status = lancer_fw_download(adapter, fw);
4245         else
4246                 status = be_fw_download(adapter, fw);
4247
4248         if (!status)
4249                 be_cmd_get_fw_ver(adapter);
4250
4251 fw_exit:
4252         release_firmware(fw);
4253         return status;
4254 }
4255
4256 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4257 {
4258         struct be_adapter *adapter = netdev_priv(dev);
4259         struct nlattr *attr, *br_spec;
4260         int rem;
4261         int status = 0;
4262         u16 mode = 0;
4263
4264         if (!sriov_enabled(adapter))
4265                 return -EOPNOTSUPP;
4266
4267         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4268
4269         nla_for_each_nested(attr, br_spec, rem) {
4270                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4271                         continue;
4272
4273                 mode = nla_get_u16(attr);
4274                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4275                         return -EINVAL;
4276
4277                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4278                                                adapter->if_handle,
4279                                                mode == BRIDGE_MODE_VEPA ?
4280                                                PORT_FWD_TYPE_VEPA :
4281                                                PORT_FWD_TYPE_VEB);
4282                 if (status)
4283                         goto err;
4284
4285                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4286                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4287
4288                 return status;
4289         }
4290 err:
4291         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4292                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4293
4294         return status;
4295 }
4296
4297 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4298                                  struct net_device *dev, u32 filter_mask)
4299 {
4300         struct be_adapter *adapter = netdev_priv(dev);
4301         int status = 0;
4302         u8 hsw_mode;
4303
4304         if (!sriov_enabled(adapter))
4305                 return 0;
4306
4307         /* BE and Lancer chips support VEB mode only */
4308         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4309                 hsw_mode = PORT_FWD_TYPE_VEB;
4310         } else {
4311                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4312                                                adapter->if_handle, &hsw_mode);
4313                 if (status)
4314                         return 0;
4315         }
4316
4317         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4318                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4319                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4320 }
4321
4322 #ifdef CONFIG_BE2NET_VXLAN
4323 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4324                               __be16 port)
4325 {
4326         struct be_adapter *adapter = netdev_priv(netdev);
4327         struct device *dev = &adapter->pdev->dev;
4328         int status;
4329
4330         if (lancer_chip(adapter) || BEx_chip(adapter))
4331                 return;
4332
4333         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4334                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4335                          be16_to_cpu(port));
4336                 dev_info(dev,
4337                          "Only one UDP port supported for VxLAN offloads\n");
4338                 return;
4339         }
4340
4341         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4342                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4343         if (status) {
4344                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4345                 goto err;
4346         }
4347
4348         status = be_cmd_set_vxlan_port(adapter, port);
4349         if (status) {
4350                 dev_warn(dev, "Failed to add VxLAN port\n");
4351                 goto err;
4352         }
4353         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4354         adapter->vxlan_port = port;
4355
4356         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4357                  be16_to_cpu(port));
4358         return;
4359 err:
4360         be_disable_vxlan_offloads(adapter);
4361         return;
4362 }
4363
4364 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4365                               __be16 port)
4366 {
4367         struct be_adapter *adapter = netdev_priv(netdev);
4368
4369         if (lancer_chip(adapter) || BEx_chip(adapter))
4370                 return;
4371
4372         if (adapter->vxlan_port != port)
4373                 return;
4374
4375         be_disable_vxlan_offloads(adapter);
4376
4377         dev_info(&adapter->pdev->dev,
4378                  "Disabled VxLAN offloads for UDP port %d\n",
4379                  be16_to_cpu(port));
4380 }
4381 #endif
4382
4383 static const struct net_device_ops be_netdev_ops = {
4384         .ndo_open               = be_open,
4385         .ndo_stop               = be_close,
4386         .ndo_start_xmit         = be_xmit,
4387         .ndo_set_rx_mode        = be_set_rx_mode,
4388         .ndo_set_mac_address    = be_mac_addr_set,
4389         .ndo_change_mtu         = be_change_mtu,
4390         .ndo_get_stats64        = be_get_stats64,
4391         .ndo_validate_addr      = eth_validate_addr,
4392         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4393         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4394         .ndo_set_vf_mac         = be_set_vf_mac,
4395         .ndo_set_vf_vlan        = be_set_vf_vlan,
4396         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4397         .ndo_get_vf_config      = be_get_vf_config,
4398         .ndo_set_vf_link_state  = be_set_vf_link_state,
4399 #ifdef CONFIG_NET_POLL_CONTROLLER
4400         .ndo_poll_controller    = be_netpoll,
4401 #endif
4402         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4403         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4404 #ifdef CONFIG_NET_RX_BUSY_POLL
4405         .ndo_busy_poll          = be_busy_poll,
4406 #endif
4407 #ifdef CONFIG_BE2NET_VXLAN
4408         .ndo_add_vxlan_port     = be_add_vxlan_port,
4409         .ndo_del_vxlan_port     = be_del_vxlan_port,
4410 #endif
4411 };
4412
4413 static void be_netdev_init(struct net_device *netdev)
4414 {
4415         struct be_adapter *adapter = netdev_priv(netdev);
4416
4417         if (skyhawk_chip(adapter)) {
4418                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4419                                            NETIF_F_TSO | NETIF_F_TSO6 |
4420                                            NETIF_F_GSO_UDP_TUNNEL;
4421                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4422         }
4423         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4424                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4425                 NETIF_F_HW_VLAN_CTAG_TX;
4426         if (be_multi_rxq(adapter))
4427                 netdev->hw_features |= NETIF_F_RXHASH;
4428
4429         netdev->features |= netdev->hw_features |
4430                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4431
4432         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4433                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4434
4435         netdev->priv_flags |= IFF_UNICAST_FLT;
4436
4437         netdev->flags |= IFF_MULTICAST;
4438
4439         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4440
4441         netdev->netdev_ops = &be_netdev_ops;
4442
4443         netdev->ethtool_ops = &be_ethtool_ops;
4444 }
4445
4446 static void be_unmap_pci_bars(struct be_adapter *adapter)
4447 {
4448         if (adapter->csr)
4449                 pci_iounmap(adapter->pdev, adapter->csr);
4450         if (adapter->db)
4451                 pci_iounmap(adapter->pdev, adapter->db);
4452 }
4453
4454 static int db_bar(struct be_adapter *adapter)
4455 {
4456         if (lancer_chip(adapter) || !be_physfn(adapter))
4457                 return 0;
4458         else
4459                 return 4;
4460 }
4461
4462 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4463 {
4464         if (skyhawk_chip(adapter)) {
4465                 adapter->roce_db.size = 4096;
4466                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4467                                                               db_bar(adapter));
4468                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4469                                                                db_bar(adapter));
4470         }
4471         return 0;
4472 }
4473
4474 static int be_map_pci_bars(struct be_adapter *adapter)
4475 {
4476         u8 __iomem *addr;
4477
4478         if (BEx_chip(adapter) && be_physfn(adapter)) {
4479                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4480                 if (!adapter->csr)
4481                         return -ENOMEM;
4482         }
4483
4484         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4485         if (!addr)
4486                 goto pci_map_err;
4487         adapter->db = addr;
4488
4489         be_roce_map_pci_bars(adapter);
4490         return 0;
4491
4492 pci_map_err:
4493         be_unmap_pci_bars(adapter);
4494         return -ENOMEM;
4495 }
4496
4497 static void be_ctrl_cleanup(struct be_adapter *adapter)
4498 {
4499         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4500
4501         be_unmap_pci_bars(adapter);
4502
4503         if (mem->va)
4504                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4505                                   mem->dma);
4506
4507         mem = &adapter->rx_filter;
4508         if (mem->va)
4509                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4510                                   mem->dma);
4511 }
4512
4513 static int be_ctrl_init(struct be_adapter *adapter)
4514 {
4515         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4516         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4517         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4518         u32 sli_intf;
4519         int status;
4520
4521         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4522         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4523                                  SLI_INTF_FAMILY_SHIFT;
4524         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4525
4526         status = be_map_pci_bars(adapter);
4527         if (status)
4528                 goto done;
4529
4530         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4531         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4532                                                 mbox_mem_alloc->size,
4533                                                 &mbox_mem_alloc->dma,
4534                                                 GFP_KERNEL);
4535         if (!mbox_mem_alloc->va) {
4536                 status = -ENOMEM;
4537                 goto unmap_pci_bars;
4538         }
4539         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4540         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4541         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4542         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4543
4544         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4545         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4546                                             rx_filter->size, &rx_filter->dma,
4547                                             GFP_KERNEL);
4548         if (!rx_filter->va) {
4549                 status = -ENOMEM;
4550                 goto free_mbox;
4551         }
4552
4553         mutex_init(&adapter->mbox_lock);
4554         spin_lock_init(&adapter->mcc_lock);
4555         spin_lock_init(&adapter->mcc_cq_lock);
4556
4557         init_completion(&adapter->et_cmd_compl);
4558         pci_save_state(adapter->pdev);
4559         return 0;
4560
4561 free_mbox:
4562         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4563                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4564
4565 unmap_pci_bars:
4566         be_unmap_pci_bars(adapter);
4567
4568 done:
4569         return status;
4570 }
4571
4572 static void be_stats_cleanup(struct be_adapter *adapter)
4573 {
4574         struct be_dma_mem *cmd = &adapter->stats_cmd;
4575
4576         if (cmd->va)
4577                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4578                                   cmd->va, cmd->dma);
4579 }
4580
4581 static int be_stats_init(struct be_adapter *adapter)
4582 {
4583         struct be_dma_mem *cmd = &adapter->stats_cmd;
4584
4585         if (lancer_chip(adapter))
4586                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4587         else if (BE2_chip(adapter))
4588                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4589         else if (BE3_chip(adapter))
4590                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4591         else
4592                 /* ALL non-BE ASICs */
4593                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4594
4595         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4596                                       GFP_KERNEL);
4597         if (!cmd->va)
4598                 return -ENOMEM;
4599         return 0;
4600 }
4601
4602 static void be_remove(struct pci_dev *pdev)
4603 {
4604         struct be_adapter *adapter = pci_get_drvdata(pdev);
4605
4606         if (!adapter)
4607                 return;
4608
4609         be_roce_dev_remove(adapter);
4610         be_intr_set(adapter, false);
4611
4612         cancel_delayed_work_sync(&adapter->func_recovery_work);
4613
4614         unregister_netdev(adapter->netdev);
4615
4616         be_clear(adapter);
4617
4618         /* tell fw we're done with firing cmds */
4619         be_cmd_fw_clean(adapter);
4620
4621         be_stats_cleanup(adapter);
4622
4623         be_ctrl_cleanup(adapter);
4624
4625         pci_disable_pcie_error_reporting(pdev);
4626
4627         pci_release_regions(pdev);
4628         pci_disable_device(pdev);
4629
4630         free_netdev(adapter->netdev);
4631 }
4632
4633 static int be_get_initial_config(struct be_adapter *adapter)
4634 {
4635         int status, level;
4636
4637         status = be_cmd_get_cntl_attributes(adapter);
4638         if (status)
4639                 return status;
4640
4641         /* Must be a power of 2 or else MODULO will BUG_ON */
4642         adapter->be_get_temp_freq = 64;
4643
4644         if (BEx_chip(adapter)) {
4645                 level = be_cmd_get_fw_log_level(adapter);
4646                 adapter->msg_enable =
4647                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4648         }
4649
4650         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4651         return 0;
4652 }
4653
4654 static int lancer_recover_func(struct be_adapter *adapter)
4655 {
4656         struct device *dev = &adapter->pdev->dev;
4657         int status;
4658
4659         status = lancer_test_and_set_rdy_state(adapter);
4660         if (status)
4661                 goto err;
4662
4663         if (netif_running(adapter->netdev))
4664                 be_close(adapter->netdev);
4665
4666         be_clear(adapter);
4667
4668         be_clear_all_error(adapter);
4669
4670         status = be_setup(adapter);
4671         if (status)
4672                 goto err;
4673
4674         if (netif_running(adapter->netdev)) {
4675                 status = be_open(adapter->netdev);
4676                 if (status)
4677                         goto err;
4678         }
4679
4680         dev_err(dev, "Adapter recovery successful\n");
4681         return 0;
4682 err:
4683         if (status == -EAGAIN)
4684                 dev_err(dev, "Waiting for resource provisioning\n");
4685         else
4686                 dev_err(dev, "Adapter recovery failed\n");
4687
4688         return status;
4689 }
4690
4691 static void be_func_recovery_task(struct work_struct *work)
4692 {
4693         struct be_adapter *adapter =
4694                 container_of(work, struct be_adapter,  func_recovery_work.work);
4695         int status = 0;
4696
4697         be_detect_error(adapter);
4698
4699         if (adapter->hw_error && lancer_chip(adapter)) {
4700
4701                 rtnl_lock();
4702                 netif_device_detach(adapter->netdev);
4703                 rtnl_unlock();
4704
4705                 status = lancer_recover_func(adapter);
4706                 if (!status)
4707                         netif_device_attach(adapter->netdev);
4708         }
4709
4710         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4711          * no need to attempt further recovery.
4712          */
4713         if (!status || status == -EAGAIN)
4714                 schedule_delayed_work(&adapter->func_recovery_work,
4715                                       msecs_to_jiffies(1000));
4716 }
4717
4718 static void be_worker(struct work_struct *work)
4719 {
4720         struct be_adapter *adapter =
4721                 container_of(work, struct be_adapter, work.work);
4722         struct be_rx_obj *rxo;
4723         int i;
4724
4725         /* when interrupts are not yet enabled, just reap any pending
4726         * mcc completions */
4727         if (!netif_running(adapter->netdev)) {
4728                 local_bh_disable();
4729                 be_process_mcc(adapter);
4730                 local_bh_enable();
4731                 goto reschedule;
4732         }
4733
4734         if (!adapter->stats_cmd_sent) {
4735                 if (lancer_chip(adapter))
4736                         lancer_cmd_get_pport_stats(adapter,
4737                                                 &adapter->stats_cmd);
4738                 else
4739                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4740         }
4741
4742         if (be_physfn(adapter) &&
4743             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4744                 be_cmd_get_die_temperature(adapter);
4745
4746         for_all_rx_queues(adapter, rxo, i) {
4747                 /* Replenish RX-queues starved due to memory
4748                  * allocation failures.
4749                  */
4750                 if (rxo->rx_post_starved)
4751                         be_post_rx_frags(rxo, GFP_KERNEL);
4752         }
4753
4754         be_eqd_update(adapter);
4755
4756 reschedule:
4757         adapter->work_counter++;
4758         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4759 }
4760
4761 /* If any VFs are already enabled don't FLR the PF */
4762 static bool be_reset_required(struct be_adapter *adapter)
4763 {
4764         return pci_num_vf(adapter->pdev) ? false : true;
4765 }
4766
4767 static char *mc_name(struct be_adapter *adapter)
4768 {
4769         char *str = ""; /* default */
4770
4771         switch (adapter->mc_type) {
4772         case UMC:
4773                 str = "UMC";
4774                 break;
4775         case FLEX10:
4776                 str = "FLEX10";
4777                 break;
4778         case vNIC1:
4779                 str = "vNIC-1";
4780                 break;
4781         case nPAR:
4782                 str = "nPAR";
4783                 break;
4784         case UFP:
4785                 str = "UFP";
4786                 break;
4787         case vNIC2:
4788                 str = "vNIC-2";
4789                 break;
4790         default:
4791                 str = "";
4792         }
4793
4794         return str;
4795 }
4796
4797 static inline char *func_name(struct be_adapter *adapter)
4798 {
4799         return be_physfn(adapter) ? "PF" : "VF";
4800 }
4801
4802 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4803 {
4804         int status = 0;
4805         struct be_adapter *adapter;
4806         struct net_device *netdev;
4807         char port_name;
4808
4809         status = pci_enable_device(pdev);
4810         if (status)
4811                 goto do_none;
4812
4813         status = pci_request_regions(pdev, DRV_NAME);
4814         if (status)
4815                 goto disable_dev;
4816         pci_set_master(pdev);
4817
4818         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4819         if (!netdev) {
4820                 status = -ENOMEM;
4821                 goto rel_reg;
4822         }
4823         adapter = netdev_priv(netdev);
4824         adapter->pdev = pdev;
4825         pci_set_drvdata(pdev, adapter);
4826         adapter->netdev = netdev;
4827         SET_NETDEV_DEV(netdev, &pdev->dev);
4828
4829         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4830         if (!status) {
4831                 netdev->features |= NETIF_F_HIGHDMA;
4832         } else {
4833                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4834                 if (status) {
4835                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4836                         goto free_netdev;
4837                 }
4838         }
4839
4840         if (be_physfn(adapter)) {
4841                 status = pci_enable_pcie_error_reporting(pdev);
4842                 if (!status)
4843                         dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4844         }
4845
4846         status = be_ctrl_init(adapter);
4847         if (status)
4848                 goto free_netdev;
4849
4850         /* sync up with fw's ready state */
4851         if (be_physfn(adapter)) {
4852                 status = be_fw_wait_ready(adapter);
4853                 if (status)
4854                         goto ctrl_clean;
4855         }
4856
4857         if (be_reset_required(adapter)) {
4858                 status = be_cmd_reset_function(adapter);
4859                 if (status)
4860                         goto ctrl_clean;
4861
4862                 /* Wait for interrupts to quiesce after an FLR */
4863                 msleep(100);
4864         }
4865
4866         /* Allow interrupts for other ULPs running on NIC function */
4867         be_intr_set(adapter, true);
4868
4869         /* tell fw we're ready to fire cmds */
4870         status = be_cmd_fw_init(adapter);
4871         if (status)
4872                 goto ctrl_clean;
4873
4874         status = be_stats_init(adapter);
4875         if (status)
4876                 goto ctrl_clean;
4877
4878         status = be_get_initial_config(adapter);
4879         if (status)
4880                 goto stats_clean;
4881
4882         INIT_DELAYED_WORK(&adapter->work, be_worker);
4883         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4884         adapter->rx_fc = adapter->tx_fc = true;
4885
4886         status = be_setup(adapter);
4887         if (status)
4888                 goto stats_clean;
4889
4890         be_netdev_init(netdev);
4891         status = register_netdev(netdev);
4892         if (status != 0)
4893                 goto unsetup;
4894
4895         be_roce_dev_add(adapter);
4896
4897         schedule_delayed_work(&adapter->func_recovery_work,
4898                               msecs_to_jiffies(1000));
4899
4900         be_cmd_query_port_name(adapter, &port_name);
4901
4902         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4903                  func_name(adapter), mc_name(adapter), port_name);
4904
4905         return 0;
4906
4907 unsetup:
4908         be_clear(adapter);
4909 stats_clean:
4910         be_stats_cleanup(adapter);
4911 ctrl_clean:
4912         be_ctrl_cleanup(adapter);
4913 free_netdev:
4914         free_netdev(netdev);
4915 rel_reg:
4916         pci_release_regions(pdev);
4917 disable_dev:
4918         pci_disable_device(pdev);
4919 do_none:
4920         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4921         return status;
4922 }
4923
4924 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4925 {
4926         struct be_adapter *adapter = pci_get_drvdata(pdev);
4927         struct net_device *netdev =  adapter->netdev;
4928
4929         if (adapter->wol_en)
4930                 be_setup_wol(adapter, true);
4931
4932         be_intr_set(adapter, false);
4933         cancel_delayed_work_sync(&adapter->func_recovery_work);
4934
4935         netif_device_detach(netdev);
4936         if (netif_running(netdev)) {
4937                 rtnl_lock();
4938                 be_close(netdev);
4939                 rtnl_unlock();
4940         }
4941         be_clear(adapter);
4942
4943         pci_save_state(pdev);
4944         pci_disable_device(pdev);
4945         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4946         return 0;
4947 }
4948
4949 static int be_resume(struct pci_dev *pdev)
4950 {
4951         int status = 0;
4952         struct be_adapter *adapter = pci_get_drvdata(pdev);
4953         struct net_device *netdev =  adapter->netdev;
4954
4955         netif_device_detach(netdev);
4956
4957         status = pci_enable_device(pdev);
4958         if (status)
4959                 return status;
4960
4961         pci_set_power_state(pdev, PCI_D0);
4962         pci_restore_state(pdev);
4963
4964         status = be_fw_wait_ready(adapter);
4965         if (status)
4966                 return status;
4967
4968         be_intr_set(adapter, true);
4969         /* tell fw we're ready to fire cmds */
4970         status = be_cmd_fw_init(adapter);
4971         if (status)
4972                 return status;
4973
4974         be_setup(adapter);
4975         if (netif_running(netdev)) {
4976                 rtnl_lock();
4977                 be_open(netdev);
4978                 rtnl_unlock();
4979         }
4980
4981         schedule_delayed_work(&adapter->func_recovery_work,
4982                               msecs_to_jiffies(1000));
4983         netif_device_attach(netdev);
4984
4985         if (adapter->wol_en)
4986                 be_setup_wol(adapter, false);
4987
4988         return 0;
4989 }
4990
4991 /*
4992  * An FLR will stop BE from DMAing any data.
4993  */
4994 static void be_shutdown(struct pci_dev *pdev)
4995 {
4996         struct be_adapter *adapter = pci_get_drvdata(pdev);
4997
4998         if (!adapter)
4999                 return;
5000
5001         cancel_delayed_work_sync(&adapter->work);
5002         cancel_delayed_work_sync(&adapter->func_recovery_work);
5003
5004         netif_device_detach(adapter->netdev);
5005
5006         be_cmd_reset_function(adapter);
5007
5008         pci_disable_device(pdev);
5009 }
5010
5011 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5012                                             pci_channel_state_t state)
5013 {
5014         struct be_adapter *adapter = pci_get_drvdata(pdev);
5015         struct net_device *netdev =  adapter->netdev;
5016
5017         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5018
5019         if (!adapter->eeh_error) {
5020                 adapter->eeh_error = true;
5021
5022                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5023
5024                 rtnl_lock();
5025                 netif_device_detach(netdev);
5026                 if (netif_running(netdev))
5027                         be_close(netdev);
5028                 rtnl_unlock();
5029
5030                 be_clear(adapter);
5031         }
5032
5033         if (state == pci_channel_io_perm_failure)
5034                 return PCI_ERS_RESULT_DISCONNECT;
5035
5036         pci_disable_device(pdev);
5037
5038         /* The error could cause the FW to trigger a flash debug dump.
5039          * Resetting the card while flash dump is in progress
5040          * can cause it not to recover; wait for it to finish.
5041          * Wait only for first function as it is needed only once per
5042          * adapter.
5043          */
5044         if (pdev->devfn == 0)
5045                 ssleep(30);
5046
5047         return PCI_ERS_RESULT_NEED_RESET;
5048 }
5049
5050 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5051 {
5052         struct be_adapter *adapter = pci_get_drvdata(pdev);
5053         int status;
5054
5055         dev_info(&adapter->pdev->dev, "EEH reset\n");
5056
5057         status = pci_enable_device(pdev);
5058         if (status)
5059                 return PCI_ERS_RESULT_DISCONNECT;
5060
5061         pci_set_master(pdev);
5062         pci_set_power_state(pdev, PCI_D0);
5063         pci_restore_state(pdev);
5064
5065         /* Check if card is ok and fw is ready */
5066         dev_info(&adapter->pdev->dev,
5067                  "Waiting for FW to be ready after EEH reset\n");
5068         status = be_fw_wait_ready(adapter);
5069         if (status)
5070                 return PCI_ERS_RESULT_DISCONNECT;
5071
5072         pci_cleanup_aer_uncorrect_error_status(pdev);
5073         be_clear_all_error(adapter);
5074         return PCI_ERS_RESULT_RECOVERED;
5075 }
5076
5077 static void be_eeh_resume(struct pci_dev *pdev)
5078 {
5079         int status = 0;
5080         struct be_adapter *adapter = pci_get_drvdata(pdev);
5081         struct net_device *netdev =  adapter->netdev;
5082
5083         dev_info(&adapter->pdev->dev, "EEH resume\n");
5084
5085         pci_save_state(pdev);
5086
5087         status = be_cmd_reset_function(adapter);
5088         if (status)
5089                 goto err;
5090
5091         /* On some BE3 FW versions, after a HW reset,
5092          * interrupts will remain disabled for each function.
5093          * So, explicitly enable interrupts
5094          */
5095         be_intr_set(adapter, true);
5096
5097         /* tell fw we're ready to fire cmds */
5098         status = be_cmd_fw_init(adapter);
5099         if (status)
5100                 goto err;
5101
5102         status = be_setup(adapter);
5103         if (status)
5104                 goto err;
5105
5106         if (netif_running(netdev)) {
5107                 status = be_open(netdev);
5108                 if (status)
5109                         goto err;
5110         }
5111
5112         schedule_delayed_work(&adapter->func_recovery_work,
5113                               msecs_to_jiffies(1000));
5114         netif_device_attach(netdev);
5115         return;
5116 err:
5117         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5118 }
5119
5120 static const struct pci_error_handlers be_eeh_handlers = {
5121         .error_detected = be_eeh_err_detected,
5122         .slot_reset = be_eeh_reset,
5123         .resume = be_eeh_resume,
5124 };
5125
5126 static struct pci_driver be_driver = {
5127         .name = DRV_NAME,
5128         .id_table = be_dev_ids,
5129         .probe = be_probe,
5130         .remove = be_remove,
5131         .suspend = be_suspend,
5132         .resume = be_resume,
5133         .shutdown = be_shutdown,
5134         .err_handler = &be_eeh_handlers
5135 };
5136
5137 static int __init be_init_module(void)
5138 {
5139         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5140             rx_frag_size != 2048) {
5141                 printk(KERN_WARNING DRV_NAME
5142                         " : Module param rx_frag_size must be 2048/4096/8192."
5143                         " Using 2048\n");
5144                 rx_frag_size = 2048;
5145         }
5146
5147         return pci_register_driver(&be_driver);
5148 }
5149 module_init(be_init_module);
5150
5151 static void __exit be_exit_module(void)
5152 {
5153         pci_unregister_driver(&be_driver);
5154 }
5155 module_exit(be_exit_module);