drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/busy_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  57
  58         /* Copy the NAPI object as it has been already initialized */
  59         from_fp->napi = to_fp->napi;
  60
  61         /* Move bnx2x_fastpath contents */
  62         memcpy(to_fp, from_fp, sizeof(*to_fp));
  63         to_fp->index = to;
  64
  65         /* Retain the tpa_info of the original `to' version as we don't want
  66          * 2 FPs to contain the same tpa_info pointer.
  67          */
  68         to_fp->tpa_info = old_tpa_info;
  69
  70         /* move sp_objs contents as well, as their indices match fp ones */
  71         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  72
  73         /* move fp_stats contents as well, as their indices match fp ones */
  74         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  75
  76         /* Update txdata pointers in fp and move txdata content accordingly:
  77          * Each fp consumes 'max_cos' txdata structures, so the index should be
  78          * decremented by max_cos x delta.
  79          */
  80
  81         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  82         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  83                                 (bp)->max_cos;
  84         if (from == FCOE_IDX(bp)) {
  85                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  86                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  87         }
  88
  89         memcpy(&bp->bnx2x_txq[new_txdata_index],
  90                &bp->bnx2x_txq[old_txdata_index],
  91                sizeof(struct bnx2x_fp_txdata));
  92         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  93 }
  94
  95 /**
  96  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  97  *
  98  * @bp:        driver handle
  99  * @buf:       character buffer to fill with the fw name
 100  * @buf_len:   length of the above buffer
 101  *
 102  */
 103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 104 {
 105         if (IS_PF(bp)) {
 106                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 107
 108                 phy_fw_ver[0] = '\0';
 109                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 110                                              phy_fw_ver, PHY_FW_VER_LEN);
 111                 strlcpy(buf, bp->fw_ver, buf_len);
 112                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 113                          "bc %d.%d.%d%s%s",
 114                          (bp->common.bc_ver & 0xff0000) >> 16,
 115                          (bp->common.bc_ver & 0xff00) >> 8,
 116                          (bp->common.bc_ver & 0xff),
 117                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 118         } else {
 119                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 120         }
 121 }
 122
 123 /**
 124  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 125  *
 126  * @bp: driver handle
 127  * @delta:      number of eth queues which were not allocated
 128  */
 129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 130 {
 131         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 132
 133         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 134          * backward along the array could cause memory to be overridden
 135          */
 136         for (cos = 1; cos < bp->max_cos; cos++) {
 137                 for (i = 0; i < old_eth_num - delta; i++) {
 138                         struct bnx2x_fastpath *fp = &bp->fp[i];
 139                         int new_idx = cos * (old_eth_num - delta) + i;
 140
 141                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 142                                sizeof(struct bnx2x_fp_txdata));
 143                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 144                 }
 145         }
 146 }
 147
 148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 149
 150 /* free skb in the packet ring at pos idx
 151  * return idx of last bd freed
 152  */
 153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 154                              u16 idx, unsigned int *pkts_compl,
 155                              unsigned int *bytes_compl)
 156 {
 157         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 158         struct eth_tx_start_bd *tx_start_bd;
 159         struct eth_tx_bd *tx_data_bd;
 160         struct sk_buff *skb = tx_buf->skb;
 161         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 162         int nbd;
 163
 164         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 165         prefetch(&skb->end);
 166
 167         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 168            txdata->txq_index, idx, tx_buf, skb);
 169
 170         /* unmap first bd */
 171         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 172         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 173                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 174
 175         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 176 #ifdef BNX2X_STOP_ON_ERROR
 177         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 178                 BNX2X_ERR("BAD nbd!\n");
 179                 bnx2x_panic();
 180         }
 181 #endif
 182         new_cons = nbd + tx_buf->first_bd;
 183
 184         /* Get the next bd */
 185         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 186
 187         /* Skip a parse bd... */
 188         --nbd;
 189         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 190
 191         /* ...and the TSO split header bd since they have no mapping */
 192         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 193                 --nbd;
 194                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 195         }
 196
 197         /* now free frags */
 198         while (nbd > 0) {
 199
 200                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 201                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 202                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 203                 if (--nbd)
 204                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 205         }
 206
 207         /* release skb */
 208         WARN_ON(!skb);
 209         if (likely(skb)) {
 210                 (*pkts_compl)++;
 211                 (*bytes_compl) += skb->len;
 212         }
 213
 214         dev_kfree_skb_any(skb);
 215         tx_buf->first_bd = 0;
 216         tx_buf->skb = NULL;
 217
 218         return new_cons;
 219 }
 220
 221 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 222 {
 223         struct netdev_queue *txq;
 224         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 225         unsigned int pkts_compl = 0, bytes_compl = 0;
 226
 227 #ifdef BNX2X_STOP_ON_ERROR
 228         if (unlikely(bp->panic))
 229                 return -1;
 230 #endif
 231
 232         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 233         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 234         sw_cons = txdata->tx_pkt_cons;
 235
 236         while (sw_cons != hw_cons) {
 237                 u16 pkt_cons;
 238
 239                 pkt_cons = TX_BD(sw_cons);
 240
 241                 DP(NETIF_MSG_TX_DONE,
 242                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 243                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 244
 245                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 246                                             &pkts_compl, &bytes_compl);
 247
 248                 sw_cons++;
 249         }
 250
 251         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 252
 253         txdata->tx_pkt_cons = sw_cons;
 254         txdata->tx_bd_cons = bd_cons;
 255
 256         /* Need to make the tx_bd_cons update visible to start_xmit()
 257          * before checking for netif_tx_queue_stopped().  Without the
 258          * memory barrier, there is a small possibility that
 259          * start_xmit() will miss it and cause the queue to be stopped
 260          * forever.
 261          * On the other hand we need an rmb() here to ensure the proper
 262          * ordering of bit testing in the following
 263          * netif_tx_queue_stopped(txq) call.
 264          */
 265         smp_mb();
 266
 267         if (unlikely(netif_tx_queue_stopped(txq))) {
 268                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 269                  * while it's empty. This could have happen if rx_action() gets
 270                  * suspended in bnx2x_tx_int() after the condition before
 271                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 272                  *
 273                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 274                  * sends some packets consuming the whole queue again->
 275                  * stops the queue
 276                  */
 277
 278                 __netif_tx_lock(txq, smp_processor_id());
 279
 280                 if ((netif_tx_queue_stopped(txq)) &&
 281                     (bp->state == BNX2X_STATE_OPEN) &&
 282                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 283                         netif_tx_wake_queue(txq);
 284
 285                 __netif_tx_unlock(txq);
 286         }
 287         return 0;
 288 }
 289
 290 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 291                                              u16 idx)
 292 {
 293         u16 last_max = fp->last_max_sge;
 294
 295         if (SUB_S16(idx, last_max) > 0)
 296                 fp->last_max_sge = idx;
 297 }
 298
 299 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 300                                          u16 sge_len,
 301                                          struct eth_end_agg_rx_cqe *cqe)
 302 {
 303         struct bnx2x *bp = fp->bp;
 304         u16 last_max, last_elem, first_elem;
 305         u16 delta = 0;
 306         u16 i;
 307
 308         if (!sge_len)
 309                 return;
 310
 311         /* First mark all used pages */
 312         for (i = 0; i < sge_len; i++)
 313                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 314                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 315
 316         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 317            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 318
 319         /* Here we assume that the last SGE index is the biggest */
 320         prefetch((void *)(fp->sge_mask));
 321         bnx2x_update_last_max_sge(fp,
 322                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 323
 324         last_max = RX_SGE(fp->last_max_sge);
 325         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 326         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 327
 328         /* If ring is not full */
 329         if (last_elem + 1 != first_elem)
 330                 last_elem++;
 331
 332         /* Now update the prod */
 333         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 334                 if (likely(fp->sge_mask[i]))
 335                         break;
 336
 337                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 338                 delta += BIT_VEC64_ELEM_SZ;
 339         }
 340
 341         if (delta > 0) {
 342                 fp->rx_sge_prod += delta;
 343                 /* clear page-end entries */
 344                 bnx2x_clear_sge_mask_next_elems(fp);
 345         }
 346
 347         DP(NETIF_MSG_RX_STATUS,
 348            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 349            fp->last_max_sge, fp->rx_sge_prod);
 350 }
 351
 352 /* Get Toeplitz hash value in the skb using the value from the
 353  * CQE (calculated by HW).
 354  */
 355 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 356                             const struct eth_fast_path_rx_cqe *cqe,
 357                             bool *l4_rxhash)
 358 {
 359         /* Get Toeplitz hash from CQE */
 360         if ((bp->dev->features & NETIF_F_RXHASH) &&
 361             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 362                 enum eth_rss_hash_type htype;
 363
 364                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 365                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 366                              (htype == TCP_IPV6_HASH_TYPE);
 367                 return le32_to_cpu(cqe->rss_hash_result);
 368         }
 369         *l4_rxhash = false;
 370         return 0;
 371 }
 372
 373 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 374                             u16 cons, u16 prod,
 375                             struct eth_fast_path_rx_cqe *cqe)
 376 {
 377         struct bnx2x *bp = fp->bp;
 378         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 379         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 380         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 381         dma_addr_t mapping;
 382         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 383         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 384
 385         /* print error if current state != stop */
 386         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 387                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 388
 389         /* Try to map an empty data buffer from the aggregation info  */
 390         mapping = dma_map_single(&bp->pdev->dev,
 391                                  first_buf->data + NET_SKB_PAD,
 392                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 393         /*
 394          *  ...if it fails - move the skb from the consumer to the producer
 395          *  and set the current aggregation state as ERROR to drop it
 396          *  when TPA_STOP arrives.
 397          */
 398
 399         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 400                 /* Move the BD from the consumer to the producer */
 401                 bnx2x_reuse_rx_data(fp, cons, prod);
 402                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 403                 return;
 404         }
 405
 406         /* move empty data from pool to prod */
 407         prod_rx_buf->data = first_buf->data;
 408         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 409         /* point prod_bd to new data */
 410         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 411         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 412
 413         /* move partial skb from cons to pool (don't unmap yet) */
 414         *first_buf = *cons_rx_buf;
 415
 416         /* mark bin state as START */
 417         tpa_info->parsing_flags =
 418                 le16_to_cpu(cqe->pars_flags.flags);
 419         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 420         tpa_info->tpa_state = BNX2X_TPA_START;
 421         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 422         tpa_info->placement_offset = cqe->placement_offset;
 423         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 424         if (fp->mode == TPA_MODE_GRO) {
 425                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 426                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 427                 tpa_info->gro_size = gro_size;
 428         }
 429
 430 #ifdef BNX2X_STOP_ON_ERROR
 431         fp->tpa_queue_used |= (1 << queue);
 432 #ifdef _ASM_GENERIC_INT_L64_H
 433         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 434 #else
 435         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 436 #endif
 437            fp->tpa_queue_used);
 438 #endif
 439 }
 440
 441 /* Timestamp option length allowed for TPA aggregation:
 442  *
 443  *              nop nop kind length echo val
 444  */
 445 #define TPA_TSTAMP_OPT_LEN      12
 446 /**
 447  * bnx2x_set_gro_params - compute GRO values
 448  *
 449  * @skb:                packet skb
 450  * @parsing_flags:      parsing flags from the START CQE
 451  * @len_on_bd:          total length of the first packet for the
 452  *                      aggregation.
 453  * @pkt_len:            length of all segments
 454  *
 455  * Approximate value of the MSS for this aggregation calculated using
 456  * the first packet of it.
 457  * Compute number of aggregated segments, and gso_type.
 458  */
 459 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 460                                  u16 len_on_bd, unsigned int pkt_len,
 461                                  u16 num_of_coalesced_segs)
 462 {
 463         /* TPA aggregation won't have either IP options or TCP options
 464          * other than timestamp or IPv6 extension headers.
 465          */
 466         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 467
 468         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 469             PRS_FLAG_OVERETH_IPV6) {
 470                 hdrs_len += sizeof(struct ipv6hdr);
 471                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 472         } else {
 473                 hdrs_len += sizeof(struct iphdr);
 474                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 475         }
 476
 477         /* Check if there was a TCP timestamp, if there is it's will
 478          * always be 12 bytes length: nop nop kind length echo val.
 479          *
 480          * Otherwise FW would close the aggregation.
 481          */
 482         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 483                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 484
 485         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 486
 487         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 488          * to skb_shinfo(skb)->gso_segs
 489          */
 490         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 491 }
 492
 493 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 494                               struct bnx2x_fastpath *fp, u16 index)
 495 {
 496         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 497         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 498         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 499         dma_addr_t mapping;
 500
 501         if (unlikely(page == NULL)) {
 502                 BNX2X_ERR("Can't alloc sge\n");
 503                 return -ENOMEM;
 504         }
 505
 506         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 507                                SGE_PAGES, DMA_FROM_DEVICE);
 508         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 509                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 510                 BNX2X_ERR("Can't map sge\n");
 511                 return -ENOMEM;
 512         }
 513
 514         sw_buf->page = page;
 515         dma_unmap_addr_set(sw_buf, mapping, mapping);
 516
 517         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 518         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 519
 520         return 0;
 521 }
 522
 523 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 524                                struct bnx2x_agg_info *tpa_info,
 525                                u16 pages,
 526                                struct sk_buff *skb,
 527                                struct eth_end_agg_rx_cqe *cqe,
 528                                u16 cqe_idx)
 529 {
 530         struct sw_rx_page *rx_pg, old_rx_pg;
 531         u32 i, frag_len, frag_size;
 532         int err, j, frag_id = 0;
 533         u16 len_on_bd = tpa_info->len_on_bd;
 534         u16 full_page = 0, gro_size = 0;
 535
 536         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 537
 538         if (fp->mode == TPA_MODE_GRO) {
 539                 gro_size = tpa_info->gro_size;
 540                 full_page = tpa_info->full_page;
 541         }
 542
 543         /* This is needed in order to enable forwarding support */
 544         if (frag_size)
 545                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 546                                      le16_to_cpu(cqe->pkt_len),
 547                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 548
 549 #ifdef BNX2X_STOP_ON_ERROR
 550         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 551                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 552                           pages, cqe_idx);
 553                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 554                 bnx2x_panic();
 555                 return -EINVAL;
 556         }
 557 #endif
 558
 559         /* Run through the SGL and compose the fragmented skb */
 560         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 561                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 562
 563                 /* FW gives the indices of the SGE as if the ring is an array
 564                    (meaning that "next" element will consume 2 indices) */
 565                 if (fp->mode == TPA_MODE_GRO)
 566                         frag_len = min_t(u32, frag_size, (u32)full_page);
 567                 else /* LRO */
 568                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 569
 570                 rx_pg = &fp->rx_page_ring[sge_idx];
 571                 old_rx_pg = *rx_pg;
 572
 573                 /* If we fail to allocate a substitute page, we simply stop
 574                    where we are and drop the whole packet */
 575                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 576                 if (unlikely(err)) {
 577                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 578                         return err;
 579                 }
 580
 581                 /* Unmap the page as we're going to pass it to the stack */
 582                 dma_unmap_page(&bp->pdev->dev,
 583                                dma_unmap_addr(&old_rx_pg, mapping),
 584                                SGE_PAGES, DMA_FROM_DEVICE);
 585                 /* Add one frag and update the appropriate fields in the skb */
 586                 if (fp->mode == TPA_MODE_LRO)
 587                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 588                 else { /* GRO */
 589                         int rem;
 590                         int offset = 0;
 591                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 592                                 int len = rem > gro_size ? gro_size : rem;
 593                                 skb_fill_page_desc(skb, frag_id++,
 594                                                    old_rx_pg.page, offset, len);
 595                                 if (offset)
 596                                         get_page(old_rx_pg.page);
 597                                 offset += len;
 598                         }
 599                 }
 600
 601                 skb->data_len += frag_len;
 602                 skb->truesize += SGE_PAGES;
 603                 skb->len += frag_len;
 604
 605                 frag_size -= frag_len;
 606         }
 607
 608         return 0;
 609 }
 610
 611 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 612 {
 613         if (fp->rx_frag_size)
 614                 put_page(virt_to_head_page(data));
 615         else
 616                 kfree(data);
 617 }
 618
 619 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 620 {
 621         if (fp->rx_frag_size)
 622                 return netdev_alloc_frag(fp->rx_frag_size);
 623
 624         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 625 }
 626
 627 #ifdef CONFIG_INET
 628 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 629 {
 630         const struct iphdr *iph = ip_hdr(skb);
 631         struct tcphdr *th;
 632
 633         skb_set_transport_header(skb, sizeof(struct iphdr));
 634         th = tcp_hdr(skb);
 635
 636         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 637                                   iph->saddr, iph->daddr, 0);
 638 }
 639
 640 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 641 {
 642         struct ipv6hdr *iph = ipv6_hdr(skb);
 643         struct tcphdr *th;
 644
 645         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 646         th = tcp_hdr(skb);
 647
 648         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 649                                   &iph->saddr, &iph->daddr, 0);
 650 }
 651
 652 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 653                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 654 {
 655         skb_set_network_header(skb, 0);
 656         gro_func(bp, skb);
 657         tcp_gro_complete(skb);
 658 }
 659 #endif
 660
 661 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 662                                struct sk_buff *skb)
 663 {
 664 #ifdef CONFIG_INET
 665         if (skb_shinfo(skb)->gso_size) {
 666                 switch (be16_to_cpu(skb->protocol)) {
 667                 case ETH_P_IP:
 668                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 669                         break;
 670                 case ETH_P_IPV6:
 671                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 672                         break;
 673                 default:
 674                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 675                                   be16_to_cpu(skb->protocol));
 676                 }
 677         }
 678 #endif
 679         napi_gro_receive(&fp->napi, skb);
 680 }
 681
 682 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 683                            struct bnx2x_agg_info *tpa_info,
 684                            u16 pages,
 685                            struct eth_end_agg_rx_cqe *cqe,
 686                            u16 cqe_idx)
 687 {
 688         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 689         u8 pad = tpa_info->placement_offset;
 690         u16 len = tpa_info->len_on_bd;
 691         struct sk_buff *skb = NULL;
 692         u8 *new_data, *data = rx_buf->data;
 693         u8 old_tpa_state = tpa_info->tpa_state;
 694
 695         tpa_info->tpa_state = BNX2X_TPA_STOP;
 696
 697         /* If we there was an error during the handling of the TPA_START -
 698          * drop this aggregation.
 699          */
 700         if (old_tpa_state == BNX2X_TPA_ERROR)
 701                 goto drop;
 702
 703         /* Try to allocate the new data */
 704         new_data = bnx2x_frag_alloc(fp);
 705         /* Unmap skb in the pool anyway, as we are going to change
 706            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 707            fails. */
 708         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 709                          fp->rx_buf_size, DMA_FROM_DEVICE);
 710         if (likely(new_data))
 711                 skb = build_skb(data, fp->rx_frag_size);
 712
 713         if (likely(skb)) {
 714 #ifdef BNX2X_STOP_ON_ERROR
 715                 if (pad + len > fp->rx_buf_size) {
 716                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 717                                   pad, len, fp->rx_buf_size);
 718                         bnx2x_panic();
 719                         return;
 720                 }
 721 #endif
 722
 723                 skb_reserve(skb, pad + NET_SKB_PAD);
 724                 skb_put(skb, len);
 725                 skb->rxhash = tpa_info->rxhash;
 726                 skb->l4_rxhash = tpa_info->l4_rxhash;
 727
 728                 skb->protocol = eth_type_trans(skb, bp->dev);
 729                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 730
 731                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 732                                          skb, cqe, cqe_idx)) {
 733                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 734                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 735                         bnx2x_gro_receive(bp, fp, skb);
 736                 } else {
 737                         DP(NETIF_MSG_RX_STATUS,
 738                            "Failed to allocate new pages - dropping packet!\n");
 739                         dev_kfree_skb_any(skb);
 740                 }
 741
 742                 /* put new data in bin */
 743                 rx_buf->data = new_data;
 744
 745                 return;
 746         }
 747         bnx2x_frag_free(fp, new_data);
 748 drop:
 749         /* drop the packet and keep the buffer in the bin */
 750         DP(NETIF_MSG_RX_STATUS,
 751            "Failed to allocate or map a new skb - dropping packet!\n");
 752         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 753 }
 754
 755 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 756                                struct bnx2x_fastpath *fp, u16 index)
 757 {
 758         u8 *data;
 759         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 760         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 761         dma_addr_t mapping;
 762
 763         data = bnx2x_frag_alloc(fp);
 764         if (unlikely(data == NULL))
 765                 return -ENOMEM;
 766
 767         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 768                                  fp->rx_buf_size,
 769                                  DMA_FROM_DEVICE);
 770         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 771                 bnx2x_frag_free(fp, data);
 772                 BNX2X_ERR("Can't map rx data\n");
 773                 return -ENOMEM;
 774         }
 775
 776         rx_buf->data = data;
 777         dma_unmap_addr_set(rx_buf, mapping, mapping);
 778
 779         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 780         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 781
 782         return 0;
 783 }
 784
 785 static
 786 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 787                                  struct bnx2x_fastpath *fp,
 788                                  struct bnx2x_eth_q_stats *qstats)
 789 {
 790         /* Do nothing if no L4 csum validation was done.
 791          * We do not check whether IP csum was validated. For IPv4 we assume
 792          * that if the card got as far as validating the L4 csum, it also
 793          * validated the IP csum. IPv6 has no IP csum.
 794          */
 795         if (cqe->fast_path_cqe.status_flags &
 796             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 797                 return;
 798
 799         /* If L4 validation was done, check if an error was found. */
 800
 801         if (cqe->fast_path_cqe.type_error_flags &
 802             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 803              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 804                 qstats->hw_csum_err++;
 805         else
 806                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 807 }
 808
 809 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 810 {
 811         struct bnx2x *bp = fp->bp;
 812         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 813         u16 sw_comp_cons, sw_comp_prod;
 814         int rx_pkt = 0;
 815         union eth_rx_cqe *cqe;
 816         struct eth_fast_path_rx_cqe *cqe_fp;
 817
 818 #ifdef BNX2X_STOP_ON_ERROR
 819         if (unlikely(bp->panic))
 820                 return 0;
 821 #endif
 822
 823         bd_cons = fp->rx_bd_cons;
 824         bd_prod = fp->rx_bd_prod;
 825         bd_prod_fw = bd_prod;
 826         sw_comp_cons = fp->rx_comp_cons;
 827         sw_comp_prod = fp->rx_comp_prod;
 828
 829         comp_ring_cons = RCQ_BD(sw_comp_cons);
 830         cqe = &fp->rx_comp_ring[comp_ring_cons];
 831         cqe_fp = &cqe->fast_path_cqe;
 832
 833         DP(NETIF_MSG_RX_STATUS,
 834            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 835
 836         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 837                 struct sw_rx_bd *rx_buf = NULL;
 838                 struct sk_buff *skb;
 839                 u8 cqe_fp_flags;
 840                 enum eth_rx_cqe_type cqe_fp_type;
 841                 u16 len, pad, queue;
 842                 u8 *data;
 843                 bool l4_rxhash;
 844
 845 #ifdef BNX2X_STOP_ON_ERROR
 846                 if (unlikely(bp->panic))
 847                         return 0;
 848 #endif
 849
 850                 bd_prod = RX_BD(bd_prod);
 851                 bd_cons = RX_BD(bd_cons);
 852
 853                 cqe_fp_flags = cqe_fp->type_error_flags;
 854                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 855
 856                 DP(NETIF_MSG_RX_STATUS,
 857                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 858                    CQE_TYPE(cqe_fp_flags),
 859                    cqe_fp_flags, cqe_fp->status_flags,
 860                    le32_to_cpu(cqe_fp->rss_hash_result),
 861                    le16_to_cpu(cqe_fp->vlan_tag),
 862                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 863
 864                 /* is this a slowpath msg? */
 865                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 866                         bnx2x_sp_event(fp, cqe);
 867                         goto next_cqe;
 868                 }
 869
 870                 rx_buf = &fp->rx_buf_ring[bd_cons];
 871                 data = rx_buf->data;
 872
 873                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 874                         struct bnx2x_agg_info *tpa_info;
 875                         u16 frag_size, pages;
 876 #ifdef BNX2X_STOP_ON_ERROR
 877                         /* sanity check */
 878                         if (fp->disable_tpa &&
 879                             (CQE_TYPE_START(cqe_fp_type) ||
 880                              CQE_TYPE_STOP(cqe_fp_type)))
 881                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 882                                           CQE_TYPE(cqe_fp_type));
 883 #endif
 884
 885                         if (CQE_TYPE_START(cqe_fp_type)) {
 886                                 u16 queue = cqe_fp->queue_index;
 887                                 DP(NETIF_MSG_RX_STATUS,
 888                                    "calling tpa_start on queue %d\n",
 889                                    queue);
 890
 891                                 bnx2x_tpa_start(fp, queue,
 892                                                 bd_cons, bd_prod,
 893                                                 cqe_fp);
 894
 895                                 goto next_rx;
 896                         }
 897                         queue = cqe->end_agg_cqe.queue_index;
 898                         tpa_info = &fp->tpa_info[queue];
 899                         DP(NETIF_MSG_RX_STATUS,
 900                            "calling tpa_stop on queue %d\n",
 901                            queue);
 902
 903                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 904                                     tpa_info->len_on_bd;
 905
 906                         if (fp->mode == TPA_MODE_GRO)
 907                                 pages = (frag_size + tpa_info->full_page - 1) /
 908                                          tpa_info->full_page;
 909                         else
 910                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 911                                         SGE_PAGE_SHIFT;
 912
 913                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 914                                        &cqe->end_agg_cqe, comp_ring_cons);
 915 #ifdef BNX2X_STOP_ON_ERROR
 916                         if (bp->panic)
 917                                 return 0;
 918 #endif
 919
 920                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 921                         goto next_cqe;
 922                 }
 923                 /* non TPA */
 924                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 925                 pad = cqe_fp->placement_offset;
 926                 dma_sync_single_for_cpu(&bp->pdev->dev,
 927                                         dma_unmap_addr(rx_buf, mapping),
 928                                         pad + RX_COPY_THRESH,
 929                                         DMA_FROM_DEVICE);
 930                 pad += NET_SKB_PAD;
 931                 prefetch(data + pad); /* speedup eth_type_trans() */
 932                 /* is this an error packet? */
 933                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 934                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 935                            "ERROR  flags %x  rx packet %u\n",
 936                            cqe_fp_flags, sw_comp_cons);
 937                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 938                         goto reuse_rx;
 939                 }
 940
 941                 /* Since we don't have a jumbo ring
 942                  * copy small packets if mtu > 1500
 943                  */
 944                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 945                     (len <= RX_COPY_THRESH)) {
 946                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 947                         if (skb == NULL) {
 948                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 949                                    "ERROR  packet dropped because of alloc failure\n");
 950                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 951                                 goto reuse_rx;
 952                         }
 953                         memcpy(skb->data, data + pad, len);
 954                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 955                 } else {
 956                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 957                                 dma_unmap_single(&bp->pdev->dev,
 958                                                  dma_unmap_addr(rx_buf, mapping),
 959                                                  fp->rx_buf_size,
 960                                                  DMA_FROM_DEVICE);
 961                                 skb = build_skb(data, fp->rx_frag_size);
 962                                 if (unlikely(!skb)) {
 963                                         bnx2x_frag_free(fp, data);
 964                                         bnx2x_fp_qstats(bp, fp)->
 965                                                         rx_skb_alloc_failed++;
 966                                         goto next_rx;
 967                                 }
 968                                 skb_reserve(skb, pad);
 969                         } else {
 970                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 971                                    "ERROR  packet dropped because of alloc failure\n");
 972                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 973 reuse_rx:
 974                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 975                                 goto next_rx;
 976                         }
 977                 }
 978
 979                 skb_put(skb, len);
 980                 skb->protocol = eth_type_trans(skb, bp->dev);
 981
 982                 /* Set Toeplitz hash for a none-LRO skb */
 983                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 984                 skb->l4_rxhash = l4_rxhash;
 985
 986                 skb_checksum_none_assert(skb);
 987
 988                 if (bp->dev->features & NETIF_F_RXCSUM)
 989                         bnx2x_csum_validate(skb, cqe, fp,
 990                                             bnx2x_fp_qstats(bp, fp));
 991
 992                 skb_record_rx_queue(skb, fp->rx_queue);
 993
 994                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 995                     PARSING_FLAGS_VLAN)
 996                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 997                                                le16_to_cpu(cqe_fp->vlan_tag));
 998
 999                 skb_mark_napi_id(skb, &fp->napi);
1000
1001                 if (bnx2x_fp_ll_polling(fp))
1002                         netif_receive_skb(skb);
1003                 else
1004                         napi_gro_receive(&fp->napi, skb);
1005 next_rx:
1006                 rx_buf->data = NULL;
1007
1008                 bd_cons = NEXT_RX_IDX(bd_cons);
1009                 bd_prod = NEXT_RX_IDX(bd_prod);
1010                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1011                 rx_pkt++;
1012 next_cqe:
1013                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1014                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1015
1016                 /* mark CQE as free */
1017                 BNX2X_SEED_CQE(cqe_fp);
1018
1019                 if (rx_pkt == budget)
1020                         break;
1021
1022                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1023                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1024                 cqe_fp = &cqe->fast_path_cqe;
1025         } /* while */
1026
1027         fp->rx_bd_cons = bd_cons;
1028         fp->rx_bd_prod = bd_prod_fw;
1029         fp->rx_comp_cons = sw_comp_cons;
1030         fp->rx_comp_prod = sw_comp_prod;
1031
1032         /* Update producers */
1033         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1034                              fp->rx_sge_prod);
1035
1036         fp->rx_pkt += rx_pkt;
1037         fp->rx_calls++;
1038
1039         return rx_pkt;
1040 }
1041
1042 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1043 {
1044         struct bnx2x_fastpath *fp = fp_cookie;
1045         struct bnx2x *bp = fp->bp;
1046         u8 cos;
1047
1048         DP(NETIF_MSG_INTR,
1049            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1050            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1051
1052         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1053
1054 #ifdef BNX2X_STOP_ON_ERROR
1055         if (unlikely(bp->panic))
1056                 return IRQ_HANDLED;
1057 #endif
1058
1059         /* Handle Rx and Tx according to MSI-X vector */
1060         for_each_cos_in_tx_queue(fp, cos)
1061                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1062
1063         prefetch(&fp->sb_running_index[SM_RX_ID]);
1064         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1065
1066         return IRQ_HANDLED;
1067 }
1068
1069 /* HW Lock for shared dual port PHYs */
1070 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1071 {
1072         mutex_lock(&bp->port.phy_mutex);
1073
1074         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1075 }
1076
1077 void bnx2x_release_phy_lock(struct bnx2x *bp)
1078 {
1079         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1080
1081         mutex_unlock(&bp->port.phy_mutex);
1082 }
1083
1084 /* calculates MF speed according to current linespeed and MF configuration */
1085 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1086 {
1087         u16 line_speed = bp->link_vars.line_speed;
1088         if (IS_MF(bp)) {
1089                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1090                                                    bp->mf_config[BP_VN(bp)]);
1091
1092                 /* Calculate the current MAX line speed limit for the MF
1093                  * devices
1094                  */
1095                 if (IS_MF_SI(bp))
1096                         line_speed = (line_speed * maxCfg) / 100;
1097                 else { /* SD mode */
1098                         u16 vn_max_rate = maxCfg * 100;
1099
1100                         if (vn_max_rate < line_speed)
1101                                 line_speed = vn_max_rate;
1102                 }
1103         }
1104
1105         return line_speed;
1106 }
1107
1108 /**
1109  * bnx2x_fill_report_data - fill link report data to report
1110  *
1111  * @bp:         driver handle
1112  * @data:       link state to update
1113  *
1114  * It uses a none-atomic bit operations because is called under the mutex.
1115  */
1116 static void bnx2x_fill_report_data(struct bnx2x *bp,
1117                                    struct bnx2x_link_report_data *data)
1118 {
1119         u16 line_speed = bnx2x_get_mf_speed(bp);
1120
1121         memset(data, 0, sizeof(*data));
1122
1123         /* Fill the report data: effective line speed */
1124         data->line_speed = line_speed;
1125
1126         /* Link is down */
1127         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1128                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1129                           &data->link_report_flags);
1130
1131         /* Full DUPLEX */
1132         if (bp->link_vars.duplex == DUPLEX_FULL)
1133                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1134
1135         /* Rx Flow Control is ON */
1136         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1137                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1138
1139         /* Tx Flow Control is ON */
1140         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1141                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1142 }
1143
1144 /**
1145  * bnx2x_link_report - report link status to OS.
1146  *
1147  * @bp:         driver handle
1148  *
1149  * Calls the __bnx2x_link_report() under the same locking scheme
1150  * as a link/PHY state managing code to ensure a consistent link
1151  * reporting.
1152  */
1153
1154 void bnx2x_link_report(struct bnx2x *bp)
1155 {
1156         bnx2x_acquire_phy_lock(bp);
1157         __bnx2x_link_report(bp);
1158         bnx2x_release_phy_lock(bp);
1159 }
1160
1161 /**
1162  * __bnx2x_link_report - report link status to OS.
1163  *
1164  * @bp:         driver handle
1165  *
1166  * None atomic implementation.
1167  * Should be called under the phy_lock.
1168  */
1169 void __bnx2x_link_report(struct bnx2x *bp)
1170 {
1171         struct bnx2x_link_report_data cur_data;
1172
1173         /* reread mf_cfg */
1174         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1175                 bnx2x_read_mf_cfg(bp);
1176
1177         /* Read the current link report info */
1178         bnx2x_fill_report_data(bp, &cur_data);
1179
1180         /* Don't report link down or exactly the same link status twice */
1181         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1182             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1183                       &bp->last_reported_link.link_report_flags) &&
1184              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1185                       &cur_data.link_report_flags)))
1186                 return;
1187
1188         bp->link_cnt++;
1189
1190         /* We are going to report a new link parameters now -
1191          * remember the current data for the next time.
1192          */
1193         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1194
1195         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1196                      &cur_data.link_report_flags)) {
1197                 netif_carrier_off(bp->dev);
1198                 netdev_err(bp->dev, "NIC Link is Down\n");
1199                 return;
1200         } else {
1201                 const char *duplex;
1202                 const char *flow;
1203
1204                 netif_carrier_on(bp->dev);
1205
1206                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1207                                        &cur_data.link_report_flags))
1208                         duplex = "full";
1209                 else
1210                         duplex = "half";
1211
1212                 /* Handle the FC at the end so that only these flags would be
1213                  * possibly set. This way we may easily check if there is no FC
1214                  * enabled.
1215                  */
1216                 if (cur_data.link_report_flags) {
1217                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1218                                      &cur_data.link_report_flags)) {
1219                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1220                                      &cur_data.link_report_flags))
1221                                         flow = "ON - receive & transmit";
1222                                 else
1223                                         flow = "ON - receive";
1224                         } else {
1225                                 flow = "ON - transmit";
1226                         }
1227                 } else {
1228                         flow = "none";
1229                 }
1230                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1231                             cur_data.line_speed, duplex, flow);
1232         }
1233 }
1234
1235 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1236 {
1237         int i;
1238
1239         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1240                 struct eth_rx_sge *sge;
1241
1242                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1243                 sge->addr_hi =
1244                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1245                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1246
1247                 sge->addr_lo =
1248                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1249                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1250         }
1251 }
1252
1253 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1254                                 struct bnx2x_fastpath *fp, int last)
1255 {
1256         int i;
1257
1258         for (i = 0; i < last; i++) {
1259                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1260                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1261                 u8 *data = first_buf->data;
1262
1263                 if (data == NULL) {
1264                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1265                         continue;
1266                 }
1267                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1268                         dma_unmap_single(&bp->pdev->dev,
1269                                          dma_unmap_addr(first_buf, mapping),
1270                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1271                 bnx2x_frag_free(fp, data);
1272                 first_buf->data = NULL;
1273         }
1274 }
1275
1276 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1277 {
1278         int j;
1279
1280         for_each_rx_queue_cnic(bp, j) {
1281                 struct bnx2x_fastpath *fp = &bp->fp[j];
1282
1283                 fp->rx_bd_cons = 0;
1284
1285                 /* Activate BD ring */
1286                 /* Warning!
1287                  * this will generate an interrupt (to the TSTORM)
1288                  * must only be done after chip is initialized
1289                  */
1290                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1291                                      fp->rx_sge_prod);
1292         }
1293 }
1294
1295 void bnx2x_init_rx_rings(struct bnx2x *bp)
1296 {
1297         int func = BP_FUNC(bp);
1298         u16 ring_prod;
1299         int i, j;
1300
1301         /* Allocate TPA resources */
1302         for_each_eth_queue(bp, j) {
1303                 struct bnx2x_fastpath *fp = &bp->fp[j];
1304
1305                 DP(NETIF_MSG_IFUP,
1306                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1307
1308                 if (!fp->disable_tpa) {
1309                         /* Fill the per-aggregation pool */
1310                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1311                                 struct bnx2x_agg_info *tpa_info =
1312                                         &fp->tpa_info[i];
1313                                 struct sw_rx_bd *first_buf =
1314                                         &tpa_info->first_buf;
1315
1316                                 first_buf->data = bnx2x_frag_alloc(fp);
1317                                 if (!first_buf->data) {
1318                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1319                                                   j);
1320                                         bnx2x_free_tpa_pool(bp, fp, i);
1321                                         fp->disable_tpa = 1;
1322                                         break;
1323                                 }
1324                                 dma_unmap_addr_set(first_buf, mapping, 0);
1325                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1326                         }
1327
1328                         /* "next page" elements initialization */
1329                         bnx2x_set_next_page_sgl(fp);
1330
1331                         /* set SGEs bit mask */
1332                         bnx2x_init_sge_ring_bit_mask(fp);
1333
1334                         /* Allocate SGEs and initialize the ring elements */
1335                         for (i = 0, ring_prod = 0;
1336                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1337
1338                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1339                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1340                                                   i);
1341                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1342                                                   j);
1343                                         /* Cleanup already allocated elements */
1344                                         bnx2x_free_rx_sge_range(bp, fp,
1345                                                                 ring_prod);
1346                                         bnx2x_free_tpa_pool(bp, fp,
1347                                                             MAX_AGG_QS(bp));
1348                                         fp->disable_tpa = 1;
1349                                         ring_prod = 0;
1350                                         break;
1351                                 }
1352                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1353                         }
1354
1355                         fp->rx_sge_prod = ring_prod;
1356                 }
1357         }
1358
1359         for_each_eth_queue(bp, j) {
1360                 struct bnx2x_fastpath *fp = &bp->fp[j];
1361
1362                 fp->rx_bd_cons = 0;
1363
1364                 /* Activate BD ring */
1365                 /* Warning!
1366                  * this will generate an interrupt (to the TSTORM)
1367                  * must only be done after chip is initialized
1368                  */
1369                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1370                                      fp->rx_sge_prod);
1371
1372                 if (j != 0)
1373                         continue;
1374
1375                 if (CHIP_IS_E1(bp)) {
1376                         REG_WR(bp, BAR_USTRORM_INTMEM +
1377                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1378                                U64_LO(fp->rx_comp_mapping));
1379                         REG_WR(bp, BAR_USTRORM_INTMEM +
1380                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1381                                U64_HI(fp->rx_comp_mapping));
1382                 }
1383         }
1384 }
1385
1386 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1387 {
1388         u8 cos;
1389         struct bnx2x *bp = fp->bp;
1390
1391         for_each_cos_in_tx_queue(fp, cos) {
1392                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1393                 unsigned pkts_compl = 0, bytes_compl = 0;
1394
1395                 u16 sw_prod = txdata->tx_pkt_prod;
1396                 u16 sw_cons = txdata->tx_pkt_cons;
1397
1398                 while (sw_cons != sw_prod) {
1399                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1400                                           &pkts_compl, &bytes_compl);
1401                         sw_cons++;
1402                 }
1403
1404                 netdev_tx_reset_queue(
1405                         netdev_get_tx_queue(bp->dev,
1406                                             txdata->txq_index));
1407         }
1408 }
1409
1410 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1411 {
1412         int i;
1413
1414         for_each_tx_queue_cnic(bp, i) {
1415                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1416         }
1417 }
1418
1419 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1420 {
1421         int i;
1422
1423         for_each_eth_queue(bp, i) {
1424                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1425         }
1426 }
1427
1428 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1429 {
1430         struct bnx2x *bp = fp->bp;
1431         int i;
1432
1433         /* ring wasn't allocated */
1434         if (fp->rx_buf_ring == NULL)
1435                 return;
1436
1437         for (i = 0; i < NUM_RX_BD; i++) {
1438                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1439                 u8 *data = rx_buf->data;
1440
1441                 if (data == NULL)
1442                         continue;
1443                 dma_unmap_single(&bp->pdev->dev,
1444                                  dma_unmap_addr(rx_buf, mapping),
1445                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1446
1447                 rx_buf->data = NULL;
1448                 bnx2x_frag_free(fp, data);
1449         }
1450 }
1451
1452 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1453 {
1454         int j;
1455
1456         for_each_rx_queue_cnic(bp, j) {
1457                 bnx2x_free_rx_bds(&bp->fp[j]);
1458         }
1459 }
1460
1461 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1462 {
1463         int j;
1464
1465         for_each_eth_queue(bp, j) {
1466                 struct bnx2x_fastpath *fp = &bp->fp[j];
1467
1468                 bnx2x_free_rx_bds(fp);
1469
1470                 if (!fp->disable_tpa)
1471                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1472         }
1473 }
1474
1475 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1476 {
1477         bnx2x_free_tx_skbs_cnic(bp);
1478         bnx2x_free_rx_skbs_cnic(bp);
1479 }
1480
1481 void bnx2x_free_skbs(struct bnx2x *bp)
1482 {
1483         bnx2x_free_tx_skbs(bp);
1484         bnx2x_free_rx_skbs(bp);
1485 }
1486
1487 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1488 {
1489         /* load old values */
1490         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1491
1492         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1493                 /* leave all but MAX value */
1494                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1495
1496                 /* set new MAX value */
1497                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1498                                 & FUNC_MF_CFG_MAX_BW_MASK;
1499
1500                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1501         }
1502 }
1503
1504 /**
1505  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1506  *
1507  * @bp:         driver handle
1508  * @nvecs:      number of vectors to be released
1509  */
1510 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1511 {
1512         int i, offset = 0;
1513
1514         if (nvecs == offset)
1515                 return;
1516
1517         /* VFs don't have a default SB */
1518         if (IS_PF(bp)) {
1519                 free_irq(bp->msix_table[offset].vector, bp->dev);
1520                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1521                    bp->msix_table[offset].vector);
1522                 offset++;
1523         }
1524
1525         if (CNIC_SUPPORT(bp)) {
1526                 if (nvecs == offset)
1527                         return;
1528                 offset++;
1529         }
1530
1531         for_each_eth_queue(bp, i) {
1532                 if (nvecs == offset)
1533                         return;
1534                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1535                    i, bp->msix_table[offset].vector);
1536
1537                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1538         }
1539 }
1540
1541 void bnx2x_free_irq(struct bnx2x *bp)
1542 {
1543         if (bp->flags & USING_MSIX_FLAG &&
1544             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1545                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1546
1547                 /* vfs don't have a default status block */
1548                 if (IS_PF(bp))
1549                         nvecs++;
1550
1551                 bnx2x_free_msix_irqs(bp, nvecs);
1552         } else {
1553                 free_irq(bp->dev->irq, bp->dev);
1554         }
1555 }
1556
1557 int bnx2x_enable_msix(struct bnx2x *bp)
1558 {
1559         int msix_vec = 0, i, rc;
1560
1561         /* VFs don't have a default status block */
1562         if (IS_PF(bp)) {
1563                 bp->msix_table[msix_vec].entry = msix_vec;
1564                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1565                                bp->msix_table[0].entry);
1566                 msix_vec++;
1567         }
1568
1569         /* Cnic requires an msix vector for itself */
1570         if (CNIC_SUPPORT(bp)) {
1571                 bp->msix_table[msix_vec].entry = msix_vec;
1572                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1573                                msix_vec, bp->msix_table[msix_vec].entry);
1574                 msix_vec++;
1575         }
1576
1577         /* We need separate vectors for ETH queues only (not FCoE) */
1578         for_each_eth_queue(bp, i) {
1579                 bp->msix_table[msix_vec].entry = msix_vec;
1580                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1581                                msix_vec, msix_vec, i);
1582                 msix_vec++;
1583         }
1584
1585         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1586            msix_vec);
1587
1588         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1589
1590         /*
1591          * reconfigure number of tx/rx queues according to available
1592          * MSI-X vectors
1593          */
1594         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1595                 /* how less vectors we will have? */
1596                 int diff = msix_vec - rc;
1597
1598                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1599
1600                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1601
1602                 if (rc) {
1603                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1604                         goto no_msix;
1605                 }
1606                 /*
1607                  * decrease number of queues by number of unallocated entries
1608                  */
1609                 bp->num_ethernet_queues -= diff;
1610                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1611
1612                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1613                                bp->num_queues);
1614         } else if (rc > 0) {
1615                 /* Get by with single vector */
1616                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1617                 if (rc) {
1618                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1619                                        rc);
1620                         goto no_msix;
1621                 }
1622
1623                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1624                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1625
1626                 BNX2X_DEV_INFO("set number of queues to 1\n");
1627                 bp->num_ethernet_queues = 1;
1628                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1629         } else if (rc < 0) {
1630                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1631                 goto no_msix;
1632         }
1633
1634         bp->flags |= USING_MSIX_FLAG;
1635
1636         return 0;
1637
1638 no_msix:
1639         /* fall to INTx if not enough memory */
1640         if (rc == -ENOMEM)
1641                 bp->flags |= DISABLE_MSI_FLAG;
1642
1643         return rc;
1644 }
1645
1646 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1647 {
1648         int i, rc, offset = 0;
1649
1650         /* no default status block for vf */
1651         if (IS_PF(bp)) {
1652                 rc = request_irq(bp->msix_table[offset++].vector,
1653                                  bnx2x_msix_sp_int, 0,
1654                                  bp->dev->name, bp->dev);
1655                 if (rc) {
1656                         BNX2X_ERR("request sp irq failed\n");
1657                         return -EBUSY;
1658                 }
1659         }
1660
1661         if (CNIC_SUPPORT(bp))
1662                 offset++;
1663
1664         for_each_eth_queue(bp, i) {
1665                 struct bnx2x_fastpath *fp = &bp->fp[i];
1666                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1667                          bp->dev->name, i);
1668
1669                 rc = request_irq(bp->msix_table[offset].vector,
1670                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1671                 if (rc) {
1672                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1673                               bp->msix_table[offset].vector, rc);
1674                         bnx2x_free_msix_irqs(bp, offset);
1675                         return -EBUSY;
1676                 }
1677
1678                 offset++;
1679         }
1680
1681         i = BNX2X_NUM_ETH_QUEUES(bp);
1682         if (IS_PF(bp)) {
1683                 offset = 1 + CNIC_SUPPORT(bp);
1684                 netdev_info(bp->dev,
1685                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1686                             bp->msix_table[0].vector,
1687                             0, bp->msix_table[offset].vector,
1688                             i - 1, bp->msix_table[offset + i - 1].vector);
1689         } else {
1690                 offset = CNIC_SUPPORT(bp);
1691                 netdev_info(bp->dev,
1692                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1693                             0, bp->msix_table[offset].vector,
1694                             i - 1, bp->msix_table[offset + i - 1].vector);
1695         }
1696         return 0;
1697 }
1698
1699 int bnx2x_enable_msi(struct bnx2x *bp)
1700 {
1701         int rc;
1702
1703         rc = pci_enable_msi(bp->pdev);
1704         if (rc) {
1705                 BNX2X_DEV_INFO("MSI is not attainable\n");
1706                 return -1;
1707         }
1708         bp->flags |= USING_MSI_FLAG;
1709
1710         return 0;
1711 }
1712
1713 static int bnx2x_req_irq(struct bnx2x *bp)
1714 {
1715         unsigned long flags;
1716         unsigned int irq;
1717
1718         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1719                 flags = 0;
1720         else
1721                 flags = IRQF_SHARED;
1722
1723         if (bp->flags & USING_MSIX_FLAG)
1724                 irq = bp->msix_table[0].vector;
1725         else
1726                 irq = bp->pdev->irq;
1727
1728         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1729 }
1730
1731 static int bnx2x_setup_irqs(struct bnx2x *bp)
1732 {
1733         int rc = 0;
1734         if (bp->flags & USING_MSIX_FLAG &&
1735             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1736                 rc = bnx2x_req_msix_irqs(bp);
1737                 if (rc)
1738                         return rc;
1739         } else {
1740                 rc = bnx2x_req_irq(bp);
1741                 if (rc) {
1742                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1743                         return rc;
1744                 }
1745                 if (bp->flags & USING_MSI_FLAG) {
1746                         bp->dev->irq = bp->pdev->irq;
1747                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1748                                     bp->dev->irq);
1749                 }
1750                 if (bp->flags & USING_MSIX_FLAG) {
1751                         bp->dev->irq = bp->msix_table[0].vector;
1752                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1753                                     bp->dev->irq);
1754                 }
1755         }
1756
1757         return 0;
1758 }
1759
1760 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1761 {
1762         int i;
1763
1764         for_each_rx_queue_cnic(bp, i) {
1765                 bnx2x_fp_init_lock(&bp->fp[i]);
1766                 napi_enable(&bnx2x_fp(bp, i, napi));
1767         }
1768 }
1769
1770 static void bnx2x_napi_enable(struct bnx2x *bp)
1771 {
1772         int i;
1773
1774         for_each_eth_queue(bp, i) {
1775                 bnx2x_fp_init_lock(&bp->fp[i]);
1776                 napi_enable(&bnx2x_fp(bp, i, napi));
1777         }
1778 }
1779
1780 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1781 {
1782         int i;
1783
1784         local_bh_disable();
1785         for_each_rx_queue_cnic(bp, i) {
1786                 napi_disable(&bnx2x_fp(bp, i, napi));
1787                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1788                         mdelay(1);
1789         }
1790         local_bh_enable();
1791 }
1792
1793 static void bnx2x_napi_disable(struct bnx2x *bp)
1794 {
1795         int i;
1796
1797         local_bh_disable();
1798         for_each_eth_queue(bp, i) {
1799                 napi_disable(&bnx2x_fp(bp, i, napi));
1800                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1801                         mdelay(1);
1802         }
1803         local_bh_enable();
1804 }
1805
1806 void bnx2x_netif_start(struct bnx2x *bp)
1807 {
1808         if (netif_running(bp->dev)) {
1809                 bnx2x_napi_enable(bp);
1810                 if (CNIC_LOADED(bp))
1811                         bnx2x_napi_enable_cnic(bp);
1812                 bnx2x_int_enable(bp);
1813                 if (bp->state == BNX2X_STATE_OPEN)
1814                         netif_tx_wake_all_queues(bp->dev);
1815         }
1816 }
1817
1818 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1819 {
1820         bnx2x_int_disable_sync(bp, disable_hw);
1821         bnx2x_napi_disable(bp);
1822         if (CNIC_LOADED(bp))
1823                 bnx2x_napi_disable_cnic(bp);
1824 }
1825
1826 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1827 {
1828         struct bnx2x *bp = netdev_priv(dev);
1829
1830         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1831                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1832                 u16 ether_type = ntohs(hdr->h_proto);
1833
1834                 /* Skip VLAN tag if present */
1835                 if (ether_type == ETH_P_8021Q) {
1836                         struct vlan_ethhdr *vhdr =
1837                                 (struct vlan_ethhdr *)skb->data;
1838
1839                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1840                 }
1841
1842                 /* If ethertype is FCoE or FIP - use FCoE ring */
1843                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1844                         return bnx2x_fcoe_tx(bp, txq_index);
1845         }
1846
1847         /* select a non-FCoE queue */
1848         return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1849 }
1850
1851 void bnx2x_set_num_queues(struct bnx2x *bp)
1852 {
1853         /* RSS queues */
1854         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1855
1856         /* override in STORAGE SD modes */
1857         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1858                 bp->num_ethernet_queues = 1;
1859
1860         /* Add special queues */
1861         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1862         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1863
1864         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1865 }
1866
1867 /**
1868  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1869  *
1870  * @bp:         Driver handle
1871  *
1872  * We currently support for at most 16 Tx queues for each CoS thus we will
1873  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1874  * bp->max_cos.
1875  *
1876  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1877  * index after all ETH L2 indices.
1878  *
1879  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1880  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1881  * 16..31,...) with indices that are not coupled with any real Tx queue.
1882  *
1883  * The proper configuration of skb->queue_mapping is handled by
1884  * bnx2x_select_queue() and __skb_tx_hash().
1885  *
1886  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1887  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1888  */
1889 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1890 {
1891         int rc, tx, rx;
1892
1893         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1894         rx = BNX2X_NUM_ETH_QUEUES(bp);
1895
1896 /* account for fcoe queue */
1897         if (include_cnic && !NO_FCOE(bp)) {
1898                 rx++;
1899                 tx++;
1900         }
1901
1902         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1903         if (rc) {
1904                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1905                 return rc;
1906         }
1907         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1908         if (rc) {
1909                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1910                 return rc;
1911         }
1912
1913         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1914                           tx, rx);
1915
1916         return rc;
1917 }
1918
1919 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1920 {
1921         int i;
1922
1923         for_each_queue(bp, i) {
1924                 struct bnx2x_fastpath *fp = &bp->fp[i];
1925                 u32 mtu;
1926
1927                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1928                 if (IS_FCOE_IDX(i))
1929                         /*
1930                          * Although there are no IP frames expected to arrive to
1931                          * this ring we still want to add an
1932                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1933                          * overrun attack.
1934                          */
1935                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1936                 else
1937                         mtu = bp->dev->mtu;
1938                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1939                                   IP_HEADER_ALIGNMENT_PADDING +
1940                                   ETH_OVREHEAD +
1941                                   mtu +
1942                                   BNX2X_FW_RX_ALIGN_END;
1943                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1944                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1945                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1946                 else
1947                         fp->rx_frag_size = 0;
1948         }
1949 }
1950
1951 static int bnx2x_init_rss(struct bnx2x *bp)
1952 {
1953         int i;
1954         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1955
1956         /* Prepare the initial contents for the indirection table if RSS is
1957          * enabled
1958          */
1959         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1960                 bp->rss_conf_obj.ind_table[i] =
1961                         bp->fp->cl_id +
1962                         ethtool_rxfh_indir_default(i, num_eth_queues);
1963
1964         /*
1965          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1966          * per-port, so if explicit configuration is needed , do it only
1967          * for a PMF.
1968          *
1969          * For 57712 and newer on the other hand it's a per-function
1970          * configuration.
1971          */
1972         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1973 }
1974
1975 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1976               bool config_hash, bool enable)
1977 {
1978         struct bnx2x_config_rss_params params = {NULL};
1979
1980         /* Although RSS is meaningless when there is a single HW queue we
1981          * still need it enabled in order to have HW Rx hash generated.
1982          *
1983          * if (!is_eth_multi(bp))
1984          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1985          */
1986
1987         params.rss_obj = rss_obj;
1988
1989         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1990
1991         if (enable) {
1992                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1993
1994                 /* RSS configuration */
1995                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1996                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1997                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1998                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1999                 if (rss_obj->udp_rss_v4)
2000                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2001                 if (rss_obj->udp_rss_v6)
2002                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2003         } else {
2004                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2005         }
2006
2007         /* Hash bits */
2008         params.rss_result_mask = MULTI_MASK;
2009
2010         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2011
2012         if (config_hash) {
2013                 /* RSS keys */
2014                 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2015                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2016         }
2017
2018         if (IS_PF(bp))
2019                 return bnx2x_config_rss(bp, &params);
2020         else
2021                 return bnx2x_vfpf_config_rss(bp, &params);
2022 }
2023
2024 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2025 {
2026         struct bnx2x_func_state_params func_params = {NULL};
2027
2028         /* Prepare parameters for function state transitions */
2029         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2030
2031         func_params.f_obj = &bp->func_obj;
2032         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2033
2034         func_params.params.hw_init.load_phase = load_code;
2035
2036         return bnx2x_func_state_change(bp, &func_params);
2037 }
2038
2039 /*
2040  * Cleans the object that have internal lists without sending
2041  * ramrods. Should be run when interrupts are disabled.
2042  */
2043 void bnx2x_squeeze_objects(struct bnx2x *bp)
2044 {
2045         int rc;
2046         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2047         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2048         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2049
2050         /***************** Cleanup MACs' object first *************************/
2051
2052         /* Wait for completion of requested */
2053         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2054         /* Perform a dry cleanup */
2055         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2056
2057         /* Clean ETH primary MAC */
2058         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2059         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2060                                  &ramrod_flags);
2061         if (rc != 0)
2062                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2063
2064         /* Cleanup UC list */
2065         vlan_mac_flags = 0;
2066         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2067         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2068                                  &ramrod_flags);
2069         if (rc != 0)
2070                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2071
2072         /***************** Now clean mcast object *****************************/
2073         rparam.mcast_obj = &bp->mcast_obj;
2074         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2075
2076         /* Add a DEL command... - Since we're doing a driver cleanup only,
2077          * we take a lock surrounding both the initial send and the CONTs,
2078          * as we don't want a true completion to disrupt us in the middle.
2079          */
2080         netif_addr_lock_bh(bp->dev);
2081         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2082         if (rc < 0)
2083                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2084                           rc);
2085
2086         /* ...and wait until all pending commands are cleared */
2087         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2088         while (rc != 0) {
2089                 if (rc < 0) {
2090                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2091                                   rc);
2092                         netif_addr_unlock_bh(bp->dev);
2093                         return;
2094                 }
2095
2096                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2097         }
2098         netif_addr_unlock_bh(bp->dev);
2099 }
2100
2101 #ifndef BNX2X_STOP_ON_ERROR
2102 #define LOAD_ERROR_EXIT(bp, label) \
2103         do { \
2104                 (bp)->state = BNX2X_STATE_ERROR; \
2105                 goto label; \
2106         } while (0)
2107
2108 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2109         do { \
2110                 bp->cnic_loaded = false; \
2111                 goto label; \
2112         } while (0)
2113 #else /*BNX2X_STOP_ON_ERROR*/
2114 #define LOAD_ERROR_EXIT(bp, label) \
2115         do { \
2116                 (bp)->state = BNX2X_STATE_ERROR; \
2117                 (bp)->panic = 1; \
2118                 return -EBUSY; \
2119         } while (0)
2120 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2121         do { \
2122                 bp->cnic_loaded = false; \
2123                 (bp)->panic = 1; \
2124                 return -EBUSY; \
2125         } while (0)
2126 #endif /*BNX2X_STOP_ON_ERROR*/
2127
2128 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2129 {
2130         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2131                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2132         return;
2133 }
2134
2135 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2136 {
2137         int num_groups, vf_headroom = 0;
2138         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2139
2140         /* number of queues for statistics is number of eth queues + FCoE */
2141         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2142
2143         /* Total number of FW statistics requests =
2144          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2145          * and fcoe l2 queue) stats + num of queues (which includes another 1
2146          * for fcoe l2 queue if applicable)
2147          */
2148         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2149
2150         /* vf stats appear in the request list, but their data is allocated by
2151          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2152          * it is used to determine where to place the vf stats queries in the
2153          * request struct
2154          */
2155         if (IS_SRIOV(bp))
2156                 vf_headroom = bnx2x_vf_headroom(bp);
2157
2158         /* Request is built from stats_query_header and an array of
2159          * stats_query_cmd_group each of which contains
2160          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2161          * configured in the stats_query_header.
2162          */
2163         num_groups =
2164                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2165                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2166                  1 : 0));
2167
2168         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2169            bp->fw_stats_num, vf_headroom, num_groups);
2170         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2171                 num_groups * sizeof(struct stats_query_cmd_group);
2172
2173         /* Data for statistics requests + stats_counter
2174          * stats_counter holds per-STORM counters that are incremented
2175          * when STORM has finished with the current request.
2176          * memory for FCoE offloaded statistics are counted anyway,
2177          * even if they will not be sent.
2178          * VF stats are not accounted for here as the data of VF stats is stored
2179          * in memory allocated by the VF, not here.
2180          */
2181         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2182                 sizeof(struct per_pf_stats) +
2183                 sizeof(struct fcoe_statistics_params) +
2184                 sizeof(struct per_queue_stats) * num_queue_stats +
2185                 sizeof(struct stats_counter);
2186
2187         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2188                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2189
2190         /* Set shortcuts */
2191         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2192         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2193         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2194                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2195         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2196                 bp->fw_stats_req_sz;
2197
2198         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2199            U64_HI(bp->fw_stats_req_mapping),
2200            U64_LO(bp->fw_stats_req_mapping));
2201         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2202            U64_HI(bp->fw_stats_data_mapping),
2203            U64_LO(bp->fw_stats_data_mapping));
2204         return 0;
2205
2206 alloc_mem_err:
2207         bnx2x_free_fw_stats_mem(bp);
2208         BNX2X_ERR("Can't allocate FW stats memory\n");
2209         return -ENOMEM;
2210 }
2211
2212 /* send load request to mcp and analyze response */
2213 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2214 {
2215         u32 param;
2216
2217         /* init fw_seq */
2218         bp->fw_seq =
2219                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2220                  DRV_MSG_SEQ_NUMBER_MASK);
2221         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2222
2223         /* Get current FW pulse sequence */
2224         bp->fw_drv_pulse_wr_seq =
2225                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2226                  DRV_PULSE_SEQ_MASK);
2227         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2228
2229         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2230
2231         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2232                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2233
2234         /* load request */
2235         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2236
2237         /* if mcp fails to respond we must abort */
2238         if (!(*load_code)) {
2239                 BNX2X_ERR("MCP response failure, aborting\n");
2240                 return -EBUSY;
2241         }
2242
2243         /* If mcp refused (e.g. other port is in diagnostic mode) we
2244          * must abort
2245          */
2246         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2247                 BNX2X_ERR("MCP refused load request, aborting\n");
2248                 return -EBUSY;
2249         }
2250         return 0;
2251 }
2252
2253 /* check whether another PF has already loaded FW to chip. In
2254  * virtualized environments a pf from another VM may have already
2255  * initialized the device including loading FW
2256  */
2257 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2258 {
2259         /* is another pf loaded on this engine? */
2260         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2261             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2262                 /* build my FW version dword */
2263                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2264                         (BCM_5710_FW_MINOR_VERSION << 8) +
2265                         (BCM_5710_FW_REVISION_VERSION << 16) +
2266                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2267
2268                 /* read loaded FW from chip */
2269                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2270
2271                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2272                    loaded_fw, my_fw);
2273
2274                 /* abort nic load if version mismatch */
2275                 if (my_fw != loaded_fw) {
2276                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2277                                   loaded_fw, my_fw);
2278                         return -EBUSY;
2279                 }
2280         }
2281         return 0;
2282 }
2283
2284 /* returns the "mcp load_code" according to global load_count array */
2285 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2286 {
2287         int path = BP_PATH(bp);
2288
2289         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2290            path, load_count[path][0], load_count[path][1],
2291            load_count[path][2]);
2292         load_count[path][0]++;
2293         load_count[path][1 + port]++;
2294         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2295            path, load_count[path][0], load_count[path][1],
2296            load_count[path][2]);
2297         if (load_count[path][0] == 1)
2298                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2299         else if (load_count[path][1 + port] == 1)
2300                 return FW_MSG_CODE_DRV_LOAD_PORT;
2301         else
2302                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2303 }
2304
2305 /* mark PMF if applicable */
2306 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2307 {
2308         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2309             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2310             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2311                 bp->port.pmf = 1;
2312                 /* We need the barrier to ensure the ordering between the
2313                  * writing to bp->port.pmf here and reading it from the
2314                  * bnx2x_periodic_task().
2315                  */
2316                 smp_mb();
2317         } else {
2318                 bp->port.pmf = 0;
2319         }
2320
2321         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2322 }
2323
2324 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2325 {
2326         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2327              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2328             (bp->common.shmem2_base)) {
2329                 if (SHMEM2_HAS(bp, dcc_support))
2330                         SHMEM2_WR(bp, dcc_support,
2331                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2332                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2333                 if (SHMEM2_HAS(bp, afex_driver_support))
2334                         SHMEM2_WR(bp, afex_driver_support,
2335                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2336         }
2337
2338         /* Set AFEX default VLAN tag to an invalid value */
2339         bp->afex_def_vlan_tag = -1;
2340 }
2341
2342 /**
2343  * bnx2x_bz_fp - zero content of the fastpath structure.
2344  *
2345  * @bp:         driver handle
2346  * @index:      fastpath index to be zeroed
2347  *
2348  * Makes sure the contents of the bp->fp[index].napi is kept
2349  * intact.
2350  */
2351 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2352 {
2353         struct bnx2x_fastpath *fp = &bp->fp[index];
2354         int cos;
2355         struct napi_struct orig_napi = fp->napi;
2356         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2357
2358         /* bzero bnx2x_fastpath contents */
2359         if (fp->tpa_info)
2360                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2361                        sizeof(struct bnx2x_agg_info));
2362         memset(fp, 0, sizeof(*fp));
2363
2364         /* Restore the NAPI object as it has been already initialized */
2365         fp->napi = orig_napi;
2366         fp->tpa_info = orig_tpa_info;
2367         fp->bp = bp;
2368         fp->index = index;
2369         if (IS_ETH_FP(fp))
2370                 fp->max_cos = bp->max_cos;
2371         else
2372                 /* Special queues support only one CoS */
2373                 fp->max_cos = 1;
2374
2375         /* Init txdata pointers */
2376         if (IS_FCOE_FP(fp))
2377                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2378         if (IS_ETH_FP(fp))
2379                 for_each_cos_in_tx_queue(fp, cos)
2380                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2381                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2382
2383         /* set the tpa flag for each queue. The tpa flag determines the queue
2384          * minimal size so it must be set prior to queue memory allocation
2385          */
2386         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2387                                   (bp->flags & GRO_ENABLE_FLAG &&
2388                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2389         if (bp->flags & TPA_ENABLE_FLAG)
2390                 fp->mode = TPA_MODE_LRO;
2391         else if (bp->flags & GRO_ENABLE_FLAG)
2392                 fp->mode = TPA_MODE_GRO;
2393
2394         /* We don't want TPA on an FCoE L2 ring */
2395         if (IS_FCOE_FP(fp))
2396                 fp->disable_tpa = 1;
2397 }
2398
2399 int bnx2x_load_cnic(struct bnx2x *bp)
2400 {
2401         int i, rc, port = BP_PORT(bp);
2402
2403         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2404
2405         mutex_init(&bp->cnic_mutex);
2406
2407         if (IS_PF(bp)) {
2408                 rc = bnx2x_alloc_mem_cnic(bp);
2409                 if (rc) {
2410                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2411                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2412                 }
2413         }
2414
2415         rc = bnx2x_alloc_fp_mem_cnic(bp);
2416         if (rc) {
2417                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2418                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2419         }
2420
2421         /* Update the number of queues with the cnic queues */
2422         rc = bnx2x_set_real_num_queues(bp, 1);
2423         if (rc) {
2424                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2425                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2426         }
2427
2428         /* Add all CNIC NAPI objects */
2429         bnx2x_add_all_napi_cnic(bp);
2430         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2431         bnx2x_napi_enable_cnic(bp);
2432
2433         rc = bnx2x_init_hw_func_cnic(bp);
2434         if (rc)
2435                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2436
2437         bnx2x_nic_init_cnic(bp);
2438
2439         if (IS_PF(bp)) {
2440                 /* Enable Timer scan */
2441                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2442
2443                 /* setup cnic queues */
2444                 for_each_cnic_queue(bp, i) {
2445                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2446                         if (rc) {
2447                                 BNX2X_ERR("Queue setup failed\n");
2448                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2449                         }
2450                 }
2451         }
2452
2453         /* Initialize Rx filter. */
2454         bnx2x_set_rx_mode_inner(bp);
2455
2456         /* re-read iscsi info */
2457         bnx2x_get_iscsi_info(bp);
2458         bnx2x_setup_cnic_irq_info(bp);
2459         bnx2x_setup_cnic_info(bp);
2460         bp->cnic_loaded = true;
2461         if (bp->state == BNX2X_STATE_OPEN)
2462                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2463
2464         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2465
2466         return 0;
2467
2468 #ifndef BNX2X_STOP_ON_ERROR
2469 load_error_cnic2:
2470         /* Disable Timer scan */
2471         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2472
2473 load_error_cnic1:
2474         bnx2x_napi_disable_cnic(bp);
2475         /* Update the number of queues without the cnic queues */
2476         rc = bnx2x_set_real_num_queues(bp, 0);
2477         if (rc)
2478                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2479 load_error_cnic0:
2480         BNX2X_ERR("CNIC-related load failed\n");
2481         bnx2x_free_fp_mem_cnic(bp);
2482         bnx2x_free_mem_cnic(bp);
2483         return rc;
2484 #endif /* ! BNX2X_STOP_ON_ERROR */
2485 }
2486
2487 /* must be called with rtnl_lock */
2488 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2489 {
2490         int port = BP_PORT(bp);
2491         int i, rc = 0, load_code = 0;
2492
2493         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2494         DP(NETIF_MSG_IFUP,
2495            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2496
2497 #ifdef BNX2X_STOP_ON_ERROR
2498         if (unlikely(bp->panic)) {
2499                 BNX2X_ERR("Can't load NIC when there is panic\n");
2500                 return -EPERM;
2501         }
2502 #endif
2503
2504         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2505
2506         /* zero the structure w/o any lock, before SP handler is initialized */
2507         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2508         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2509                 &bp->last_reported_link.link_report_flags);
2510
2511         if (IS_PF(bp))
2512                 /* must be called before memory allocation and HW init */
2513                 bnx2x_ilt_set_info(bp);
2514
2515         /*
2516          * Zero fastpath structures preserving invariants like napi, which are
2517          * allocated only once, fp index, max_cos, bp pointer.
2518          * Also set fp->disable_tpa and txdata_ptr.
2519          */
2520         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2521         for_each_queue(bp, i)
2522                 bnx2x_bz_fp(bp, i);
2523         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2524                                   bp->num_cnic_queues) *
2525                                   sizeof(struct bnx2x_fp_txdata));
2526
2527         bp->fcoe_init = false;
2528
2529         /* Set the receive queues buffer size */
2530         bnx2x_set_rx_buf_size(bp);
2531
2532         if (IS_PF(bp)) {
2533                 rc = bnx2x_alloc_mem(bp);
2534                 if (rc) {
2535                         BNX2X_ERR("Unable to allocate bp memory\n");
2536                         return rc;
2537                 }
2538         }
2539
2540         /* Allocated memory for FW statistics  */
2541         if (bnx2x_alloc_fw_stats_mem(bp))
2542                 LOAD_ERROR_EXIT(bp, load_error0);
2543
2544         /* need to be done after alloc mem, since it's self adjusting to amount
2545          * of memory available for RSS queues
2546          */
2547         rc = bnx2x_alloc_fp_mem(bp);
2548         if (rc) {
2549                 BNX2X_ERR("Unable to allocate memory for fps\n");
2550                 LOAD_ERROR_EXIT(bp, load_error0);
2551         }
2552
2553         /* request pf to initialize status blocks */
2554         if (IS_VF(bp)) {
2555                 rc = bnx2x_vfpf_init(bp);
2556                 if (rc)
2557                         LOAD_ERROR_EXIT(bp, load_error0);
2558         }
2559
2560         /* As long as bnx2x_alloc_mem() may possibly update
2561          * bp->num_queues, bnx2x_set_real_num_queues() should always
2562          * come after it. At this stage cnic queues are not counted.
2563          */
2564         rc = bnx2x_set_real_num_queues(bp, 0);
2565         if (rc) {
2566                 BNX2X_ERR("Unable to set real_num_queues\n");
2567                 LOAD_ERROR_EXIT(bp, load_error0);
2568         }
2569
2570         /* configure multi cos mappings in kernel.
2571          * this configuration may be overridden by a multi class queue
2572          * discipline or by a dcbx negotiation result.
2573          */
2574         bnx2x_setup_tc(bp->dev, bp->max_cos);
2575
2576         /* Add all NAPI objects */
2577         bnx2x_add_all_napi(bp);
2578         DP(NETIF_MSG_IFUP, "napi added\n");
2579         bnx2x_napi_enable(bp);
2580
2581         if (IS_PF(bp)) {
2582                 /* set pf load just before approaching the MCP */
2583                 bnx2x_set_pf_load(bp);
2584
2585                 /* if mcp exists send load request and analyze response */
2586                 if (!BP_NOMCP(bp)) {
2587                         /* attempt to load pf */
2588                         rc = bnx2x_nic_load_request(bp, &load_code);
2589                         if (rc)
2590                                 LOAD_ERROR_EXIT(bp, load_error1);
2591
2592                         /* what did mcp say? */
2593                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2594                         if (rc) {
2595                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2596                                 LOAD_ERROR_EXIT(bp, load_error2);
2597                         }
2598                 } else {
2599                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2600                 }
2601
2602                 /* mark pmf if applicable */
2603                 bnx2x_nic_load_pmf(bp, load_code);
2604
2605                 /* Init Function state controlling object */
2606                 bnx2x__init_func_obj(bp);
2607
2608                 /* Initialize HW */
2609                 rc = bnx2x_init_hw(bp, load_code);
2610                 if (rc) {
2611                         BNX2X_ERR("HW init failed, aborting\n");
2612                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2613                         LOAD_ERROR_EXIT(bp, load_error2);
2614                 }
2615         }
2616
2617         bnx2x_pre_irq_nic_init(bp);
2618
2619         /* Connect to IRQs */
2620         rc = bnx2x_setup_irqs(bp);
2621         if (rc) {
2622                 BNX2X_ERR("setup irqs failed\n");
2623                 if (IS_PF(bp))
2624                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2625                 LOAD_ERROR_EXIT(bp, load_error2);
2626         }
2627
2628         /* Init per-function objects */
2629         if (IS_PF(bp)) {
2630                 /* Setup NIC internals and enable interrupts */
2631                 bnx2x_post_irq_nic_init(bp, load_code);
2632
2633                 bnx2x_init_bp_objs(bp);
2634                 bnx2x_iov_nic_init(bp);
2635
2636                 /* Set AFEX default VLAN tag to an invalid value */
2637                 bp->afex_def_vlan_tag = -1;
2638                 bnx2x_nic_load_afex_dcc(bp, load_code);
2639                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2640                 rc = bnx2x_func_start(bp);
2641                 if (rc) {
2642                         BNX2X_ERR("Function start failed!\n");
2643                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2644
2645                         LOAD_ERROR_EXIT(bp, load_error3);
2646                 }
2647
2648                 /* Send LOAD_DONE command to MCP */
2649                 if (!BP_NOMCP(bp)) {
2650                         load_code = bnx2x_fw_command(bp,
2651                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2652                         if (!load_code) {
2653                                 BNX2X_ERR("MCP response failure, aborting\n");
2654                                 rc = -EBUSY;
2655                                 LOAD_ERROR_EXIT(bp, load_error3);
2656                         }
2657                 }
2658
2659                 /* initialize FW coalescing state machines in RAM */
2660                 bnx2x_update_coalesce(bp);
2661         }
2662
2663         /* setup the leading queue */
2664         rc = bnx2x_setup_leading(bp);
2665         if (rc) {
2666                 BNX2X_ERR("Setup leading failed!\n");
2667                 LOAD_ERROR_EXIT(bp, load_error3);
2668         }
2669
2670         /* set up the rest of the queues */
2671         for_each_nondefault_eth_queue(bp, i) {
2672                 if (IS_PF(bp))
2673                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2674                 else /* VF */
2675                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2676                 if (rc) {
2677                         BNX2X_ERR("Queue %d setup failed\n", i);
2678                         LOAD_ERROR_EXIT(bp, load_error3);
2679                 }
2680         }
2681
2682         /* setup rss */
2683         rc = bnx2x_init_rss(bp);
2684         if (rc) {
2685                 BNX2X_ERR("PF RSS init failed\n");
2686                 LOAD_ERROR_EXIT(bp, load_error3);
2687         }
2688
2689         /* Now when Clients are configured we are ready to work */
2690         bp->state = BNX2X_STATE_OPEN;
2691
2692         /* Configure a ucast MAC */
2693         if (IS_PF(bp))
2694                 rc = bnx2x_set_eth_mac(bp, true);
2695         else /* vf */
2696                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2697                                            true);
2698         if (rc) {
2699                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2700                 LOAD_ERROR_EXIT(bp, load_error3);
2701         }
2702
2703         if (IS_PF(bp) && bp->pending_max) {
2704                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2705                 bp->pending_max = 0;
2706         }
2707
2708         if (bp->port.pmf) {
2709                 rc = bnx2x_initial_phy_init(bp, load_mode);
2710                 if (rc)
2711                         LOAD_ERROR_EXIT(bp, load_error3);
2712         }
2713         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2714
2715         /* Start fast path */
2716
2717         /* Initialize Rx filter. */
2718         bnx2x_set_rx_mode_inner(bp);
2719
2720         /* Start the Tx */
2721         switch (load_mode) {
2722         case LOAD_NORMAL:
2723                 /* Tx queue should be only re-enabled */
2724                 netif_tx_wake_all_queues(bp->dev);
2725                 break;
2726
2727         case LOAD_OPEN:
2728                 netif_tx_start_all_queues(bp->dev);
2729                 smp_mb__after_clear_bit();
2730                 break;
2731
2732         case LOAD_DIAG:
2733         case LOAD_LOOPBACK_EXT:
2734                 bp->state = BNX2X_STATE_DIAG;
2735                 break;
2736
2737         default:
2738                 break;
2739         }
2740
2741         if (bp->port.pmf)
2742                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2743         else
2744                 bnx2x__link_status_update(bp);
2745
2746         /* start the timer */
2747         mod_timer(&bp->timer, jiffies + bp->current_interval);
2748
2749         if (CNIC_ENABLED(bp))
2750                 bnx2x_load_cnic(bp);
2751
2752         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2753                 /* mark driver is loaded in shmem2 */
2754                 u32 val;
2755                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2756                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2757                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2758                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2759         }
2760
2761         /* Wait for all pending SP commands to complete */
2762         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2763                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2764                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2765                 return -EBUSY;
2766         }
2767
2768         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2769         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2770                 bnx2x_dcbx_init(bp, false);
2771
2772         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2773
2774         return 0;
2775
2776 #ifndef BNX2X_STOP_ON_ERROR
2777 load_error3:
2778         if (IS_PF(bp)) {
2779                 bnx2x_int_disable_sync(bp, 1);
2780
2781                 /* Clean queueable objects */
2782                 bnx2x_squeeze_objects(bp);
2783         }
2784
2785         /* Free SKBs, SGEs, TPA pool and driver internals */
2786         bnx2x_free_skbs(bp);
2787         for_each_rx_queue(bp, i)
2788                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2789
2790         /* Release IRQs */
2791         bnx2x_free_irq(bp);
2792 load_error2:
2793         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2794                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2795                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2796         }
2797
2798         bp->port.pmf = 0;
2799 load_error1:
2800         bnx2x_napi_disable(bp);
2801         bnx2x_del_all_napi(bp);
2802
2803         /* clear pf_load status, as it was already set */
2804         if (IS_PF(bp))
2805                 bnx2x_clear_pf_load(bp);
2806 load_error0:
2807         bnx2x_free_fp_mem(bp);
2808         bnx2x_free_fw_stats_mem(bp);
2809         bnx2x_free_mem(bp);
2810
2811         return rc;
2812 #endif /* ! BNX2X_STOP_ON_ERROR */
2813 }
2814
2815 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2816 {
2817         u8 rc = 0, cos, i;
2818
2819         /* Wait until tx fastpath tasks complete */
2820         for_each_tx_queue(bp, i) {
2821                 struct bnx2x_fastpath *fp = &bp->fp[i];
2822
2823                 for_each_cos_in_tx_queue(fp, cos)
2824                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2825                 if (rc)
2826                         return rc;
2827         }
2828         return 0;
2829 }
2830
2831 /* must be called with rtnl_lock */
2832 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2833 {
2834         int i;
2835         bool global = false;
2836
2837         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2838
2839         /* mark driver is unloaded in shmem2 */
2840         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2841                 u32 val;
2842                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2843                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2844                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2845         }
2846
2847         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2848             (bp->state == BNX2X_STATE_CLOSED ||
2849              bp->state == BNX2X_STATE_ERROR)) {
2850                 /* We can get here if the driver has been unloaded
2851                  * during parity error recovery and is either waiting for a
2852                  * leader to complete or for other functions to unload and
2853                  * then ifdown has been issued. In this case we want to
2854                  * unload and let other functions to complete a recovery
2855                  * process.
2856                  */
2857                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2858                 bp->is_leader = 0;
2859                 bnx2x_release_leader_lock(bp);
2860                 smp_mb();
2861
2862                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2863                 BNX2X_ERR("Can't unload in closed or error state\n");
2864                 return -EINVAL;
2865         }
2866
2867         /* Nothing to do during unload if previous bnx2x_nic_load()
2868          * have not completed successfully - all resources are released.
2869          *
2870          * we can get here only after unsuccessful ndo_* callback, during which
2871          * dev->IFF_UP flag is still on.
2872          */
2873         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2874                 return 0;
2875
2876         /* It's important to set the bp->state to the value different from
2877          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2878          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2879          */
2880         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2881         smp_mb();
2882
2883         /* indicate to VFs that the PF is going down */
2884         bnx2x_iov_channel_down(bp);
2885
2886         if (CNIC_LOADED(bp))
2887                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2888
2889         /* Stop Tx */
2890         bnx2x_tx_disable(bp);
2891         netdev_reset_tc(bp->dev);
2892
2893         bp->rx_mode = BNX2X_RX_MODE_NONE;
2894
2895         del_timer_sync(&bp->timer);
2896
2897         if (IS_PF(bp)) {
2898                 /* Set ALWAYS_ALIVE bit in shmem */
2899                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2900                 bnx2x_drv_pulse(bp);
2901                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2902                 bnx2x_save_statistics(bp);
2903         }
2904
2905         /* wait till consumers catch up with producers in all queues */
2906         bnx2x_drain_tx_queues(bp);
2907
2908         /* if VF indicate to PF this function is going down (PF will delete sp
2909          * elements and clear initializations
2910          */
2911         if (IS_VF(bp))
2912                 bnx2x_vfpf_close_vf(bp);
2913         else if (unload_mode != UNLOAD_RECOVERY)
2914                 /* if this is a normal/close unload need to clean up chip*/
2915                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2916         else {
2917                 /* Send the UNLOAD_REQUEST to the MCP */
2918                 bnx2x_send_unload_req(bp, unload_mode);
2919
2920                 /* Prevent transactions to host from the functions on the
2921                  * engine that doesn't reset global blocks in case of global
2922                  * attention once global blocks are reset and gates are opened
2923                  * (the engine which leader will perform the recovery
2924                  * last).
2925                  */
2926                 if (!CHIP_IS_E1x(bp))
2927                         bnx2x_pf_disable(bp);
2928
2929                 /* Disable HW interrupts, NAPI */
2930                 bnx2x_netif_stop(bp, 1);
2931                 /* Delete all NAPI objects */
2932                 bnx2x_del_all_napi(bp);
2933                 if (CNIC_LOADED(bp))
2934                         bnx2x_del_all_napi_cnic(bp);
2935                 /* Release IRQs */
2936                 bnx2x_free_irq(bp);
2937
2938                 /* Report UNLOAD_DONE to MCP */
2939                 bnx2x_send_unload_done(bp, false);
2940         }
2941
2942         /*
2943          * At this stage no more interrupts will arrive so we may safely clean
2944          * the queueable objects here in case they failed to get cleaned so far.
2945          */
2946         if (IS_PF(bp))
2947                 bnx2x_squeeze_objects(bp);
2948
2949         /* There should be no more pending SP commands at this stage */
2950         bp->sp_state = 0;
2951
2952         bp->port.pmf = 0;
2953
2954         /* Free SKBs, SGEs, TPA pool and driver internals */
2955         bnx2x_free_skbs(bp);
2956         if (CNIC_LOADED(bp))
2957                 bnx2x_free_skbs_cnic(bp);
2958         for_each_rx_queue(bp, i)
2959                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2960
2961         bnx2x_free_fp_mem(bp);
2962         if (CNIC_LOADED(bp))
2963                 bnx2x_free_fp_mem_cnic(bp);
2964
2965         if (IS_PF(bp)) {
2966                 if (CNIC_LOADED(bp))
2967                         bnx2x_free_mem_cnic(bp);
2968         }
2969         bnx2x_free_mem(bp);
2970
2971         bp->state = BNX2X_STATE_CLOSED;
2972         bp->cnic_loaded = false;
2973
2974         /* Check if there are pending parity attentions. If there are - set
2975          * RECOVERY_IN_PROGRESS.
2976          */
2977         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2978                 bnx2x_set_reset_in_progress(bp);
2979
2980                 /* Set RESET_IS_GLOBAL if needed */
2981                 if (global)
2982                         bnx2x_set_reset_global(bp);
2983         }
2984
2985         /* The last driver must disable a "close the gate" if there is no
2986          * parity attention or "process kill" pending.
2987          */
2988         if (IS_PF(bp) &&
2989             !bnx2x_clear_pf_load(bp) &&
2990             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2991                 bnx2x_disable_close_the_gate(bp);
2992
2993         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2994
2995         return 0;
2996 }
2997
2998 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2999 {
3000         u16 pmcsr;
3001
3002         /* If there is no power capability, silently succeed */
3003         if (!bp->pm_cap) {
3004                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3005                 return 0;
3006         }
3007
3008         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
3009
3010         switch (state) {
3011         case PCI_D0:
3012                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3013                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3014                                        PCI_PM_CTRL_PME_STATUS));
3015
3016                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3017                         /* delay required during transition out of D3hot */
3018                         msleep(20);
3019                 break;
3020
3021         case PCI_D3hot:
3022                 /* If there are other clients above don't
3023                    shut down the power */
3024                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3025                         return 0;
3026                 /* Don't shut down the power for emulation and FPGA */
3027                 if (CHIP_REV_IS_SLOW(bp))
3028                         return 0;
3029
3030                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3031                 pmcsr |= 3;
3032
3033                 if (bp->wol)
3034                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3035
3036                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
3037                                       pmcsr);
3038
3039                 /* No more memory access after this point until
3040                 * device is brought back to D0.
3041                 */
3042                 break;
3043
3044         default:
3045                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3046                 return -EINVAL;
3047         }
3048         return 0;
3049 }
3050
3051 /*
3052  * net_device service functions
3053  */
3054 int bnx2x_poll(struct napi_struct *napi, int budget)
3055 {
3056         int work_done = 0;
3057         u8 cos;
3058         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3059                                                  napi);
3060         struct bnx2x *bp = fp->bp;
3061
3062         while (1) {
3063 #ifdef BNX2X_STOP_ON_ERROR
3064                 if (unlikely(bp->panic)) {
3065                         napi_complete(napi);
3066                         return 0;
3067                 }
3068 #endif
3069                 if (!bnx2x_fp_lock_napi(fp))
3070                         return work_done;
3071
3072                 for_each_cos_in_tx_queue(fp, cos)
3073                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3074                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3075
3076                 if (bnx2x_has_rx_work(fp)) {
3077                         work_done += bnx2x_rx_int(fp, budget - work_done);
3078
3079                         /* must not complete if we consumed full budget */
3080                         if (work_done >= budget) {
3081                                 bnx2x_fp_unlock_napi(fp);
3082                                 break;
3083                         }
3084                 }
3085
3086                 /* Fall out from the NAPI loop if needed */
3087                 if (!bnx2x_fp_unlock_napi(fp) &&
3088                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3089
3090                         /* No need to update SB for FCoE L2 ring as long as
3091                          * it's connected to the default SB and the SB
3092                          * has been updated when NAPI was scheduled.
3093                          */
3094                         if (IS_FCOE_FP(fp)) {
3095                                 napi_complete(napi);
3096                                 break;
3097                         }
3098                         bnx2x_update_fpsb_idx(fp);
3099                         /* bnx2x_has_rx_work() reads the status block,
3100                          * thus we need to ensure that status block indices
3101                          * have been actually read (bnx2x_update_fpsb_idx)
3102                          * prior to this check (bnx2x_has_rx_work) so that
3103                          * we won't write the "newer" value of the status block
3104                          * to IGU (if there was a DMA right after
3105                          * bnx2x_has_rx_work and if there is no rmb, the memory
3106                          * reading (bnx2x_update_fpsb_idx) may be postponed
3107                          * to right before bnx2x_ack_sb). In this case there
3108                          * will never be another interrupt until there is
3109                          * another update of the status block, while there
3110                          * is still unhandled work.
3111                          */
3112                         rmb();
3113
3114                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3115                                 napi_complete(napi);
3116                                 /* Re-enable interrupts */
3117                                 DP(NETIF_MSG_RX_STATUS,
3118                                    "Update index to %d\n", fp->fp_hc_idx);
3119                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3120                                              le16_to_cpu(fp->fp_hc_idx),
3121                                              IGU_INT_ENABLE, 1);
3122                                 break;
3123                         }
3124                 }
3125         }
3126
3127         return work_done;
3128 }
3129
3130 #ifdef CONFIG_NET_RX_BUSY_POLL
3131 /* must be called with local_bh_disable()d */
3132 int bnx2x_low_latency_recv(struct napi_struct *napi)
3133 {
3134         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3135                                                  napi);
3136         struct bnx2x *bp = fp->bp;
3137         int found = 0;
3138
3139         if ((bp->state == BNX2X_STATE_CLOSED) ||
3140             (bp->state == BNX2X_STATE_ERROR) ||
3141             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3142                 return LL_FLUSH_FAILED;
3143
3144         if (!bnx2x_fp_lock_poll(fp))
3145                 return LL_FLUSH_BUSY;
3146
3147         if (bnx2x_has_rx_work(fp))
3148                 found = bnx2x_rx_int(fp, 4);
3149
3150         bnx2x_fp_unlock_poll(fp);
3151
3152         return found;
3153 }
3154 #endif
3155
3156 /* we split the first BD into headers and data BDs
3157  * to ease the pain of our fellow microcode engineers
3158  * we use one mapping for both BDs
3159  */
3160 static u16 bnx2x_tx_split(struct bnx2x *bp,
3161                           struct bnx2x_fp_txdata *txdata,
3162                           struct sw_tx_bd *tx_buf,
3163                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3164                           u16 bd_prod)
3165 {
3166         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3167         struct eth_tx_bd *d_tx_bd;
3168         dma_addr_t mapping;
3169         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3170
3171         /* first fix first BD */
3172         h_tx_bd->nbytes = cpu_to_le16(hlen);
3173
3174         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3175            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3176
3177         /* now get a new data BD
3178          * (after the pbd) and fill it */
3179         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3180         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3181
3182         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3183                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3184
3185         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3186         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3187         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3188
3189         /* this marks the BD as one that has no individual mapping */
3190         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3191
3192         DP(NETIF_MSG_TX_QUEUED,
3193            "TSO split data size is %d (%x:%x)\n",
3194            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3195
3196         /* update tx_bd */
3197         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3198
3199         return bd_prod;
3200 }
3201
3202 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3203 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3204 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3205 {
3206         __sum16 tsum = (__force __sum16) csum;
3207
3208         if (fix > 0)
3209                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3210                                   csum_partial(t_header - fix, fix, 0)));
3211
3212         else if (fix < 0)
3213                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3214                                   csum_partial(t_header, -fix, 0)));
3215
3216         return bswab16(tsum);
3217 }
3218
3219 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3220 {
3221         u32 rc;
3222         __u8 prot = 0;
3223         __be16 protocol;
3224
3225         if (skb->ip_summed != CHECKSUM_PARTIAL)
3226                 return XMIT_PLAIN;
3227
3228         protocol = vlan_get_protocol(skb);
3229         if (protocol == htons(ETH_P_IPV6)) {
3230                 rc = XMIT_CSUM_V6;
3231                 prot = ipv6_hdr(skb)->nexthdr;
3232         } else {
3233                 rc = XMIT_CSUM_V4;
3234                 prot = ip_hdr(skb)->protocol;
3235         }
3236
3237         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3238                 if (inner_ip_hdr(skb)->version == 6) {
3239                         rc |= XMIT_CSUM_ENC_V6;
3240                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3241                                 rc |= XMIT_CSUM_TCP;
3242                 } else {
3243                         rc |= XMIT_CSUM_ENC_V4;
3244                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3245                                 rc |= XMIT_CSUM_TCP;
3246                 }
3247         }
3248         if (prot == IPPROTO_TCP)
3249                 rc |= XMIT_CSUM_TCP;
3250
3251         if (skb_is_gso_v6(skb)) {
3252                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3253                 if (rc & XMIT_CSUM_ENC)
3254                         rc |= XMIT_GSO_ENC_V6;
3255         } else if (skb_is_gso(skb)) {
3256                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3257                 if (rc & XMIT_CSUM_ENC)
3258                         rc |= XMIT_GSO_ENC_V4;
3259         }
3260
3261         return rc;
3262 }
3263
3264 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3265 /* check if packet requires linearization (packet is too fragmented)
3266    no need to check fragmentation if page size > 8K (there will be no
3267    violation to FW restrictions) */
3268 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3269                              u32 xmit_type)
3270 {
3271         int to_copy = 0;
3272         int hlen = 0;
3273         int first_bd_sz = 0;
3274
3275         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3276         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3277
3278                 if (xmit_type & XMIT_GSO) {
3279                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3280                         /* Check if LSO packet needs to be copied:
3281                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3282                         int wnd_size = MAX_FETCH_BD - 3;
3283                         /* Number of windows to check */
3284                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3285                         int wnd_idx = 0;
3286                         int frag_idx = 0;
3287                         u32 wnd_sum = 0;
3288
3289                         /* Headers length */
3290                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3291                                 tcp_hdrlen(skb);
3292
3293                         /* Amount of data (w/o headers) on linear part of SKB*/
3294                         first_bd_sz = skb_headlen(skb) - hlen;
3295
3296                         wnd_sum  = first_bd_sz;
3297
3298                         /* Calculate the first sum - it's special */
3299                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3300                                 wnd_sum +=
3301                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3302
3303                         /* If there was data on linear skb data - check it */
3304                         if (first_bd_sz > 0) {
3305                                 if (unlikely(wnd_sum < lso_mss)) {
3306                                         to_copy = 1;
3307                                         goto exit_lbl;
3308                                 }
3309
3310                                 wnd_sum -= first_bd_sz;
3311                         }
3312
3313                         /* Others are easier: run through the frag list and
3314                            check all windows */
3315                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3316                                 wnd_sum +=
3317                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3318
3319                                 if (unlikely(wnd_sum < lso_mss)) {
3320                                         to_copy = 1;
3321                                         break;
3322                                 }
3323                                 wnd_sum -=
3324                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3325                         }
3326                 } else {
3327                         /* in non-LSO too fragmented packet should always
3328                            be linearized */
3329                         to_copy = 1;
3330                 }
3331         }
3332
3333 exit_lbl:
3334         if (unlikely(to_copy))
3335                 DP(NETIF_MSG_TX_QUEUED,
3336                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3337                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3338                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3339
3340         return to_copy;
3341 }
3342 #endif
3343
3344 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3345                                  u32 xmit_type)
3346 {
3347         struct ipv6hdr *ipv6;
3348
3349         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3350                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3351                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3352
3353         if (xmit_type & XMIT_GSO_ENC_V6)
3354                 ipv6 = inner_ipv6_hdr(skb);
3355         else if (xmit_type & XMIT_GSO_V6)
3356                 ipv6 = ipv6_hdr(skb);
3357         else
3358                 ipv6 = NULL;
3359
3360         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3361                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3362 }
3363
3364 /**
3365  * bnx2x_set_pbd_gso - update PBD in GSO case.
3366  *
3367  * @skb:        packet skb
3368  * @pbd:        parse BD
3369  * @xmit_type:  xmit flags
3370  */
3371 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3372                               struct eth_tx_parse_bd_e1x *pbd,
3373                               struct eth_tx_start_bd *tx_start_bd,
3374                               u32 xmit_type)
3375 {
3376         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3377         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3378         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3379
3380         if (xmit_type & XMIT_GSO_V4) {
3381                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3382                 pbd->tcp_pseudo_csum =
3383                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3384                                                    ip_hdr(skb)->daddr,
3385                                                    0, IPPROTO_TCP, 0));
3386
3387                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3388                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3389         } else {
3390                 pbd->tcp_pseudo_csum =
3391                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3392                                                  &ipv6_hdr(skb)->daddr,
3393                                                  0, IPPROTO_TCP, 0));
3394         }
3395
3396         pbd->global_data |=
3397                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3398 }
3399
3400 /**
3401  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3402  *
3403  * @bp:                 driver handle
3404  * @skb:                packet skb
3405  * @parsing_data:       data to be updated
3406  * @xmit_type:          xmit flags
3407  *
3408  * 57712/578xx related, when skb has encapsulation
3409  */
3410 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3411                                  u32 *parsing_data, u32 xmit_type)
3412 {
3413         *parsing_data |=
3414                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3415                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3416                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3417
3418         if (xmit_type & XMIT_CSUM_TCP) {
3419                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3420                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3421                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3422
3423                 return skb_inner_transport_header(skb) +
3424                         inner_tcp_hdrlen(skb) - skb->data;
3425         }
3426
3427         /* We support checksum offload for TCP and UDP only.
3428          * No need to pass the UDP header length - it's a constant.
3429          */
3430         return skb_inner_transport_header(skb) +
3431                 sizeof(struct udphdr) - skb->data;
3432 }
3433
3434 /**
3435  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3436  *
3437  * @bp:                 driver handle
3438  * @skb:                packet skb
3439  * @parsing_data:       data to be updated
3440  * @xmit_type:          xmit flags
3441  *
3442  * 57712/578xx related
3443  */
3444 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3445                                 u32 *parsing_data, u32 xmit_type)
3446 {
3447         *parsing_data |=
3448                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3449                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3450                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3451
3452         if (xmit_type & XMIT_CSUM_TCP) {
3453                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3454                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3455                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3456
3457                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3458         }
3459         /* We support checksum offload for TCP and UDP only.
3460          * No need to pass the UDP header length - it's a constant.
3461          */
3462         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3463 }
3464
3465 /* set FW indication according to inner or outer protocols if tunneled */
3466 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3467                                struct eth_tx_start_bd *tx_start_bd,
3468                                u32 xmit_type)
3469 {
3470         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3471
3472         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3473                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3474
3475         if (!(xmit_type & XMIT_CSUM_TCP))
3476                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3477 }
3478
3479 /**
3480  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3481  *
3482  * @bp:         driver handle
3483  * @skb:        packet skb
3484  * @pbd:        parse BD to be updated
3485  * @xmit_type:  xmit flags
3486  */
3487 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3488                              struct eth_tx_parse_bd_e1x *pbd,
3489                              u32 xmit_type)
3490 {
3491         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3492
3493         /* for now NS flag is not used in Linux */
3494         pbd->global_data =
3495                 cpu_to_le16(hlen |
3496                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3497                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3498
3499         pbd->ip_hlen_w = (skb_transport_header(skb) -
3500                         skb_network_header(skb)) >> 1;
3501
3502         hlen += pbd->ip_hlen_w;
3503
3504         /* We support checksum offload for TCP and UDP only */
3505         if (xmit_type & XMIT_CSUM_TCP)
3506                 hlen += tcp_hdrlen(skb) / 2;
3507         else
3508                 hlen += sizeof(struct udphdr) / 2;
3509
3510         pbd->total_hlen_w = cpu_to_le16(hlen);
3511         hlen = hlen*2;
3512
3513         if (xmit_type & XMIT_CSUM_TCP) {
3514                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3515
3516         } else {
3517                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3518
3519                 DP(NETIF_MSG_TX_QUEUED,
3520                    "hlen %d  fix %d  csum before fix %x\n",
3521                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3522
3523                 /* HW bug: fixup the CSUM */
3524                 pbd->tcp_pseudo_csum =
3525                         bnx2x_csum_fix(skb_transport_header(skb),
3526                                        SKB_CS(skb), fix);
3527
3528                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3529                    pbd->tcp_pseudo_csum);
3530         }
3531
3532         return hlen;
3533 }
3534
3535 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3536                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3537                                       struct eth_tx_parse_2nd_bd *pbd2,
3538                                       u16 *global_data,
3539                                       u32 xmit_type)
3540 {
3541         u16 hlen_w = 0;
3542         u8 outerip_off, outerip_len = 0;
3543
3544         /* from outer IP to transport */
3545         hlen_w = (skb_inner_transport_header(skb) -
3546                   skb_network_header(skb)) >> 1;
3547
3548         /* transport len */
3549         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3550
3551         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3552
3553         /* outer IP header info */
3554         if (xmit_type & XMIT_CSUM_V4) {
3555                 struct iphdr *iph = ip_hdr(skb);
3556                 u32 csum = (__force u32)(~iph->check) -
3557                            (__force u32)iph->tot_len -
3558                            (__force u32)iph->frag_off;
3559
3560                 pbd2->fw_ip_csum_wo_len_flags_frag =
3561                         bswab16(csum_fold((__force __wsum)csum));
3562         } else {
3563                 pbd2->fw_ip_hdr_to_payload_w =
3564                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3565         }
3566
3567         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3568
3569         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3570
3571         if (xmit_type & XMIT_GSO_V4) {
3572                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3573
3574                 pbd_e2->data.tunnel_data.pseudo_csum =
3575                         bswab16(~csum_tcpudp_magic(
3576                                         inner_ip_hdr(skb)->saddr,
3577                                         inner_ip_hdr(skb)->daddr,
3578                                         0, IPPROTO_TCP, 0));
3579
3580                 outerip_len = ip_hdr(skb)->ihl << 1;
3581         } else {
3582                 pbd_e2->data.tunnel_data.pseudo_csum =
3583                         bswab16(~csum_ipv6_magic(
3584                                         &inner_ipv6_hdr(skb)->saddr,
3585                                         &inner_ipv6_hdr(skb)->daddr,
3586                                         0, IPPROTO_TCP, 0));
3587         }
3588
3589         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3590
3591         *global_data |=
3592                 outerip_off |
3593                 (!!(xmit_type & XMIT_CSUM_V6) <<
3594                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3595                 (outerip_len <<
3596                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3597                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3598                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3599
3600         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3601                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3602                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3603         }
3604 }
3605
3606 /* called with netif_tx_lock
3607  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3608  * netif_wake_queue()
3609  */
3610 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3611 {
3612         struct bnx2x *bp = netdev_priv(dev);
3613
3614         struct netdev_queue *txq;
3615         struct bnx2x_fp_txdata *txdata;
3616         struct sw_tx_bd *tx_buf;
3617         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3618         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3619         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3620         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3621         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3622         u32 pbd_e2_parsing_data = 0;
3623         u16 pkt_prod, bd_prod;
3624         int nbd, txq_index;
3625         dma_addr_t mapping;
3626         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3627         int i;
3628         u8 hlen = 0;
3629         __le16 pkt_size = 0;
3630         struct ethhdr *eth;
3631         u8 mac_type = UNICAST_ADDRESS;
3632
3633 #ifdef BNX2X_STOP_ON_ERROR
3634         if (unlikely(bp->panic))
3635                 return NETDEV_TX_BUSY;
3636 #endif
3637
3638         txq_index = skb_get_queue_mapping(skb);
3639         txq = netdev_get_tx_queue(dev, txq_index);
3640
3641         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3642
3643         txdata = &bp->bnx2x_txq[txq_index];
3644
3645         /* enable this debug print to view the transmission queue being used
3646         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3647            txq_index, fp_index, txdata_index); */
3648
3649         /* enable this debug print to view the transmission details
3650         DP(NETIF_MSG_TX_QUEUED,
3651            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3652            txdata->cid, fp_index, txdata_index, txdata, fp); */
3653
3654         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3655                         skb_shinfo(skb)->nr_frags +
3656                         BDS_PER_TX_PKT +
3657                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3658                 /* Handle special storage cases separately */
3659                 if (txdata->tx_ring_size == 0) {
3660                         struct bnx2x_eth_q_stats *q_stats =
3661                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3662                         q_stats->driver_filtered_tx_pkt++;
3663                         dev_kfree_skb(skb);
3664                         return NETDEV_TX_OK;
3665                 }
3666                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3667                 netif_tx_stop_queue(txq);
3668                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3669
3670                 return NETDEV_TX_BUSY;
3671         }
3672
3673         DP(NETIF_MSG_TX_QUEUED,
3674            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3675            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3676            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3677            skb->len);
3678
3679         eth = (struct ethhdr *)skb->data;
3680
3681         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3682         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3683                 if (is_broadcast_ether_addr(eth->h_dest))
3684                         mac_type = BROADCAST_ADDRESS;
3685                 else
3686                         mac_type = MULTICAST_ADDRESS;
3687         }
3688
3689 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3690         /* First, check if we need to linearize the skb (due to FW
3691            restrictions). No need to check fragmentation if page size > 8K
3692            (there will be no violation to FW restrictions) */
3693         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3694                 /* Statistics of linearization */
3695                 bp->lin_cnt++;
3696                 if (skb_linearize(skb) != 0) {
3697                         DP(NETIF_MSG_TX_QUEUED,
3698                            "SKB linearization failed - silently dropping this SKB\n");
3699                         dev_kfree_skb_any(skb);
3700                         return NETDEV_TX_OK;
3701                 }
3702         }
3703 #endif
3704         /* Map skb linear data for DMA */
3705         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3706                                  skb_headlen(skb), DMA_TO_DEVICE);
3707         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3708                 DP(NETIF_MSG_TX_QUEUED,
3709                    "SKB mapping failed - silently dropping this SKB\n");
3710                 dev_kfree_skb_any(skb);
3711                 return NETDEV_TX_OK;
3712         }
3713         /*
3714         Please read carefully. First we use one BD which we mark as start,
3715         then we have a parsing info BD (used for TSO or xsum),
3716         and only then we have the rest of the TSO BDs.
3717         (don't forget to mark the last one as last,
3718         and to unmap only AFTER you write to the BD ...)
3719         And above all, all pdb sizes are in words - NOT DWORDS!
3720         */
3721
3722         /* get current pkt produced now - advance it just before sending packet
3723          * since mapping of pages may fail and cause packet to be dropped
3724          */
3725         pkt_prod = txdata->tx_pkt_prod;
3726         bd_prod = TX_BD(txdata->tx_bd_prod);
3727
3728         /* get a tx_buf and first BD
3729          * tx_start_bd may be changed during SPLIT,
3730          * but first_bd will always stay first
3731          */
3732         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3733         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3734         first_bd = tx_start_bd;
3735
3736         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3737
3738         /* header nbd: indirectly zero other flags! */
3739         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3740
3741         /* remember the first BD of the packet */
3742         tx_buf->first_bd = txdata->tx_bd_prod;
3743         tx_buf->skb = skb;
3744         tx_buf->flags = 0;
3745
3746         DP(NETIF_MSG_TX_QUEUED,
3747            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3748            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3749
3750         if (vlan_tx_tag_present(skb)) {
3751                 tx_start_bd->vlan_or_ethertype =
3752                     cpu_to_le16(vlan_tx_tag_get(skb));
3753                 tx_start_bd->bd_flags.as_bitfield |=
3754                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3755         } else {
3756                 /* when transmitting in a vf, start bd must hold the ethertype
3757                  * for fw to enforce it
3758                  */
3759                 if (IS_VF(bp))
3760                         tx_start_bd->vlan_or_ethertype =
3761                                 cpu_to_le16(ntohs(eth->h_proto));
3762                 else
3763                         /* used by FW for packet accounting */
3764                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3765         }
3766
3767         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3768
3769         /* turn on parsing and get a BD */
3770         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3771
3772         if (xmit_type & XMIT_CSUM)
3773                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3774
3775         if (!CHIP_IS_E1x(bp)) {
3776                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3777                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3778
3779                 if (xmit_type & XMIT_CSUM_ENC) {
3780                         u16 global_data = 0;
3781
3782                         /* Set PBD in enc checksum offload case */
3783                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3784                                                       &pbd_e2_parsing_data,
3785                                                       xmit_type);
3786
3787                         /* turn on 2nd parsing and get a BD */
3788                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3789
3790                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3791
3792                         memset(pbd2, 0, sizeof(*pbd2));
3793
3794                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3795                                 (skb_inner_network_header(skb) -
3796                                  skb->data) >> 1;
3797
3798                         if (xmit_type & XMIT_GSO_ENC)
3799                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3800                                                           &global_data,
3801                                                           xmit_type);
3802
3803                         pbd2->global_data = cpu_to_le16(global_data);
3804
3805                         /* add addition parse BD indication to start BD */
3806                         SET_FLAG(tx_start_bd->general_data,
3807                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3808                         /* set encapsulation flag in start BD */
3809                         SET_FLAG(tx_start_bd->general_data,
3810                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3811                         nbd++;
3812                 } else if (xmit_type & XMIT_CSUM) {
3813                         /* Set PBD in checksum offload case w/o encapsulation */
3814                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3815                                                      &pbd_e2_parsing_data,
3816                                                      xmit_type);
3817                 }
3818
3819                 /* Add the macs to the parsing BD this is a vf */
3820                 if (IS_VF(bp)) {
3821                         /* override GRE parameters in BD */
3822                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3823                                               &pbd_e2->data.mac_addr.src_mid,
3824                                               &pbd_e2->data.mac_addr.src_lo,
3825                                               eth->h_source);
3826
3827                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3828                                               &pbd_e2->data.mac_addr.dst_mid,
3829                                               &pbd_e2->data.mac_addr.dst_lo,
3830                                               eth->h_dest);
3831                 }
3832
3833                 SET_FLAG(pbd_e2_parsing_data,
3834                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3835         } else {
3836                 u16 global_data = 0;
3837                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3838                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3839                 /* Set PBD in checksum offload case */
3840                 if (xmit_type & XMIT_CSUM)
3841                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3842
3843                 SET_FLAG(global_data,
3844                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3845                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3846         }
3847
3848         /* Setup the data pointer of the first BD of the packet */
3849         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3850         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3851         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3852         pkt_size = tx_start_bd->nbytes;
3853
3854         DP(NETIF_MSG_TX_QUEUED,
3855            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3856            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3857            le16_to_cpu(tx_start_bd->nbytes),
3858            tx_start_bd->bd_flags.as_bitfield,
3859            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3860
3861         if (xmit_type & XMIT_GSO) {
3862
3863                 DP(NETIF_MSG_TX_QUEUED,
3864                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3865                    skb->len, hlen, skb_headlen(skb),
3866                    skb_shinfo(skb)->gso_size);
3867
3868                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3869
3870                 if (unlikely(skb_headlen(skb) > hlen)) {
3871                         nbd++;
3872                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3873                                                  &tx_start_bd, hlen,
3874                                                  bd_prod);
3875                 }
3876                 if (!CHIP_IS_E1x(bp))
3877                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3878                                              xmit_type);
3879                 else
3880                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3881         }
3882
3883         /* Set the PBD's parsing_data field if not zero
3884          * (for the chips newer than 57711).
3885          */
3886         if (pbd_e2_parsing_data)
3887                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3888
3889         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3890
3891         /* Handle fragmented skb */
3892         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3893                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3894
3895                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3896                                            skb_frag_size(frag), DMA_TO_DEVICE);
3897                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3898                         unsigned int pkts_compl = 0, bytes_compl = 0;
3899
3900                         DP(NETIF_MSG_TX_QUEUED,
3901                            "Unable to map page - dropping packet...\n");
3902
3903                         /* we need unmap all buffers already mapped
3904                          * for this SKB;
3905                          * first_bd->nbd need to be properly updated
3906                          * before call to bnx2x_free_tx_pkt
3907                          */
3908                         first_bd->nbd = cpu_to_le16(nbd);
3909                         bnx2x_free_tx_pkt(bp, txdata,
3910                                           TX_BD(txdata->tx_pkt_prod),
3911                                           &pkts_compl, &bytes_compl);
3912                         return NETDEV_TX_OK;
3913                 }
3914
3915                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3916                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3917                 if (total_pkt_bd == NULL)
3918                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3919
3920                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3921                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3922                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3923                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3924                 nbd++;
3925
3926                 DP(NETIF_MSG_TX_QUEUED,
3927                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3928                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3929                    le16_to_cpu(tx_data_bd->nbytes));
3930         }
3931
3932         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3933
3934         /* update with actual num BDs */
3935         first_bd->nbd = cpu_to_le16(nbd);
3936
3937         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3938
3939         /* now send a tx doorbell, counting the next BD
3940          * if the packet contains or ends with it
3941          */
3942         if (TX_BD_POFF(bd_prod) < nbd)
3943                 nbd++;
3944
3945         /* total_pkt_bytes should be set on the first data BD if
3946          * it's not an LSO packet and there is more than one
3947          * data BD. In this case pkt_size is limited by an MTU value.
3948          * However we prefer to set it for an LSO packet (while we don't
3949          * have to) in order to save some CPU cycles in a none-LSO
3950          * case, when we much more care about them.
3951          */
3952         if (total_pkt_bd != NULL)
3953                 total_pkt_bd->total_pkt_bytes = pkt_size;
3954
3955         if (pbd_e1x)
3956                 DP(NETIF_MSG_TX_QUEUED,
3957                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3958                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3959                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3960                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3961                     le16_to_cpu(pbd_e1x->total_hlen_w));
3962         if (pbd_e2)
3963                 DP(NETIF_MSG_TX_QUEUED,
3964                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3965                    pbd_e2,
3966                    pbd_e2->data.mac_addr.dst_hi,
3967                    pbd_e2->data.mac_addr.dst_mid,
3968                    pbd_e2->data.mac_addr.dst_lo,
3969                    pbd_e2->data.mac_addr.src_hi,
3970                    pbd_e2->data.mac_addr.src_mid,
3971                    pbd_e2->data.mac_addr.src_lo,
3972                    pbd_e2->parsing_data);
3973         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3974
3975         netdev_tx_sent_queue(txq, skb->len);
3976
3977         skb_tx_timestamp(skb);
3978
3979         txdata->tx_pkt_prod++;
3980         /*
3981          * Make sure that the BD data is updated before updating the producer
3982          * since FW might read the BD right after the producer is updated.
3983          * This is only applicable for weak-ordered memory model archs such
3984          * as IA-64. The following barrier is also mandatory since FW will
3985          * assumes packets must have BDs.
3986          */
3987         wmb();
3988
3989         txdata->tx_db.data.prod += nbd;
3990         barrier();
3991
3992         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3993
3994         mmiowb();
3995
3996         txdata->tx_bd_prod += nbd;
3997
3998         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3999                 netif_tx_stop_queue(txq);
4000
4001                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4002                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4003                  * fp->bd_tx_cons */
4004                 smp_mb();
4005
4006                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4007                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4008                         netif_tx_wake_queue(txq);
4009         }
4010         txdata->tx_pkt++;
4011
4012         return NETDEV_TX_OK;
4013 }
4014
4015 /**
4016  * bnx2x_setup_tc - routine to configure net_device for multi tc
4017  *
4018  * @netdev: net device to configure
4019  * @tc: number of traffic classes to enable
4020  *
4021  * callback connected to the ndo_setup_tc function pointer
4022  */
4023 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4024 {
4025         int cos, prio, count, offset;
4026         struct bnx2x *bp = netdev_priv(dev);
4027
4028         /* setup tc must be called under rtnl lock */
4029         ASSERT_RTNL();
4030
4031         /* no traffic classes requested. Aborting */
4032         if (!num_tc) {
4033                 netdev_reset_tc(dev);
4034                 return 0;
4035         }
4036
4037         /* requested to support too many traffic classes */
4038         if (num_tc > bp->max_cos) {
4039                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4040                           num_tc, bp->max_cos);
4041                 return -EINVAL;
4042         }
4043
4044         /* declare amount of supported traffic classes */
4045         if (netdev_set_num_tc(dev, num_tc)) {
4046                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4047                 return -EINVAL;
4048         }
4049
4050         /* configure priority to traffic class mapping */
4051         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4052                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4053                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4054                    "mapping priority %d to tc %d\n",
4055                    prio, bp->prio_to_cos[prio]);
4056         }
4057
4058         /* Use this configuration to differentiate tc0 from other COSes
4059            This can be used for ets or pfc, and save the effort of setting
4060            up a multio class queue disc or negotiating DCBX with a switch
4061         netdev_set_prio_tc_map(dev, 0, 0);
4062         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4063         for (prio = 1; prio < 16; prio++) {
4064                 netdev_set_prio_tc_map(dev, prio, 1);
4065                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4066         } */
4067
4068         /* configure traffic class to transmission queue mapping */
4069         for (cos = 0; cos < bp->max_cos; cos++) {
4070                 count = BNX2X_NUM_ETH_QUEUES(bp);
4071                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4072                 netdev_set_tc_queue(dev, cos, count, offset);
4073                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4074                    "mapping tc %d to offset %d count %d\n",
4075                    cos, offset, count);
4076         }
4077
4078         return 0;
4079 }
4080
4081 /* called with rtnl_lock */
4082 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4083 {
4084         struct sockaddr *addr = p;
4085         struct bnx2x *bp = netdev_priv(dev);
4086         int rc = 0;
4087
4088         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4089                 BNX2X_ERR("Requested MAC address is not valid\n");
4090                 return -EINVAL;
4091         }
4092
4093         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4094             !is_zero_ether_addr(addr->sa_data)) {
4095                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4096                 return -EINVAL;
4097         }
4098
4099         if (netif_running(dev))  {
4100                 rc = bnx2x_set_eth_mac(bp, false);
4101                 if (rc)
4102                         return rc;
4103         }
4104
4105         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4106
4107         if (netif_running(dev))
4108                 rc = bnx2x_set_eth_mac(bp, true);
4109
4110         return rc;
4111 }
4112
4113 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4114 {
4115         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4116         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4117         u8 cos;
4118
4119         /* Common */
4120
4121         if (IS_FCOE_IDX(fp_index)) {
4122                 memset(sb, 0, sizeof(union host_hc_status_block));
4123                 fp->status_blk_mapping = 0;
4124         } else {
4125                 /* status blocks */
4126                 if (!CHIP_IS_E1x(bp))
4127                         BNX2X_PCI_FREE(sb->e2_sb,
4128                                        bnx2x_fp(bp, fp_index,
4129                                                 status_blk_mapping),
4130                                        sizeof(struct host_hc_status_block_e2));
4131                 else
4132                         BNX2X_PCI_FREE(sb->e1x_sb,
4133                                        bnx2x_fp(bp, fp_index,
4134                                                 status_blk_mapping),
4135                                        sizeof(struct host_hc_status_block_e1x));
4136         }
4137
4138         /* Rx */
4139         if (!skip_rx_queue(bp, fp_index)) {
4140                 bnx2x_free_rx_bds(fp);
4141
4142                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4143                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4144                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4145                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4146                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4147
4148                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4149                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4150                                sizeof(struct eth_fast_path_rx_cqe) *
4151                                NUM_RCQ_BD);
4152
4153                 /* SGE ring */
4154                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4155                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4156                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4157                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4158         }
4159
4160         /* Tx */
4161         if (!skip_tx_queue(bp, fp_index)) {
4162                 /* fastpath tx rings: tx_buf tx_desc */
4163                 for_each_cos_in_tx_queue(fp, cos) {
4164                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4165
4166                         DP(NETIF_MSG_IFDOWN,
4167                            "freeing tx memory of fp %d cos %d cid %d\n",
4168                            fp_index, cos, txdata->cid);
4169
4170                         BNX2X_FREE(txdata->tx_buf_ring);
4171                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4172                                 txdata->tx_desc_mapping,
4173                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4174                 }
4175         }
4176         /* end of fastpath */
4177 }
4178
4179 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4180 {
4181         int i;
4182         for_each_cnic_queue(bp, i)
4183                 bnx2x_free_fp_mem_at(bp, i);
4184 }
4185
4186 void bnx2x_free_fp_mem(struct bnx2x *bp)
4187 {
4188         int i;
4189         for_each_eth_queue(bp, i)
4190                 bnx2x_free_fp_mem_at(bp, i);
4191 }
4192
4193 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4194 {
4195         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4196         if (!CHIP_IS_E1x(bp)) {
4197                 bnx2x_fp(bp, index, sb_index_values) =
4198                         (__le16 *)status_blk.e2_sb->sb.index_values;
4199                 bnx2x_fp(bp, index, sb_running_index) =
4200                         (__le16 *)status_blk.e2_sb->sb.running_index;
4201         } else {
4202                 bnx2x_fp(bp, index, sb_index_values) =
4203                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4204                 bnx2x_fp(bp, index, sb_running_index) =
4205                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4206         }
4207 }
4208
4209 /* Returns the number of actually allocated BDs */
4210 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4211                               int rx_ring_size)
4212 {
4213         struct bnx2x *bp = fp->bp;
4214         u16 ring_prod, cqe_ring_prod;
4215         int i, failure_cnt = 0;
4216
4217         fp->rx_comp_cons = 0;
4218         cqe_ring_prod = ring_prod = 0;
4219
4220         /* This routine is called only during fo init so
4221          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4222          */
4223         for (i = 0; i < rx_ring_size; i++) {
4224                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4225                         failure_cnt++;
4226                         continue;
4227                 }
4228                 ring_prod = NEXT_RX_IDX(ring_prod);
4229                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4230                 WARN_ON(ring_prod <= (i - failure_cnt));
4231         }
4232
4233         if (failure_cnt)
4234                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4235                           i - failure_cnt, fp->index);
4236
4237         fp->rx_bd_prod = ring_prod;
4238         /* Limit the CQE producer by the CQE ring size */
4239         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4240                                cqe_ring_prod);
4241         fp->rx_pkt = fp->rx_calls = 0;
4242
4243         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4244
4245         return i - failure_cnt;
4246 }
4247
4248 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4249 {
4250         int i;
4251
4252         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4253                 struct eth_rx_cqe_next_page *nextpg;
4254
4255                 nextpg = (struct eth_rx_cqe_next_page *)
4256                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4257                 nextpg->addr_hi =
4258                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4259                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4260                 nextpg->addr_lo =
4261                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4262                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4263         }
4264 }
4265
4266 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4267 {
4268         union host_hc_status_block *sb;
4269         struct bnx2x_fastpath *fp = &bp->fp[index];
4270         int ring_size = 0;
4271         u8 cos;
4272         int rx_ring_size = 0;
4273
4274         if (!bp->rx_ring_size &&
4275             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4276                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4277                 bp->rx_ring_size = rx_ring_size;
4278         } else if (!bp->rx_ring_size) {
4279                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4280
4281                 if (CHIP_IS_E3(bp)) {
4282                         u32 cfg = SHMEM_RD(bp,
4283                                            dev_info.port_hw_config[BP_PORT(bp)].
4284                                            default_cfg);
4285
4286                         /* Decrease ring size for 1G functions */
4287                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4288                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4289                                 rx_ring_size /= 10;
4290                 }
4291
4292                 /* allocate at least number of buffers required by FW */
4293                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4294                                      MIN_RX_SIZE_TPA, rx_ring_size);
4295
4296                 bp->rx_ring_size = rx_ring_size;
4297         } else /* if rx_ring_size specified - use it */
4298                 rx_ring_size = bp->rx_ring_size;
4299
4300         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4301
4302         /* Common */
4303         sb = &bnx2x_fp(bp, index, status_blk);
4304
4305         if (!IS_FCOE_IDX(index)) {
4306                 /* status blocks */
4307                 if (!CHIP_IS_E1x(bp))
4308                         BNX2X_PCI_ALLOC(sb->e2_sb,
4309                                 &bnx2x_fp(bp, index, status_blk_mapping),
4310                                 sizeof(struct host_hc_status_block_e2));
4311                 else
4312                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4313                                 &bnx2x_fp(bp, index, status_blk_mapping),
4314                             sizeof(struct host_hc_status_block_e1x));
4315         }
4316
4317         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4318          * set shortcuts for it.
4319          */
4320         if (!IS_FCOE_IDX(index))
4321                 set_sb_shortcuts(bp, index);
4322
4323         /* Tx */
4324         if (!skip_tx_queue(bp, index)) {
4325                 /* fastpath tx rings: tx_buf tx_desc */
4326                 for_each_cos_in_tx_queue(fp, cos) {
4327                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4328
4329                         DP(NETIF_MSG_IFUP,
4330                            "allocating tx memory of fp %d cos %d\n",
4331                            index, cos);
4332
4333                         BNX2X_ALLOC(txdata->tx_buf_ring,
4334                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4335                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4336                                 &txdata->tx_desc_mapping,
4337                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4338                 }
4339         }
4340
4341         /* Rx */
4342         if (!skip_rx_queue(bp, index)) {
4343                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4344                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4345                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4346                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4347                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4348                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4349
4350                 /* Seed all CQEs by 1s */
4351                 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4352                                  &bnx2x_fp(bp, index, rx_comp_mapping),
4353                                  sizeof(struct eth_fast_path_rx_cqe) *
4354                                  NUM_RCQ_BD);
4355
4356                 /* SGE ring */
4357                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4358                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4359                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4360                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4361                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4362                 /* RX BD ring */
4363                 bnx2x_set_next_page_rx_bd(fp);
4364
4365                 /* CQ ring */
4366                 bnx2x_set_next_page_rx_cq(fp);
4367
4368                 /* BDs */
4369                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4370                 if (ring_size < rx_ring_size)
4371                         goto alloc_mem_err;
4372         }
4373
4374         return 0;
4375
4376 /* handles low memory cases */
4377 alloc_mem_err:
4378         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4379                                                 index, ring_size);
4380         /* FW will drop all packets if queue is not big enough,
4381          * In these cases we disable the queue
4382          * Min size is different for OOO, TPA and non-TPA queues
4383          */
4384         if (ring_size < (fp->disable_tpa ?
4385                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4386                         /* release memory allocated for this queue */
4387                         bnx2x_free_fp_mem_at(bp, index);
4388                         return -ENOMEM;
4389         }
4390         return 0;
4391 }
4392
4393 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4394 {
4395         if (!NO_FCOE(bp))
4396                 /* FCoE */
4397                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4398                         /* we will fail load process instead of mark
4399                          * NO_FCOE_FLAG
4400                          */
4401                         return -ENOMEM;
4402
4403         return 0;
4404 }
4405
4406 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4407 {
4408         int i;
4409
4410         /* 1. Allocate FP for leading - fatal if error
4411          * 2. Allocate RSS - fix number of queues if error
4412          */
4413
4414         /* leading */
4415         if (bnx2x_alloc_fp_mem_at(bp, 0))
4416                 return -ENOMEM;
4417
4418         /* RSS */
4419         for_each_nondefault_eth_queue(bp, i)
4420                 if (bnx2x_alloc_fp_mem_at(bp, i))
4421                         break;
4422
4423         /* handle memory failures */
4424         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4425                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4426
4427                 WARN_ON(delta < 0);
4428                 bnx2x_shrink_eth_fp(bp, delta);
4429                 if (CNIC_SUPPORT(bp))
4430                         /* move non eth FPs next to last eth FP
4431                          * must be done in that order
4432                          * FCOE_IDX < FWD_IDX < OOO_IDX
4433                          */
4434
4435                         /* move FCoE fp even NO_FCOE_FLAG is on */
4436                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4437                 bp->num_ethernet_queues -= delta;
4438                 bp->num_queues = bp->num_ethernet_queues +
4439                                  bp->num_cnic_queues;
4440                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4441                           bp->num_queues + delta, bp->num_queues);
4442         }
4443
4444         return 0;
4445 }
4446
4447 void bnx2x_free_mem_bp(struct bnx2x *bp)
4448 {
4449         int i;
4450
4451         for (i = 0; i < bp->fp_array_size; i++)
4452                 kfree(bp->fp[i].tpa_info);
4453         kfree(bp->fp);
4454         kfree(bp->sp_objs);
4455         kfree(bp->fp_stats);
4456         kfree(bp->bnx2x_txq);
4457         kfree(bp->msix_table);
4458         kfree(bp->ilt);
4459 }
4460
4461 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4462 {
4463         struct bnx2x_fastpath *fp;
4464         struct msix_entry *tbl;
4465         struct bnx2x_ilt *ilt;
4466         int msix_table_size = 0;
4467         int fp_array_size, txq_array_size;
4468         int i;
4469
4470         /*
4471          * The biggest MSI-X table we might need is as a maximum number of fast
4472          * path IGU SBs plus default SB (for PF only).
4473          */
4474         msix_table_size = bp->igu_sb_cnt;
4475         if (IS_PF(bp))
4476                 msix_table_size++;
4477         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4478
4479         /* fp array: RSS plus CNIC related L2 queues */
4480         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4481         bp->fp_array_size = fp_array_size;
4482         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4483
4484         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4485         if (!fp)
4486                 goto alloc_err;
4487         for (i = 0; i < bp->fp_array_size; i++) {
4488                 fp[i].tpa_info =
4489                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4490                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4491                 if (!(fp[i].tpa_info))
4492                         goto alloc_err;
4493         }
4494
4495         bp->fp = fp;
4496
4497         /* allocate sp objs */
4498         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4499                               GFP_KERNEL);
4500         if (!bp->sp_objs)
4501                 goto alloc_err;
4502
4503         /* allocate fp_stats */
4504         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4505                                GFP_KERNEL);
4506         if (!bp->fp_stats)
4507                 goto alloc_err;
4508
4509         /* Allocate memory for the transmission queues array */
4510         txq_array_size =
4511                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4512         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4513
4514         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4515                                 GFP_KERNEL);
4516         if (!bp->bnx2x_txq)
4517                 goto alloc_err;
4518
4519         /* msix table */
4520         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4521         if (!tbl)
4522                 goto alloc_err;
4523         bp->msix_table = tbl;
4524
4525         /* ilt */
4526         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4527         if (!ilt)
4528                 goto alloc_err;
4529         bp->ilt = ilt;
4530
4531         return 0;
4532 alloc_err:
4533         bnx2x_free_mem_bp(bp);
4534         return -ENOMEM;
4535 }
4536
4537 int bnx2x_reload_if_running(struct net_device *dev)
4538 {
4539         struct bnx2x *bp = netdev_priv(dev);
4540
4541         if (unlikely(!netif_running(dev)))
4542                 return 0;
4543
4544         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4545         return bnx2x_nic_load(bp, LOAD_NORMAL);
4546 }
4547
4548 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4549 {
4550         u32 sel_phy_idx = 0;
4551         if (bp->link_params.num_phys <= 1)
4552                 return INT_PHY;
4553
4554         if (bp->link_vars.link_up) {
4555                 sel_phy_idx = EXT_PHY1;
4556                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4557                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4558                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4559                         sel_phy_idx = EXT_PHY2;
4560         } else {
4561
4562                 switch (bnx2x_phy_selection(&bp->link_params)) {
4563                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4564                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4565                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4566                        sel_phy_idx = EXT_PHY1;
4567                        break;
4568                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4569                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4570                        sel_phy_idx = EXT_PHY2;
4571                        break;
4572                 }
4573         }
4574
4575         return sel_phy_idx;
4576 }
4577 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4578 {
4579         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4580         /*
4581          * The selected activated PHY is always after swapping (in case PHY
4582          * swapping is enabled). So when swapping is enabled, we need to reverse
4583          * the configuration
4584          */
4585
4586         if (bp->link_params.multi_phy_config &
4587             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4588                 if (sel_phy_idx == EXT_PHY1)
4589                         sel_phy_idx = EXT_PHY2;
4590                 else if (sel_phy_idx == EXT_PHY2)
4591                         sel_phy_idx = EXT_PHY1;
4592         }
4593         return LINK_CONFIG_IDX(sel_phy_idx);
4594 }
4595
4596 #ifdef NETDEV_FCOE_WWNN
4597 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4598 {
4599         struct bnx2x *bp = netdev_priv(dev);
4600         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4601
4602         switch (type) {
4603         case NETDEV_FCOE_WWNN:
4604                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4605                                 cp->fcoe_wwn_node_name_lo);
4606                 break;
4607         case NETDEV_FCOE_WWPN:
4608                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4609                                 cp->fcoe_wwn_port_name_lo);
4610                 break;
4611         default:
4612                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4613                 return -EINVAL;
4614         }
4615
4616         return 0;
4617 }
4618 #endif
4619
4620 /* called with rtnl_lock */
4621 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4622 {
4623         struct bnx2x *bp = netdev_priv(dev);
4624
4625         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4626                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4627                 return -EAGAIN;
4628         }
4629
4630         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4631             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4632                 BNX2X_ERR("Can't support requested MTU size\n");
4633                 return -EINVAL;
4634         }
4635
4636         /* This does not race with packet allocation
4637          * because the actual alloc size is
4638          * only updated as part of load
4639          */
4640         dev->mtu = new_mtu;
4641
4642         return bnx2x_reload_if_running(dev);
4643 }
4644
4645 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4646                                      netdev_features_t features)
4647 {
4648         struct bnx2x *bp = netdev_priv(dev);
4649
4650         /* TPA requires Rx CSUM offloading */
4651         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4652                 features &= ~NETIF_F_LRO;
4653                 features &= ~NETIF_F_GRO;
4654         }
4655
4656         return features;
4657 }
4658
4659 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4660 {
4661         struct bnx2x *bp = netdev_priv(dev);
4662         u32 flags = bp->flags;
4663         u32 changes;
4664         bool bnx2x_reload = false;
4665
4666         if (features & NETIF_F_LRO)
4667                 flags |= TPA_ENABLE_FLAG;
4668         else
4669                 flags &= ~TPA_ENABLE_FLAG;
4670
4671         if (features & NETIF_F_GRO)
4672                 flags |= GRO_ENABLE_FLAG;
4673         else
4674                 flags &= ~GRO_ENABLE_FLAG;
4675
4676         if (features & NETIF_F_LOOPBACK) {
4677                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4678                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4679                         bnx2x_reload = true;
4680                 }
4681         } else {
4682                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4683                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4684                         bnx2x_reload = true;
4685                 }
4686         }
4687
4688         changes = flags ^ bp->flags;
4689
4690         /* if GRO is changed while LRO is enabled, don't force a reload */
4691         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4692                 changes &= ~GRO_ENABLE_FLAG;
4693
4694         if (changes)
4695                 bnx2x_reload = true;
4696
4697         bp->flags = flags;
4698
4699         if (bnx2x_reload) {
4700                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4701                         return bnx2x_reload_if_running(dev);
4702                 /* else: bnx2x_nic_load() will be called at end of recovery */
4703         }
4704
4705         return 0;
4706 }
4707
4708 void bnx2x_tx_timeout(struct net_device *dev)
4709 {
4710         struct bnx2x *bp = netdev_priv(dev);
4711
4712 #ifdef BNX2X_STOP_ON_ERROR
4713         if (!bp->panic)
4714                 bnx2x_panic();
4715 #endif
4716
4717         smp_mb__before_clear_bit();
4718         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4719         smp_mb__after_clear_bit();
4720
4721         /* This allows the netif to be shutdown gracefully before resetting */
4722         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4723 }
4724
4725 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4726 {
4727         struct net_device *dev = pci_get_drvdata(pdev);
4728         struct bnx2x *bp;
4729
4730         if (!dev) {
4731                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4732                 return -ENODEV;
4733         }
4734         bp = netdev_priv(dev);
4735
4736         rtnl_lock();
4737
4738         pci_save_state(pdev);
4739
4740         if (!netif_running(dev)) {
4741                 rtnl_unlock();
4742                 return 0;
4743         }
4744
4745         netif_device_detach(dev);
4746
4747         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4748
4749         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4750
4751         rtnl_unlock();
4752
4753         return 0;
4754 }
4755
4756 int bnx2x_resume(struct pci_dev *pdev)
4757 {
4758         struct net_device *dev = pci_get_drvdata(pdev);
4759         struct bnx2x *bp;
4760         int rc;
4761
4762         if (!dev) {
4763                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4764                 return -ENODEV;
4765         }
4766         bp = netdev_priv(dev);
4767
4768         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4769                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4770                 return -EAGAIN;
4771         }
4772
4773         rtnl_lock();
4774
4775         pci_restore_state(pdev);
4776
4777         if (!netif_running(dev)) {
4778                 rtnl_unlock();
4779                 return 0;
4780         }
4781
4782         bnx2x_set_power_state(bp, PCI_D0);
4783         netif_device_attach(dev);
4784
4785         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4786
4787         rtnl_unlock();
4788
4789         return rc;
4790 }
4791
4792 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4793                               u32 cid)
4794 {
4795         if (!cxt) {
4796                 BNX2X_ERR("bad context pointer %p\n", cxt);
4797                 return;
4798         }
4799
4800         /* ustorm cxt validation */
4801         cxt->ustorm_ag_context.cdu_usage =
4802                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4803                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4804         /* xcontext validation */
4805         cxt->xstorm_ag_context.cdu_reserved =
4806                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4807                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4808 }
4809
4810 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4811                                     u8 fw_sb_id, u8 sb_index,
4812                                     u8 ticks)
4813 {
4814         u32 addr = BAR_CSTRORM_INTMEM +
4815                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4816         REG_WR8(bp, addr, ticks);
4817         DP(NETIF_MSG_IFUP,
4818            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4819            port, fw_sb_id, sb_index, ticks);
4820 }
4821
4822 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4823                                     u16 fw_sb_id, u8 sb_index,
4824                                     u8 disable)
4825 {
4826         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4827         u32 addr = BAR_CSTRORM_INTMEM +
4828                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4829         u8 flags = REG_RD8(bp, addr);
4830         /* clear and set */
4831         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4832         flags |= enable_flag;
4833         REG_WR8(bp, addr, flags);
4834         DP(NETIF_MSG_IFUP,
4835            "port %x fw_sb_id %d sb_index %d disable %d\n",
4836            port, fw_sb_id, sb_index, disable);
4837 }
4838
4839 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4840                                     u8 sb_index, u8 disable, u16 usec)
4841 {
4842         int port = BP_PORT(bp);
4843         u8 ticks = usec / BNX2X_BTR;
4844
4845         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4846
4847         disable = disable ? 1 : (usec ? 0 : 1);
4848         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4849 }