drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32 /**
  33  * bnx2x_move_fp - move content of the fastpath structure.
  34  *
  35  * @bp:         driver handle
  36  * @from:       source FP index
  37  * @to:         destination FP index
  38  *
  39  * Makes sure the contents of the bp->fp[to].napi is kept
  40  * intact. This is done by first copying the napi struct from
  41  * the target to the source, and then mem copying the entire
  42  * source onto the target. Update txdata pointers and related
  43  * content.
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  50         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  51         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  52         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  53         int old_max_eth_txqs, new_max_eth_txqs;
  54         int old_txdata_index = 0, new_txdata_index = 0;
  55
  56         /* Copy the NAPI object as it has been already initialized */
  57         from_fp->napi = to_fp->napi;
  58
  59         /* Move bnx2x_fastpath contents */
  60         memcpy(to_fp, from_fp, sizeof(*to_fp));
  61         to_fp->index = to;
  62
  63         /* move sp_objs contents as well, as their indices match fp ones */
  64         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  65
  66         /* move fp_stats contents as well, as their indices match fp ones */
  67         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  68
  69         /* Update txdata pointers in fp and move txdata content accordingly:
  70          * Each fp consumes 'max_cos' txdata structures, so the index should be
  71          * decremented by max_cos x delta.
  72          */
  73
  74         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  75         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  76                                 (bp)->max_cos;
  77         if (from == FCOE_IDX(bp)) {
  78                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  79                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80         }
  81
  82         memcpy(&bp->bnx2x_txq[new_txdata_index],
  83                &bp->bnx2x_txq[old_txdata_index],
  84                sizeof(struct bnx2x_fp_txdata));
  85         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  86 }
  87
  88 /**
  89  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  90  *
  91  * @bp:        driver handle
  92  * @buf:       character buffer to fill with the fw name
  93  * @buf_len:   length of the above buffer
  94  *
  95  */
  96 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
  97 {
  98         if (IS_PF(bp)) {
  99                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 100
 101                 phy_fw_ver[0] = '\0';
 102                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 103                                              phy_fw_ver, PHY_FW_VER_LEN);
 104                 strlcpy(buf, bp->fw_ver, buf_len);
 105                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 106                          "bc %d.%d.%d%s%s",
 107                          (bp->common.bc_ver & 0xff0000) >> 16,
 108                          (bp->common.bc_ver & 0xff00) >> 8,
 109                          (bp->common.bc_ver & 0xff),
 110                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 111         } else {
 112                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 113         }
 114 }
 115
 116 /**
 117  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 118  *
 119  * @bp: driver handle
 120  * @delta:      number of eth queues which were not allocated
 121  */
 122 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 123 {
 124         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 125
 126         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 127          * backward along the array could cause memory to be overriden
 128          */
 129         for (cos = 1; cos < bp->max_cos; cos++) {
 130                 for (i = 0; i < old_eth_num - delta; i++) {
 131                         struct bnx2x_fastpath *fp = &bp->fp[i];
 132                         int new_idx = cos * (old_eth_num - delta) + i;
 133
 134                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 135                                sizeof(struct bnx2x_fp_txdata));
 136                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 137                 }
 138         }
 139 }
 140
 141 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 142
 143 /* free skb in the packet ring at pos idx
 144  * return idx of last bd freed
 145  */
 146 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 147                              u16 idx, unsigned int *pkts_compl,
 148                              unsigned int *bytes_compl)
 149 {
 150         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 151         struct eth_tx_start_bd *tx_start_bd;
 152         struct eth_tx_bd *tx_data_bd;
 153         struct sk_buff *skb = tx_buf->skb;
 154         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 155         int nbd;
 156
 157         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 158         prefetch(&skb->end);
 159
 160         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 161            txdata->txq_index, idx, tx_buf, skb);
 162
 163         /* unmap first bd */
 164         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 165         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 166                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 167
 168
 169         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 170 #ifdef BNX2X_STOP_ON_ERROR
 171         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 172                 BNX2X_ERR("BAD nbd!\n");
 173                 bnx2x_panic();
 174         }
 175 #endif
 176         new_cons = nbd + tx_buf->first_bd;
 177
 178         /* Get the next bd */
 179         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 180
 181         /* Skip a parse bd... */
 182         --nbd;
 183         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 184
 185         /* ...and the TSO split header bd since they have no mapping */
 186         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 187                 --nbd;
 188                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 189         }
 190
 191         /* now free frags */
 192         while (nbd > 0) {
 193
 194                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 195                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 196                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 197                 if (--nbd)
 198                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 199         }
 200
 201         /* release skb */
 202         WARN_ON(!skb);
 203         if (likely(skb)) {
 204                 (*pkts_compl)++;
 205                 (*bytes_compl) += skb->len;
 206         }
 207
 208         dev_kfree_skb_any(skb);
 209         tx_buf->first_bd = 0;
 210         tx_buf->skb = NULL;
 211
 212         return new_cons;
 213 }
 214
 215 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 216 {
 217         struct netdev_queue *txq;
 218         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 219         unsigned int pkts_compl = 0, bytes_compl = 0;
 220
 221 #ifdef BNX2X_STOP_ON_ERROR
 222         if (unlikely(bp->panic))
 223                 return -1;
 224 #endif
 225
 226         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 227         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 228         sw_cons = txdata->tx_pkt_cons;
 229
 230         while (sw_cons != hw_cons) {
 231                 u16 pkt_cons;
 232
 233                 pkt_cons = TX_BD(sw_cons);
 234
 235                 DP(NETIF_MSG_TX_DONE,
 236                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 237                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 238
 239                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 240                                             &pkts_compl, &bytes_compl);
 241
 242                 sw_cons++;
 243         }
 244
 245         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 246
 247         txdata->tx_pkt_cons = sw_cons;
 248         txdata->tx_bd_cons = bd_cons;
 249
 250         /* Need to make the tx_bd_cons update visible to start_xmit()
 251          * before checking for netif_tx_queue_stopped().  Without the
 252          * memory barrier, there is a small possibility that
 253          * start_xmit() will miss it and cause the queue to be stopped
 254          * forever.
 255          * On the other hand we need an rmb() here to ensure the proper
 256          * ordering of bit testing in the following
 257          * netif_tx_queue_stopped(txq) call.
 258          */
 259         smp_mb();
 260
 261         if (unlikely(netif_tx_queue_stopped(txq))) {
 262                 /* Taking tx_lock() is needed to prevent reenabling the queue
 263                  * while it's empty. This could have happen if rx_action() gets
 264                  * suspended in bnx2x_tx_int() after the condition before
 265                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 266                  *
 267                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 268                  * sends some packets consuming the whole queue again->
 269                  * stops the queue
 270                  */
 271
 272                 __netif_tx_lock(txq, smp_processor_id());
 273
 274                 if ((netif_tx_queue_stopped(txq)) &&
 275                     (bp->state == BNX2X_STATE_OPEN) &&
 276                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 277                         netif_tx_wake_queue(txq);
 278
 279                 __netif_tx_unlock(txq);
 280         }
 281         return 0;
 282 }
 283
 284 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 285                                              u16 idx)
 286 {
 287         u16 last_max = fp->last_max_sge;
 288
 289         if (SUB_S16(idx, last_max) > 0)
 290                 fp->last_max_sge = idx;
 291 }
 292
 293 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 294                                          u16 sge_len,
 295                                          struct eth_end_agg_rx_cqe *cqe)
 296 {
 297         struct bnx2x *bp = fp->bp;
 298         u16 last_max, last_elem, first_elem;
 299         u16 delta = 0;
 300         u16 i;
 301
 302         if (!sge_len)
 303                 return;
 304
 305         /* First mark all used pages */
 306         for (i = 0; i < sge_len; i++)
 307                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 308                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 309
 310         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 311            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 312
 313         /* Here we assume that the last SGE index is the biggest */
 314         prefetch((void *)(fp->sge_mask));
 315         bnx2x_update_last_max_sge(fp,
 316                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 317
 318         last_max = RX_SGE(fp->last_max_sge);
 319         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 320         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 321
 322         /* If ring is not full */
 323         if (last_elem + 1 != first_elem)
 324                 last_elem++;
 325
 326         /* Now update the prod */
 327         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 328                 if (likely(fp->sge_mask[i]))
 329                         break;
 330
 331                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 332                 delta += BIT_VEC64_ELEM_SZ;
 333         }
 334
 335         if (delta > 0) {
 336                 fp->rx_sge_prod += delta;
 337                 /* clear page-end entries */
 338                 bnx2x_clear_sge_mask_next_elems(fp);
 339         }
 340
 341         DP(NETIF_MSG_RX_STATUS,
 342            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 343            fp->last_max_sge, fp->rx_sge_prod);
 344 }
 345
 346 /* Get Toeplitz hash value in the skb using the value from the
 347  * CQE (calculated by HW).
 348  */
 349 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 350                             const struct eth_fast_path_rx_cqe *cqe,
 351                             bool *l4_rxhash)
 352 {
 353         /* Get Toeplitz hash from CQE */
 354         if ((bp->dev->features & NETIF_F_RXHASH) &&
 355             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 356                 enum eth_rss_hash_type htype;
 357
 358                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 359                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 360                              (htype == TCP_IPV6_HASH_TYPE);
 361                 return le32_to_cpu(cqe->rss_hash_result);
 362         }
 363         *l4_rxhash = false;
 364         return 0;
 365 }
 366
 367 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 368                             u16 cons, u16 prod,
 369                             struct eth_fast_path_rx_cqe *cqe)
 370 {
 371         struct bnx2x *bp = fp->bp;
 372         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 373         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 374         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 375         dma_addr_t mapping;
 376         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 377         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 378
 379         /* print error if current state != stop */
 380         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 381                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 382
 383         /* Try to map an empty data buffer from the aggregation info  */
 384         mapping = dma_map_single(&bp->pdev->dev,
 385                                  first_buf->data + NET_SKB_PAD,
 386                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 387         /*
 388          *  ...if it fails - move the skb from the consumer to the producer
 389          *  and set the current aggregation state as ERROR to drop it
 390          *  when TPA_STOP arrives.
 391          */
 392
 393         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 394                 /* Move the BD from the consumer to the producer */
 395                 bnx2x_reuse_rx_data(fp, cons, prod);
 396                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 397                 return;
 398         }
 399
 400         /* move empty data from pool to prod */
 401         prod_rx_buf->data = first_buf->data;
 402         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 403         /* point prod_bd to new data */
 404         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 405         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 406
 407         /* move partial skb from cons to pool (don't unmap yet) */
 408         *first_buf = *cons_rx_buf;
 409
 410         /* mark bin state as START */
 411         tpa_info->parsing_flags =
 412                 le16_to_cpu(cqe->pars_flags.flags);
 413         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 414         tpa_info->tpa_state = BNX2X_TPA_START;
 415         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 416         tpa_info->placement_offset = cqe->placement_offset;
 417         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 418         if (fp->mode == TPA_MODE_GRO) {
 419                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 420                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 421                 tpa_info->gro_size = gro_size;
 422         }
 423
 424 #ifdef BNX2X_STOP_ON_ERROR
 425         fp->tpa_queue_used |= (1 << queue);
 426 #ifdef _ASM_GENERIC_INT_L64_H
 427         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 428 #else
 429         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 430 #endif
 431            fp->tpa_queue_used);
 432 #endif
 433 }
 434
 435 /* Timestamp option length allowed for TPA aggregation:
 436  *
 437  *              nop nop kind length echo val
 438  */
 439 #define TPA_TSTAMP_OPT_LEN      12
 440 /**
 441  * bnx2x_set_gro_params - compute GRO values
 442  *
 443  * @skb:                packet skb
 444  * @parsing_flags:      parsing flags from the START CQE
 445  * @len_on_bd:          total length of the first packet for the
 446  *                      aggregation.
 447  * @pkt_len:            length of all segments
 448  *
 449  * Approximate value of the MSS for this aggregation calculated using
 450  * the first packet of it.
 451  * Compute number of aggregated segments, and gso_type.
 452  */
 453 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 454                                  u16 len_on_bd, unsigned int pkt_len,
 455                                  u16 num_of_coalesced_segs)
 456 {
 457         /* TPA aggregation won't have either IP options or TCP options
 458          * other than timestamp or IPv6 extension headers.
 459          */
 460         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 461
 462         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 463             PRS_FLAG_OVERETH_IPV6) {
 464                 hdrs_len += sizeof(struct ipv6hdr);
 465                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 466         } else {
 467                 hdrs_len += sizeof(struct iphdr);
 468                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 469         }
 470
 471         /* Check if there was a TCP timestamp, if there is it's will
 472          * always be 12 bytes length: nop nop kind length echo val.
 473          *
 474          * Otherwise FW would close the aggregation.
 475          */
 476         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 477                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 478
 479         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 480
 481         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 482          * to skb_shinfo(skb)->gso_segs
 483          */
 484         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 485 }
 486
 487 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 488                               struct bnx2x_fastpath *fp, u16 index)
 489 {
 490         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 491         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 492         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 493         dma_addr_t mapping;
 494
 495         if (unlikely(page == NULL)) {
 496                 BNX2X_ERR("Can't alloc sge\n");
 497                 return -ENOMEM;
 498         }
 499
 500         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 501                                SGE_PAGES, DMA_FROM_DEVICE);
 502         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 503                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 504                 BNX2X_ERR("Can't map sge\n");
 505                 return -ENOMEM;
 506         }
 507
 508         sw_buf->page = page;
 509         dma_unmap_addr_set(sw_buf, mapping, mapping);
 510
 511         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 512         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 513
 514         return 0;
 515 }
 516
 517 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 518                                struct bnx2x_agg_info *tpa_info,
 519                                u16 pages,
 520                                struct sk_buff *skb,
 521                                struct eth_end_agg_rx_cqe *cqe,
 522                                u16 cqe_idx)
 523 {
 524         struct sw_rx_page *rx_pg, old_rx_pg;
 525         u32 i, frag_len, frag_size;
 526         int err, j, frag_id = 0;
 527         u16 len_on_bd = tpa_info->len_on_bd;
 528         u16 full_page = 0, gro_size = 0;
 529
 530         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 531
 532         if (fp->mode == TPA_MODE_GRO) {
 533                 gro_size = tpa_info->gro_size;
 534                 full_page = tpa_info->full_page;
 535         }
 536
 537         /* This is needed in order to enable forwarding support */
 538         if (frag_size)
 539                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 540                                      le16_to_cpu(cqe->pkt_len),
 541                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 542
 543 #ifdef BNX2X_STOP_ON_ERROR
 544         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 545                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 546                           pages, cqe_idx);
 547                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 548                 bnx2x_panic();
 549                 return -EINVAL;
 550         }
 551 #endif
 552
 553         /* Run through the SGL and compose the fragmented skb */
 554         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 555                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 556
 557                 /* FW gives the indices of the SGE as if the ring is an array
 558                    (meaning that "next" element will consume 2 indices) */
 559                 if (fp->mode == TPA_MODE_GRO)
 560                         frag_len = min_t(u32, frag_size, (u32)full_page);
 561                 else /* LRO */
 562                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 563
 564                 rx_pg = &fp->rx_page_ring[sge_idx];
 565                 old_rx_pg = *rx_pg;
 566
 567                 /* If we fail to allocate a substitute page, we simply stop
 568                    where we are and drop the whole packet */
 569                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 570                 if (unlikely(err)) {
 571                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 572                         return err;
 573                 }
 574
 575                 /* Unmap the page as we r going to pass it to the stack */
 576                 dma_unmap_page(&bp->pdev->dev,
 577                                dma_unmap_addr(&old_rx_pg, mapping),
 578                                SGE_PAGES, DMA_FROM_DEVICE);
 579                 /* Add one frag and update the appropriate fields in the skb */
 580                 if (fp->mode == TPA_MODE_LRO)
 581                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 582                 else { /* GRO */
 583                         int rem;
 584                         int offset = 0;
 585                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 586                                 int len = rem > gro_size ? gro_size : rem;
 587                                 skb_fill_page_desc(skb, frag_id++,
 588                                                    old_rx_pg.page, offset, len);
 589                                 if (offset)
 590                                         get_page(old_rx_pg.page);
 591                                 offset += len;
 592                         }
 593                 }
 594
 595                 skb->data_len += frag_len;
 596                 skb->truesize += SGE_PAGES;
 597                 skb->len += frag_len;
 598
 599                 frag_size -= frag_len;
 600         }
 601
 602         return 0;
 603 }
 604
 605 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 606 {
 607         if (fp->rx_frag_size)
 608                 put_page(virt_to_head_page(data));
 609         else
 610                 kfree(data);
 611 }
 612
 613 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 614 {
 615         if (fp->rx_frag_size)
 616                 return netdev_alloc_frag(fp->rx_frag_size);
 617
 618         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 619 }
 620
 621 #ifdef CONFIG_INET
 622 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 623 {
 624         const struct iphdr *iph = ip_hdr(skb);
 625         struct tcphdr *th;
 626
 627         skb_set_transport_header(skb, sizeof(struct iphdr));
 628         th = tcp_hdr(skb);
 629
 630         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 631                                   iph->saddr, iph->daddr, 0);
 632 }
 633
 634 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 635 {
 636         struct ipv6hdr *iph = ipv6_hdr(skb);
 637         struct tcphdr *th;
 638
 639         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 640         th = tcp_hdr(skb);
 641
 642         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 643                                   &iph->saddr, &iph->daddr, 0);
 644 }
 645 #endif
 646
 647 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 648                                struct sk_buff *skb)
 649 {
 650 #ifdef CONFIG_INET
 651         if (skb_shinfo(skb)->gso_size) {
 652                 skb_set_network_header(skb, 0);
 653                 switch (be16_to_cpu(skb->protocol)) {
 654                 case ETH_P_IP:
 655                         bnx2x_gro_ip_csum(bp, skb);
 656                         break;
 657                 case ETH_P_IPV6:
 658                         bnx2x_gro_ipv6_csum(bp, skb);
 659                         break;
 660                 default:
 661                         BNX2X_ERR("FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 662                                   be16_to_cpu(skb->protocol));
 663                 }
 664                 tcp_gro_complete(skb);
 665         }
 666 #endif
 667         napi_gro_receive(&fp->napi, skb);
 668 }
 669
 670 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 671                            struct bnx2x_agg_info *tpa_info,
 672                            u16 pages,
 673                            struct eth_end_agg_rx_cqe *cqe,
 674                            u16 cqe_idx)
 675 {
 676         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 677         u8 pad = tpa_info->placement_offset;
 678         u16 len = tpa_info->len_on_bd;
 679         struct sk_buff *skb = NULL;
 680         u8 *new_data, *data = rx_buf->data;
 681         u8 old_tpa_state = tpa_info->tpa_state;
 682
 683         tpa_info->tpa_state = BNX2X_TPA_STOP;
 684
 685         /* If we there was an error during the handling of the TPA_START -
 686          * drop this aggregation.
 687          */
 688         if (old_tpa_state == BNX2X_TPA_ERROR)
 689                 goto drop;
 690
 691         /* Try to allocate the new data */
 692         new_data = bnx2x_frag_alloc(fp);
 693         /* Unmap skb in the pool anyway, as we are going to change
 694            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 695            fails. */
 696         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 697                          fp->rx_buf_size, DMA_FROM_DEVICE);
 698         if (likely(new_data))
 699                 skb = build_skb(data, fp->rx_frag_size);
 700
 701         if (likely(skb)) {
 702 #ifdef BNX2X_STOP_ON_ERROR
 703                 if (pad + len > fp->rx_buf_size) {
 704                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 705                                   pad, len, fp->rx_buf_size);
 706                         bnx2x_panic();
 707                         return;
 708                 }
 709 #endif
 710
 711                 skb_reserve(skb, pad + NET_SKB_PAD);
 712                 skb_put(skb, len);
 713                 skb->rxhash = tpa_info->rxhash;
 714                 skb->l4_rxhash = tpa_info->l4_rxhash;
 715
 716                 skb->protocol = eth_type_trans(skb, bp->dev);
 717                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 718
 719                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 720                                          skb, cqe, cqe_idx)) {
 721                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 722                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 723                         bnx2x_gro_receive(bp, fp, skb);
 724                 } else {
 725                         DP(NETIF_MSG_RX_STATUS,
 726                            "Failed to allocate new pages - dropping packet!\n");
 727                         dev_kfree_skb_any(skb);
 728                 }
 729
 730
 731                 /* put new data in bin */
 732                 rx_buf->data = new_data;
 733
 734                 return;
 735         }
 736         bnx2x_frag_free(fp, new_data);
 737 drop:
 738         /* drop the packet and keep the buffer in the bin */
 739         DP(NETIF_MSG_RX_STATUS,
 740            "Failed to allocate or map a new skb - dropping packet!\n");
 741         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 742 }
 743
 744 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 745                                struct bnx2x_fastpath *fp, u16 index)
 746 {
 747         u8 *data;
 748         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 749         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 750         dma_addr_t mapping;
 751
 752         data = bnx2x_frag_alloc(fp);
 753         if (unlikely(data == NULL))
 754                 return -ENOMEM;
 755
 756         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 757                                  fp->rx_buf_size,
 758                                  DMA_FROM_DEVICE);
 759         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 760                 bnx2x_frag_free(fp, data);
 761                 BNX2X_ERR("Can't map rx data\n");
 762                 return -ENOMEM;
 763         }
 764
 765         rx_buf->data = data;
 766         dma_unmap_addr_set(rx_buf, mapping, mapping);
 767
 768         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 769         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 770
 771         return 0;
 772 }
 773
 774 static
 775 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 776                                  struct bnx2x_fastpath *fp,
 777                                  struct bnx2x_eth_q_stats *qstats)
 778 {
 779         /* Do nothing if no L4 csum validation was done.
 780          * We do not check whether IP csum was validated. For IPv4 we assume
 781          * that if the card got as far as validating the L4 csum, it also
 782          * validated the IP csum. IPv6 has no IP csum.
 783          */
 784         if (cqe->fast_path_cqe.status_flags &
 785             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 786                 return;
 787
 788         /* If L4 validation was done, check if an error was found. */
 789
 790         if (cqe->fast_path_cqe.type_error_flags &
 791             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 792              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 793                 qstats->hw_csum_err++;
 794         else
 795                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 796 }
 797
 798 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 799 {
 800         struct bnx2x *bp = fp->bp;
 801         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 802         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 803         int rx_pkt = 0;
 804
 805 #ifdef BNX2X_STOP_ON_ERROR
 806         if (unlikely(bp->panic))
 807                 return 0;
 808 #endif
 809
 810         /* CQ "next element" is of the size of the regular element,
 811            that's why it's ok here */
 812         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 813         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 814                 hw_comp_cons++;
 815
 816         bd_cons = fp->rx_bd_cons;
 817         bd_prod = fp->rx_bd_prod;
 818         bd_prod_fw = bd_prod;
 819         sw_comp_cons = fp->rx_comp_cons;
 820         sw_comp_prod = fp->rx_comp_prod;
 821
 822         /* Memory barrier necessary as speculative reads of the rx
 823          * buffer can be ahead of the index in the status block
 824          */
 825         rmb();
 826
 827         DP(NETIF_MSG_RX_STATUS,
 828            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 829            fp->index, hw_comp_cons, sw_comp_cons);
 830
 831         while (sw_comp_cons != hw_comp_cons) {
 832                 struct sw_rx_bd *rx_buf = NULL;
 833                 struct sk_buff *skb;
 834                 union eth_rx_cqe *cqe;
 835                 struct eth_fast_path_rx_cqe *cqe_fp;
 836                 u8 cqe_fp_flags;
 837                 enum eth_rx_cqe_type cqe_fp_type;
 838                 u16 len, pad, queue;
 839                 u8 *data;
 840                 bool l4_rxhash;
 841
 842 #ifdef BNX2X_STOP_ON_ERROR
 843                 if (unlikely(bp->panic))
 844                         return 0;
 845 #endif
 846
 847                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 848                 bd_prod = RX_BD(bd_prod);
 849                 bd_cons = RX_BD(bd_cons);
 850
 851                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 852                 cqe_fp = &cqe->fast_path_cqe;
 853                 cqe_fp_flags = cqe_fp->type_error_flags;
 854                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 855
 856                 DP(NETIF_MSG_RX_STATUS,
 857                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 858                    CQE_TYPE(cqe_fp_flags),
 859                    cqe_fp_flags, cqe_fp->status_flags,
 860                    le32_to_cpu(cqe_fp->rss_hash_result),
 861                    le16_to_cpu(cqe_fp->vlan_tag),
 862                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 863
 864                 /* is this a slowpath msg? */
 865                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 866                         bnx2x_sp_event(fp, cqe);
 867                         goto next_cqe;
 868                 }
 869
 870                 rx_buf = &fp->rx_buf_ring[bd_cons];
 871                 data = rx_buf->data;
 872
 873                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 874                         struct bnx2x_agg_info *tpa_info;
 875                         u16 frag_size, pages;
 876 #ifdef BNX2X_STOP_ON_ERROR
 877                         /* sanity check */
 878                         if (fp->disable_tpa &&
 879                             (CQE_TYPE_START(cqe_fp_type) ||
 880                              CQE_TYPE_STOP(cqe_fp_type)))
 881                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 882                                           CQE_TYPE(cqe_fp_type));
 883 #endif
 884
 885                         if (CQE_TYPE_START(cqe_fp_type)) {
 886                                 u16 queue = cqe_fp->queue_index;
 887                                 DP(NETIF_MSG_RX_STATUS,
 888                                    "calling tpa_start on queue %d\n",
 889                                    queue);
 890
 891                                 bnx2x_tpa_start(fp, queue,
 892                                                 bd_cons, bd_prod,
 893                                                 cqe_fp);
 894
 895                                 goto next_rx;
 896
 897                         }
 898                         queue = cqe->end_agg_cqe.queue_index;
 899                         tpa_info = &fp->tpa_info[queue];
 900                         DP(NETIF_MSG_RX_STATUS,
 901                            "calling tpa_stop on queue %d\n",
 902                            queue);
 903
 904                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 905                                     tpa_info->len_on_bd;
 906
 907                         if (fp->mode == TPA_MODE_GRO)
 908                                 pages = (frag_size + tpa_info->full_page - 1) /
 909                                          tpa_info->full_page;
 910                         else
 911                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 912                                         SGE_PAGE_SHIFT;
 913
 914                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 915                                        &cqe->end_agg_cqe, comp_ring_cons);
 916 #ifdef BNX2X_STOP_ON_ERROR
 917                         if (bp->panic)
 918                                 return 0;
 919 #endif
 920
 921                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 922                         goto next_cqe;
 923                 }
 924                 /* non TPA */
 925                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 926                 pad = cqe_fp->placement_offset;
 927                 dma_sync_single_for_cpu(&bp->pdev->dev,
 928                                         dma_unmap_addr(rx_buf, mapping),
 929                                         pad + RX_COPY_THRESH,
 930                                         DMA_FROM_DEVICE);
 931                 pad += NET_SKB_PAD;
 932                 prefetch(data + pad); /* speedup eth_type_trans() */
 933                 /* is this an error packet? */
 934                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 935                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 936                            "ERROR  flags %x  rx packet %u\n",
 937                            cqe_fp_flags, sw_comp_cons);
 938                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 939                         goto reuse_rx;
 940                 }
 941
 942                 /* Since we don't have a jumbo ring
 943                  * copy small packets if mtu > 1500
 944                  */
 945                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 946                     (len <= RX_COPY_THRESH)) {
 947                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 948                         if (skb == NULL) {
 949                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 950                                    "ERROR  packet dropped because of alloc failure\n");
 951                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 952                                 goto reuse_rx;
 953                         }
 954                         memcpy(skb->data, data + pad, len);
 955                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 956                 } else {
 957                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 958                                 dma_unmap_single(&bp->pdev->dev,
 959                                                  dma_unmap_addr(rx_buf, mapping),
 960                                                  fp->rx_buf_size,
 961                                                  DMA_FROM_DEVICE);
 962                                 skb = build_skb(data, fp->rx_frag_size);
 963                                 if (unlikely(!skb)) {
 964                                         bnx2x_frag_free(fp, data);
 965                                         bnx2x_fp_qstats(bp, fp)->
 966                                                         rx_skb_alloc_failed++;
 967                                         goto next_rx;
 968                                 }
 969                                 skb_reserve(skb, pad);
 970                         } else {
 971                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 972                                    "ERROR  packet dropped because of alloc failure\n");
 973                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 974 reuse_rx:
 975                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 976                                 goto next_rx;
 977                         }
 978                 }
 979
 980                 skb_put(skb, len);
 981                 skb->protocol = eth_type_trans(skb, bp->dev);
 982
 983                 /* Set Toeplitz hash for a none-LRO skb */
 984                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 985                 skb->l4_rxhash = l4_rxhash;
 986
 987                 skb_checksum_none_assert(skb);
 988
 989                 if (bp->dev->features & NETIF_F_RXCSUM)
 990                         bnx2x_csum_validate(skb, cqe, fp,
 991                                             bnx2x_fp_qstats(bp, fp));
 992
 993                 skb_record_rx_queue(skb, fp->rx_queue);
 994
 995                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 996                     PARSING_FLAGS_VLAN)
 997                         __vlan_hwaccel_put_tag(skb,
 998                                                le16_to_cpu(cqe_fp->vlan_tag));
 999                 napi_gro_receive(&fp->napi, skb);
1000
1001
1002 next_rx:
1003                 rx_buf->data = NULL;
1004
1005                 bd_cons = NEXT_RX_IDX(bd_cons);
1006                 bd_prod = NEXT_RX_IDX(bd_prod);
1007                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1008                 rx_pkt++;
1009 next_cqe:
1010                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1011                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1012
1013                 if (rx_pkt == budget)
1014                         break;
1015         } /* while */
1016
1017         fp->rx_bd_cons = bd_cons;
1018         fp->rx_bd_prod = bd_prod_fw;
1019         fp->rx_comp_cons = sw_comp_cons;
1020         fp->rx_comp_prod = sw_comp_prod;
1021
1022         /* Update producers */
1023         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1024                              fp->rx_sge_prod);
1025
1026         fp->rx_pkt += rx_pkt;
1027         fp->rx_calls++;
1028
1029         return rx_pkt;
1030 }
1031
1032 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1033 {
1034         struct bnx2x_fastpath *fp = fp_cookie;
1035         struct bnx2x *bp = fp->bp;
1036         u8 cos;
1037
1038         DP(NETIF_MSG_INTR,
1039            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1040            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1041         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1042
1043 #ifdef BNX2X_STOP_ON_ERROR
1044         if (unlikely(bp->panic))
1045                 return IRQ_HANDLED;
1046 #endif
1047
1048         /* Handle Rx and Tx according to MSI-X vector */
1049         prefetch(fp->rx_cons_sb);
1050
1051         for_each_cos_in_tx_queue(fp, cos)
1052                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1053
1054         prefetch(&fp->sb_running_index[SM_RX_ID]);
1055         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1056
1057         return IRQ_HANDLED;
1058 }
1059
1060 /* HW Lock for shared dual port PHYs */
1061 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1062 {
1063         mutex_lock(&bp->port.phy_mutex);
1064
1065         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1066 }
1067
1068 void bnx2x_release_phy_lock(struct bnx2x *bp)
1069 {
1070         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1071
1072         mutex_unlock(&bp->port.phy_mutex);
1073 }
1074
1075 /* calculates MF speed according to current linespeed and MF configuration */
1076 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1077 {
1078         u16 line_speed = bp->link_vars.line_speed;
1079         if (IS_MF(bp)) {
1080                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1081                                                    bp->mf_config[BP_VN(bp)]);
1082
1083                 /* Calculate the current MAX line speed limit for the MF
1084                  * devices
1085                  */
1086                 if (IS_MF_SI(bp))
1087                         line_speed = (line_speed * maxCfg) / 100;
1088                 else { /* SD mode */
1089                         u16 vn_max_rate = maxCfg * 100;
1090
1091                         if (vn_max_rate < line_speed)
1092                                 line_speed = vn_max_rate;
1093                 }
1094         }
1095
1096         return line_speed;
1097 }
1098
1099 /**
1100  * bnx2x_fill_report_data - fill link report data to report
1101  *
1102  * @bp:         driver handle
1103  * @data:       link state to update
1104  *
1105  * It uses a none-atomic bit operations because is called under the mutex.
1106  */
1107 static void bnx2x_fill_report_data(struct bnx2x *bp,
1108                                    struct bnx2x_link_report_data *data)
1109 {
1110         u16 line_speed = bnx2x_get_mf_speed(bp);
1111
1112         memset(data, 0, sizeof(*data));
1113
1114         /* Fill the report data: efective line speed */
1115         data->line_speed = line_speed;
1116
1117         /* Link is down */
1118         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1119                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1120                           &data->link_report_flags);
1121
1122         /* Full DUPLEX */
1123         if (bp->link_vars.duplex == DUPLEX_FULL)
1124                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1125
1126         /* Rx Flow Control is ON */
1127         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1128                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1129
1130         /* Tx Flow Control is ON */
1131         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1132                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1133 }
1134
1135 /**
1136  * bnx2x_link_report - report link status to OS.
1137  *
1138  * @bp:         driver handle
1139  *
1140  * Calls the __bnx2x_link_report() under the same locking scheme
1141  * as a link/PHY state managing code to ensure a consistent link
1142  * reporting.
1143  */
1144
1145 void bnx2x_link_report(struct bnx2x *bp)
1146 {
1147         bnx2x_acquire_phy_lock(bp);
1148         __bnx2x_link_report(bp);
1149         bnx2x_release_phy_lock(bp);
1150 }
1151
1152 /**
1153  * __bnx2x_link_report - report link status to OS.
1154  *
1155  * @bp:         driver handle
1156  *
1157  * None atomic inmlementation.
1158  * Should be called under the phy_lock.
1159  */
1160 void __bnx2x_link_report(struct bnx2x *bp)
1161 {
1162         struct bnx2x_link_report_data cur_data;
1163
1164         /* reread mf_cfg */
1165         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1166                 bnx2x_read_mf_cfg(bp);
1167
1168         /* Read the current link report info */
1169         bnx2x_fill_report_data(bp, &cur_data);
1170
1171         /* Don't report link down or exactly the same link status twice */
1172         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1173             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1174                       &bp->last_reported_link.link_report_flags) &&
1175              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1176                       &cur_data.link_report_flags)))
1177                 return;
1178
1179         bp->link_cnt++;
1180
1181         /* We are going to report a new link parameters now -
1182          * remember the current data for the next time.
1183          */
1184         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1185
1186         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1187                      &cur_data.link_report_flags)) {
1188                 netif_carrier_off(bp->dev);
1189                 netdev_err(bp->dev, "NIC Link is Down\n");
1190                 return;
1191         } else {
1192                 const char *duplex;
1193                 const char *flow;
1194
1195                 netif_carrier_on(bp->dev);
1196
1197                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1198                                        &cur_data.link_report_flags))
1199                         duplex = "full";
1200                 else
1201                         duplex = "half";
1202
1203                 /* Handle the FC at the end so that only these flags would be
1204                  * possibly set. This way we may easily check if there is no FC
1205                  * enabled.
1206                  */
1207                 if (cur_data.link_report_flags) {
1208                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1209                                      &cur_data.link_report_flags)) {
1210                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1211                                      &cur_data.link_report_flags))
1212                                         flow = "ON - receive & transmit";
1213                                 else
1214                                         flow = "ON - receive";
1215                         } else {
1216                                 flow = "ON - transmit";
1217                         }
1218                 } else {
1219                         flow = "none";
1220                 }
1221                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1222                             cur_data.line_speed, duplex, flow);
1223         }
1224 }
1225
1226 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1227 {
1228         int i;
1229
1230         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1231                 struct eth_rx_sge *sge;
1232
1233                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1234                 sge->addr_hi =
1235                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1236                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1237
1238                 sge->addr_lo =
1239                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1240                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1241         }
1242 }
1243
1244 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1245                                 struct bnx2x_fastpath *fp, int last)
1246 {
1247         int i;
1248
1249         for (i = 0; i < last; i++) {
1250                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1251                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1252                 u8 *data = first_buf->data;
1253
1254                 if (data == NULL) {
1255                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1256                         continue;
1257                 }
1258                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1259                         dma_unmap_single(&bp->pdev->dev,
1260                                          dma_unmap_addr(first_buf, mapping),
1261                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1262                 bnx2x_frag_free(fp, data);
1263                 first_buf->data = NULL;
1264         }
1265 }
1266
1267 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1268 {
1269         int j;
1270
1271         for_each_rx_queue_cnic(bp, j) {
1272                 struct bnx2x_fastpath *fp = &bp->fp[j];
1273
1274                 fp->rx_bd_cons = 0;
1275
1276                 /* Activate BD ring */
1277                 /* Warning!
1278                  * this will generate an interrupt (to the TSTORM)
1279                  * must only be done after chip is initialized
1280                  */
1281                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1282                                      fp->rx_sge_prod);
1283         }
1284 }
1285
1286 void bnx2x_init_rx_rings(struct bnx2x *bp)
1287 {
1288         int func = BP_FUNC(bp);
1289         u16 ring_prod;
1290         int i, j;
1291
1292         /* Allocate TPA resources */
1293         for_each_eth_queue(bp, j) {
1294                 struct bnx2x_fastpath *fp = &bp->fp[j];
1295
1296                 DP(NETIF_MSG_IFUP,
1297                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1298
1299                 if (!fp->disable_tpa) {
1300                         /* Fill the per-aggregtion pool */
1301                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1302                                 struct bnx2x_agg_info *tpa_info =
1303                                         &fp->tpa_info[i];
1304                                 struct sw_rx_bd *first_buf =
1305                                         &tpa_info->first_buf;
1306
1307                                 first_buf->data = bnx2x_frag_alloc(fp);
1308                                 if (!first_buf->data) {
1309                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1310                                                   j);
1311                                         bnx2x_free_tpa_pool(bp, fp, i);
1312                                         fp->disable_tpa = 1;
1313                                         break;
1314                                 }
1315                                 dma_unmap_addr_set(first_buf, mapping, 0);
1316                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1317                         }
1318
1319                         /* "next page" elements initialization */
1320                         bnx2x_set_next_page_sgl(fp);
1321
1322                         /* set SGEs bit mask */
1323                         bnx2x_init_sge_ring_bit_mask(fp);
1324
1325                         /* Allocate SGEs and initialize the ring elements */
1326                         for (i = 0, ring_prod = 0;
1327                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1328
1329                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1330                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1331                                                   i);
1332                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1333                                                   j);
1334                                         /* Cleanup already allocated elements */
1335                                         bnx2x_free_rx_sge_range(bp, fp,
1336                                                                 ring_prod);
1337                                         bnx2x_free_tpa_pool(bp, fp,
1338                                                             MAX_AGG_QS(bp));
1339                                         fp->disable_tpa = 1;
1340                                         ring_prod = 0;
1341                                         break;
1342                                 }
1343                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1344                         }
1345
1346                         fp->rx_sge_prod = ring_prod;
1347                 }
1348         }
1349
1350         for_each_eth_queue(bp, j) {
1351                 struct bnx2x_fastpath *fp = &bp->fp[j];
1352
1353                 fp->rx_bd_cons = 0;
1354
1355                 /* Activate BD ring */
1356                 /* Warning!
1357                  * this will generate an interrupt (to the TSTORM)
1358                  * must only be done after chip is initialized
1359                  */
1360                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1361                                      fp->rx_sge_prod);
1362
1363                 if (j != 0)
1364                         continue;
1365
1366                 if (CHIP_IS_E1(bp)) {
1367                         REG_WR(bp, BAR_USTRORM_INTMEM +
1368                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1369                                U64_LO(fp->rx_comp_mapping));
1370                         REG_WR(bp, BAR_USTRORM_INTMEM +
1371                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1372                                U64_HI(fp->rx_comp_mapping));
1373                 }
1374         }
1375 }
1376
1377 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1378 {
1379         u8 cos;
1380         struct bnx2x *bp = fp->bp;
1381
1382         for_each_cos_in_tx_queue(fp, cos) {
1383                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1384                 unsigned pkts_compl = 0, bytes_compl = 0;
1385
1386                 u16 sw_prod = txdata->tx_pkt_prod;
1387                 u16 sw_cons = txdata->tx_pkt_cons;
1388
1389                 while (sw_cons != sw_prod) {
1390                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1391                                           &pkts_compl, &bytes_compl);
1392                         sw_cons++;
1393                 }
1394
1395                 netdev_tx_reset_queue(
1396                         netdev_get_tx_queue(bp->dev,
1397                                             txdata->txq_index));
1398         }
1399 }
1400
1401 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1402 {
1403         int i;
1404
1405         for_each_tx_queue_cnic(bp, i) {
1406                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1407         }
1408 }
1409
1410 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1411 {
1412         int i;
1413
1414         for_each_eth_queue(bp, i) {
1415                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1416         }
1417 }
1418
1419 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1420 {
1421         struct bnx2x *bp = fp->bp;
1422         int i;
1423
1424         /* ring wasn't allocated */
1425         if (fp->rx_buf_ring == NULL)
1426                 return;
1427
1428         for (i = 0; i < NUM_RX_BD; i++) {
1429                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1430                 u8 *data = rx_buf->data;
1431
1432                 if (data == NULL)
1433                         continue;
1434                 dma_unmap_single(&bp->pdev->dev,
1435                                  dma_unmap_addr(rx_buf, mapping),
1436                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1437
1438                 rx_buf->data = NULL;
1439                 bnx2x_frag_free(fp, data);
1440         }
1441 }
1442
1443 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1444 {
1445         int j;
1446
1447         for_each_rx_queue_cnic(bp, j) {
1448                 bnx2x_free_rx_bds(&bp->fp[j]);
1449         }
1450 }
1451
1452 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1453 {
1454         int j;
1455
1456         for_each_eth_queue(bp, j) {
1457                 struct bnx2x_fastpath *fp = &bp->fp[j];
1458
1459                 bnx2x_free_rx_bds(fp);
1460
1461                 if (!fp->disable_tpa)
1462                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1463         }
1464 }
1465
1466 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1467 {
1468         bnx2x_free_tx_skbs_cnic(bp);
1469         bnx2x_free_rx_skbs_cnic(bp);
1470 }
1471
1472 void bnx2x_free_skbs(struct bnx2x *bp)
1473 {
1474         bnx2x_free_tx_skbs(bp);
1475         bnx2x_free_rx_skbs(bp);
1476 }
1477
1478 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1479 {
1480         /* load old values */
1481         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1482
1483         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1484                 /* leave all but MAX value */
1485                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1486
1487                 /* set new MAX value */
1488                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1489                                 & FUNC_MF_CFG_MAX_BW_MASK;
1490
1491                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1492         }
1493 }
1494
1495 /**
1496  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1497  *
1498  * @bp:         driver handle
1499  * @nvecs:      number of vectors to be released
1500  */
1501 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1502 {
1503         int i, offset = 0;
1504
1505         if (nvecs == offset)
1506                 return;
1507
1508         /* VFs don't have a default SB */
1509         if (IS_PF(bp)) {
1510                 free_irq(bp->msix_table[offset].vector, bp->dev);
1511                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1512                    bp->msix_table[offset].vector);
1513                 offset++;
1514         }
1515
1516         if (CNIC_SUPPORT(bp)) {
1517                 if (nvecs == offset)
1518                         return;
1519                 offset++;
1520         }
1521
1522         for_each_eth_queue(bp, i) {
1523                 if (nvecs == offset)
1524                         return;
1525                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1526                    i, bp->msix_table[offset].vector);
1527
1528                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1529         }
1530 }
1531
1532 void bnx2x_free_irq(struct bnx2x *bp)
1533 {
1534         if (bp->flags & USING_MSIX_FLAG &&
1535             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1536                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1537
1538                 /* vfs don't have a default status block */
1539                 if (IS_PF(bp))
1540                         nvecs++;
1541
1542                 bnx2x_free_msix_irqs(bp, nvecs);
1543         } else {
1544                 free_irq(bp->dev->irq, bp->dev);
1545         }
1546 }
1547
1548 int bnx2x_enable_msix(struct bnx2x *bp)
1549 {
1550         int msix_vec = 0, i, rc;
1551
1552         /* VFs don't have a default status block */
1553         if (IS_PF(bp)) {
1554                 bp->msix_table[msix_vec].entry = msix_vec;
1555                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1556                                bp->msix_table[0].entry);
1557                 msix_vec++;
1558         }
1559
1560         /* Cnic requires an msix vector for itself */
1561         if (CNIC_SUPPORT(bp)) {
1562                 bp->msix_table[msix_vec].entry = msix_vec;
1563                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1564                                msix_vec, bp->msix_table[msix_vec].entry);
1565                 msix_vec++;
1566         }
1567
1568         /* We need separate vectors for ETH queues only (not FCoE) */
1569         for_each_eth_queue(bp, i) {
1570                 bp->msix_table[msix_vec].entry = msix_vec;
1571                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1572                                msix_vec, msix_vec, i);
1573                 msix_vec++;
1574         }
1575
1576         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1577            msix_vec);
1578
1579         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1580
1581         /*
1582          * reconfigure number of tx/rx queues according to available
1583          * MSI-X vectors
1584          */
1585         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1586                 /* how less vectors we will have? */
1587                 int diff = msix_vec - rc;
1588
1589                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1590
1591                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1592
1593                 if (rc) {
1594                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1595                         goto no_msix;
1596                 }
1597                 /*
1598                  * decrease number of queues by number of unallocated entries
1599                  */
1600                 bp->num_ethernet_queues -= diff;
1601                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1602
1603                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1604                                bp->num_queues);
1605         } else if (rc > 0) {
1606                 /* Get by with single vector */
1607                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1608                 if (rc) {
1609                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1610                                        rc);
1611                         goto no_msix;
1612                 }
1613
1614                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1615                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1616
1617                 BNX2X_DEV_INFO("set number of queues to 1\n");
1618                 bp->num_ethernet_queues = 1;
1619                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1620         } else if (rc < 0) {
1621                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1622                 goto no_msix;
1623         }
1624
1625         bp->flags |= USING_MSIX_FLAG;
1626
1627         return 0;
1628
1629 no_msix:
1630         /* fall to INTx if not enough memory */
1631         if (rc == -ENOMEM)
1632                 bp->flags |= DISABLE_MSI_FLAG;
1633
1634         return rc;
1635 }
1636
1637 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1638 {
1639         int i, rc, offset = 0;
1640
1641         /* no default status block for vf */
1642         if (IS_PF(bp)) {
1643                 rc = request_irq(bp->msix_table[offset++].vector,
1644                                  bnx2x_msix_sp_int, 0,
1645                                  bp->dev->name, bp->dev);
1646                 if (rc) {
1647                         BNX2X_ERR("request sp irq failed\n");
1648                         return -EBUSY;
1649                 }
1650         }
1651
1652         if (CNIC_SUPPORT(bp))
1653                 offset++;
1654
1655         for_each_eth_queue(bp, i) {
1656                 struct bnx2x_fastpath *fp = &bp->fp[i];
1657                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1658                          bp->dev->name, i);
1659
1660                 rc = request_irq(bp->msix_table[offset].vector,
1661                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1662                 if (rc) {
1663                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1664                               bp->msix_table[offset].vector, rc);
1665                         bnx2x_free_msix_irqs(bp, offset);
1666                         return -EBUSY;
1667                 }
1668
1669                 offset++;
1670         }
1671
1672         i = BNX2X_NUM_ETH_QUEUES(bp);
1673         if (IS_PF(bp)) {
1674                 offset = 1 + CNIC_SUPPORT(bp);
1675                 netdev_info(bp->dev,
1676                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1677                             bp->msix_table[0].vector,
1678                             0, bp->msix_table[offset].vector,
1679                             i - 1, bp->msix_table[offset + i - 1].vector);
1680         } else {
1681                 offset = CNIC_SUPPORT(bp);
1682                 netdev_info(bp->dev,
1683                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1684                             0, bp->msix_table[offset].vector,
1685                             i - 1, bp->msix_table[offset + i - 1].vector);
1686         }
1687         return 0;
1688 }
1689
1690 int bnx2x_enable_msi(struct bnx2x *bp)
1691 {
1692         int rc;
1693
1694         rc = pci_enable_msi(bp->pdev);
1695         if (rc) {
1696                 BNX2X_DEV_INFO("MSI is not attainable\n");
1697                 return -1;
1698         }
1699         bp->flags |= USING_MSI_FLAG;
1700
1701         return 0;
1702 }
1703
1704 static int bnx2x_req_irq(struct bnx2x *bp)
1705 {
1706         unsigned long flags;
1707         unsigned int irq;
1708
1709         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1710                 flags = 0;
1711         else
1712                 flags = IRQF_SHARED;
1713
1714         if (bp->flags & USING_MSIX_FLAG)
1715                 irq = bp->msix_table[0].vector;
1716         else
1717                 irq = bp->pdev->irq;
1718
1719         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1720 }
1721
1722 static int bnx2x_setup_irqs(struct bnx2x *bp)
1723 {
1724         int rc = 0;
1725         if (bp->flags & USING_MSIX_FLAG &&
1726             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1727                 rc = bnx2x_req_msix_irqs(bp);
1728                 if (rc)
1729                         return rc;
1730         } else {
1731                 rc = bnx2x_req_irq(bp);
1732                 if (rc) {
1733                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1734                         return rc;
1735                 }
1736                 if (bp->flags & USING_MSI_FLAG) {
1737                         bp->dev->irq = bp->pdev->irq;
1738                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1739                                     bp->dev->irq);
1740                 }
1741                 if (bp->flags & USING_MSIX_FLAG) {
1742                         bp->dev->irq = bp->msix_table[0].vector;
1743                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1744                                     bp->dev->irq);
1745                 }
1746         }
1747
1748         return 0;
1749 }
1750
1751 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1752 {
1753         int i;
1754
1755         for_each_rx_queue_cnic(bp, i)
1756                 napi_enable(&bnx2x_fp(bp, i, napi));
1757 }
1758
1759 static void bnx2x_napi_enable(struct bnx2x *bp)
1760 {
1761         int i;
1762
1763         for_each_eth_queue(bp, i)
1764                 napi_enable(&bnx2x_fp(bp, i, napi));
1765 }
1766
1767 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1768 {
1769         int i;
1770
1771         for_each_rx_queue_cnic(bp, i)
1772                 napi_disable(&bnx2x_fp(bp, i, napi));
1773 }
1774
1775 static void bnx2x_napi_disable(struct bnx2x *bp)
1776 {
1777         int i;
1778
1779         for_each_eth_queue(bp, i)
1780                 napi_disable(&bnx2x_fp(bp, i, napi));
1781 }
1782
1783 void bnx2x_netif_start(struct bnx2x *bp)
1784 {
1785         if (netif_running(bp->dev)) {
1786                 bnx2x_napi_enable(bp);
1787                 if (CNIC_LOADED(bp))
1788                         bnx2x_napi_enable_cnic(bp);
1789                 bnx2x_int_enable(bp);
1790                 if (bp->state == BNX2X_STATE_OPEN)
1791                         netif_tx_wake_all_queues(bp->dev);
1792         }
1793 }
1794
1795 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1796 {
1797         bnx2x_int_disable_sync(bp, disable_hw);
1798         bnx2x_napi_disable(bp);
1799         if (CNIC_LOADED(bp))
1800                 bnx2x_napi_disable_cnic(bp);
1801 }
1802
1803 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1804 {
1805         struct bnx2x *bp = netdev_priv(dev);
1806
1807         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1808                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1809                 u16 ether_type = ntohs(hdr->h_proto);
1810
1811                 /* Skip VLAN tag if present */
1812                 if (ether_type == ETH_P_8021Q) {
1813                         struct vlan_ethhdr *vhdr =
1814                                 (struct vlan_ethhdr *)skb->data;
1815
1816                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1817                 }
1818
1819                 /* If ethertype is FCoE or FIP - use FCoE ring */
1820                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1821                         return bnx2x_fcoe_tx(bp, txq_index);
1822         }
1823
1824         /* select a non-FCoE queue */
1825         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1826 }
1827
1828 void bnx2x_set_num_queues(struct bnx2x *bp)
1829 {
1830         /* RSS queues */
1831         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1832
1833         /* override in STORAGE SD modes */
1834         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1835                 bp->num_ethernet_queues = 1;
1836
1837         /* Add special queues */
1838         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1839         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1840
1841         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1842 }
1843
1844 /**
1845  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1846  *
1847  * @bp:         Driver handle
1848  *
1849  * We currently support for at most 16 Tx queues for each CoS thus we will
1850  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1851  * bp->max_cos.
1852  *
1853  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1854  * index after all ETH L2 indices.
1855  *
1856  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1857  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1858  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1859  *
1860  * The proper configuration of skb->queue_mapping is handled by
1861  * bnx2x_select_queue() and __skb_tx_hash().
1862  *
1863  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1864  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1865  */
1866 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1867 {
1868         int rc, tx, rx;
1869
1870         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1871         rx = BNX2X_NUM_ETH_QUEUES(bp);
1872
1873 /* account for fcoe queue */
1874         if (include_cnic && !NO_FCOE(bp)) {
1875                 rx++;
1876                 tx++;
1877         }
1878
1879         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1880         if (rc) {
1881                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1882                 return rc;
1883         }
1884         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1885         if (rc) {
1886                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1887                 return rc;
1888         }
1889
1890         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1891                           tx, rx);
1892
1893         return rc;
1894 }
1895
1896 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1897 {
1898         int i;
1899
1900         for_each_queue(bp, i) {
1901                 struct bnx2x_fastpath *fp = &bp->fp[i];
1902                 u32 mtu;
1903
1904                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1905                 if (IS_FCOE_IDX(i))
1906                         /*
1907                          * Although there are no IP frames expected to arrive to
1908                          * this ring we still want to add an
1909                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1910                          * overrun attack.
1911                          */
1912                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1913                 else
1914                         mtu = bp->dev->mtu;
1915                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1916                                   IP_HEADER_ALIGNMENT_PADDING +
1917                                   ETH_OVREHEAD +
1918                                   mtu +
1919                                   BNX2X_FW_RX_ALIGN_END;
1920                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1921                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1922                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1923                 else
1924                         fp->rx_frag_size = 0;
1925         }
1926 }
1927
1928 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1929 {
1930         int i;
1931         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1932
1933         /* Prepare the initial contents fo the indirection table if RSS is
1934          * enabled
1935          */
1936         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1937                 bp->rss_conf_obj.ind_table[i] =
1938                         bp->fp->cl_id +
1939                         ethtool_rxfh_indir_default(i, num_eth_queues);
1940
1941         /*
1942          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1943          * per-port, so if explicit configuration is needed , do it only
1944          * for a PMF.
1945          *
1946          * For 57712 and newer on the other hand it's a per-function
1947          * configuration.
1948          */
1949         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1950 }
1951
1952 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1953                         bool config_hash)
1954 {
1955         struct bnx2x_config_rss_params params = {NULL};
1956
1957         /* Although RSS is meaningless when there is a single HW queue we
1958          * still need it enabled in order to have HW Rx hash generated.
1959          *
1960          * if (!is_eth_multi(bp))
1961          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1962          */
1963
1964         params.rss_obj = rss_obj;
1965
1966         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1967
1968         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1969
1970         /* RSS configuration */
1971         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1972         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1973         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1974         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1975         if (rss_obj->udp_rss_v4)
1976                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1977         if (rss_obj->udp_rss_v6)
1978                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1979
1980         /* Hash bits */
1981         params.rss_result_mask = MULTI_MASK;
1982
1983         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1984
1985         if (config_hash) {
1986                 /* RSS keys */
1987                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1988                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1989         }
1990
1991         return bnx2x_config_rss(bp, &params);
1992 }
1993
1994 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1995 {
1996         struct bnx2x_func_state_params func_params = {NULL};
1997
1998         /* Prepare parameters for function state transitions */
1999         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2000
2001         func_params.f_obj = &bp->func_obj;
2002         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2003
2004         func_params.params.hw_init.load_phase = load_code;
2005
2006         return bnx2x_func_state_change(bp, &func_params);
2007 }
2008
2009 /*
2010  * Cleans the object that have internal lists without sending
2011  * ramrods. Should be run when interrutps are disabled.
2012  */
2013 void bnx2x_squeeze_objects(struct bnx2x *bp)
2014 {
2015         int rc;
2016         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2017         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2018         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2019
2020         /***************** Cleanup MACs' object first *************************/
2021
2022         /* Wait for completion of requested */
2023         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2024         /* Perform a dry cleanup */
2025         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2026
2027         /* Clean ETH primary MAC */
2028         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2029         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2030                                  &ramrod_flags);
2031         if (rc != 0)
2032                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2033
2034         /* Cleanup UC list */
2035         vlan_mac_flags = 0;
2036         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2037         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2038                                  &ramrod_flags);
2039         if (rc != 0)
2040                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2041
2042         /***************** Now clean mcast object *****************************/
2043         rparam.mcast_obj = &bp->mcast_obj;
2044         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2045
2046         /* Add a DEL command... */
2047         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2048         if (rc < 0)
2049                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2050                           rc);
2051
2052         /* ...and wait until all pending commands are cleared */
2053         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2054         while (rc != 0) {
2055                 if (rc < 0) {
2056                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2057                                   rc);
2058                         return;
2059                 }
2060
2061                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2062         }
2063 }
2064
2065 #ifndef BNX2X_STOP_ON_ERROR
2066 #define LOAD_ERROR_EXIT(bp, label) \
2067         do { \
2068                 (bp)->state = BNX2X_STATE_ERROR; \
2069                 goto label; \
2070         } while (0)
2071
2072 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2073         do { \
2074                 bp->cnic_loaded = false; \
2075                 goto label; \
2076         } while (0)
2077 #else /*BNX2X_STOP_ON_ERROR*/
2078 #define LOAD_ERROR_EXIT(bp, label) \
2079         do { \
2080                 (bp)->state = BNX2X_STATE_ERROR; \
2081                 (bp)->panic = 1; \
2082                 return -EBUSY; \
2083         } while (0)
2084 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2085         do { \
2086                 bp->cnic_loaded = false; \
2087                 (bp)->panic = 1; \
2088                 return -EBUSY; \
2089         } while (0)
2090 #endif /*BNX2X_STOP_ON_ERROR*/
2091
2092 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2093 {
2094         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2095                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2096         return;
2097 }
2098
2099 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2100 {
2101         int num_groups, vf_headroom = 0;
2102         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2103
2104         /* number of queues for statistics is number of eth queues + FCoE */
2105         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2106
2107         /* Total number of FW statistics requests =
2108          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2109          * and fcoe l2 queue) stats + num of queues (which includes another 1
2110          * for fcoe l2 queue if applicable)
2111          */
2112         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2113
2114         /* vf stats appear in the request list, but their data is allocated by
2115          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2116          * it is used to determine where to place the vf stats queries in the
2117          * request struct
2118          */
2119         if (IS_SRIOV(bp))
2120                 vf_headroom = bnx2x_vf_headroom(bp);
2121
2122         /* Request is built from stats_query_header and an array of
2123          * stats_query_cmd_group each of which contains
2124          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2125          * configured in the stats_query_header.
2126          */
2127         num_groups =
2128                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2129                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2130                  1 : 0));
2131
2132         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2133            bp->fw_stats_num, vf_headroom, num_groups);
2134         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2135                 num_groups * sizeof(struct stats_query_cmd_group);
2136
2137         /* Data for statistics requests + stats_counter
2138          * stats_counter holds per-STORM counters that are incremented
2139          * when STORM has finished with the current request.
2140          * memory for FCoE offloaded statistics are counted anyway,
2141          * even if they will not be sent.
2142          * VF stats are not accounted for here as the data of VF stats is stored
2143          * in memory allocated by the VF, not here.
2144          */
2145         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2146                 sizeof(struct per_pf_stats) +
2147                 sizeof(struct fcoe_statistics_params) +
2148                 sizeof(struct per_queue_stats) * num_queue_stats +
2149                 sizeof(struct stats_counter);
2150
2151         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2152                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2153
2154         /* Set shortcuts */
2155         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2156         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2157         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2158                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2159         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2160                 bp->fw_stats_req_sz;
2161
2162         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x",
2163            U64_HI(bp->fw_stats_req_mapping),
2164            U64_LO(bp->fw_stats_req_mapping));
2165         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x",
2166            U64_HI(bp->fw_stats_data_mapping),
2167            U64_LO(bp->fw_stats_data_mapping));
2168         return 0;
2169
2170 alloc_mem_err:
2171         bnx2x_free_fw_stats_mem(bp);
2172         BNX2X_ERR("Can't allocate FW stats memory\n");
2173         return -ENOMEM;
2174 }
2175
2176 /* send load request to mcp and analyze response */
2177 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2178 {
2179         /* init fw_seq */
2180         bp->fw_seq =
2181                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2182                  DRV_MSG_SEQ_NUMBER_MASK);
2183         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2184
2185         /* Get current FW pulse sequence */
2186         bp->fw_drv_pulse_wr_seq =
2187                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2188                  DRV_PULSE_SEQ_MASK);
2189         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2190
2191         /* load request */
2192         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2193                                         DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2194
2195         /* if mcp fails to respond we must abort */
2196         if (!(*load_code)) {
2197                 BNX2X_ERR("MCP response failure, aborting\n");
2198                 return -EBUSY;
2199         }
2200
2201         /* If mcp refused (e.g. other port is in diagnostic mode) we
2202          * must abort
2203          */
2204         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2205                 BNX2X_ERR("MCP refused load request, aborting\n");
2206                 return -EBUSY;
2207         }
2208         return 0;
2209 }
2210
2211 /* check whether another PF has already loaded FW to chip. In
2212  * virtualized environments a pf from another VM may have already
2213  * initialized the device including loading FW
2214  */
2215 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2216 {
2217         /* is another pf loaded on this engine? */
2218         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2219             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2220                 /* build my FW version dword */
2221                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2222                         (BCM_5710_FW_MINOR_VERSION << 8) +
2223                         (BCM_5710_FW_REVISION_VERSION << 16) +
2224                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2225
2226                 /* read loaded FW from chip */
2227                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2228
2229                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2230                    loaded_fw, my_fw);
2231
2232                 /* abort nic load if version mismatch */
2233                 if (my_fw != loaded_fw) {
2234                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. aborting\n",
2235                                   loaded_fw, my_fw);
2236                         return -EBUSY;
2237                 }
2238         }
2239         return 0;
2240 }
2241
2242 /* returns the "mcp load_code" according to global load_count array */
2243 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2244 {
2245         int path = BP_PATH(bp);
2246
2247         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2248            path, load_count[path][0], load_count[path][1],
2249            load_count[path][2]);
2250         load_count[path][0]++;
2251         load_count[path][1 + port]++;
2252         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2253            path, load_count[path][0], load_count[path][1],
2254            load_count[path][2]);
2255         if (load_count[path][0] == 1)
2256                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2257         else if (load_count[path][1 + port] == 1)
2258                 return FW_MSG_CODE_DRV_LOAD_PORT;
2259         else
2260                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2261 }
2262
2263 /* mark PMF if applicable */
2264 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2265 {
2266         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2267             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2268             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2269                 bp->port.pmf = 1;
2270                 /* We need the barrier to ensure the ordering between the
2271                  * writing to bp->port.pmf here and reading it from the
2272                  * bnx2x_periodic_task().
2273                  */
2274                 smp_mb();
2275         } else {
2276                 bp->port.pmf = 0;
2277         }
2278
2279         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2280 }
2281
2282 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2283 {
2284         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2285              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2286             (bp->common.shmem2_base)) {
2287                 if (SHMEM2_HAS(bp, dcc_support))
2288                         SHMEM2_WR(bp, dcc_support,
2289                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2290                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2291                 if (SHMEM2_HAS(bp, afex_driver_support))
2292                         SHMEM2_WR(bp, afex_driver_support,
2293                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2294         }
2295
2296         /* Set AFEX default VLAN tag to an invalid value */
2297         bp->afex_def_vlan_tag = -1;
2298 }
2299
2300 /**
2301  * bnx2x_bz_fp - zero content of the fastpath structure.
2302  *
2303  * @bp:         driver handle
2304  * @index:      fastpath index to be zeroed
2305  *
2306  * Makes sure the contents of the bp->fp[index].napi is kept
2307  * intact.
2308  */
2309 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2310 {
2311         struct bnx2x_fastpath *fp = &bp->fp[index];
2312
2313         int cos;
2314         struct napi_struct orig_napi = fp->napi;
2315         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2316         /* bzero bnx2x_fastpath contents */
2317         if (fp->tpa_info)
2318                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2319                        sizeof(struct bnx2x_agg_info));
2320         memset(fp, 0, sizeof(*fp));
2321
2322         /* Restore the NAPI object as it has been already initialized */
2323         fp->napi = orig_napi;
2324         fp->tpa_info = orig_tpa_info;
2325         fp->bp = bp;
2326         fp->index = index;
2327         if (IS_ETH_FP(fp))
2328                 fp->max_cos = bp->max_cos;
2329         else
2330                 /* Special queues support only one CoS */
2331                 fp->max_cos = 1;
2332
2333         /* Init txdata pointers */
2334         if (IS_FCOE_FP(fp))
2335                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2336         if (IS_ETH_FP(fp))
2337                 for_each_cos_in_tx_queue(fp, cos)
2338                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2339                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2340
2341         /*
2342          * set the tpa flag for each queue. The tpa flag determines the queue
2343          * minimal size so it must be set prior to queue memory allocation
2344          */
2345         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2346                                   (bp->flags & GRO_ENABLE_FLAG &&
2347                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2348         if (bp->flags & TPA_ENABLE_FLAG)
2349                 fp->mode = TPA_MODE_LRO;
2350         else if (bp->flags & GRO_ENABLE_FLAG)
2351                 fp->mode = TPA_MODE_GRO;
2352
2353         /* We don't want TPA on an FCoE L2 ring */
2354         if (IS_FCOE_FP(fp))
2355                 fp->disable_tpa = 1;
2356 }
2357
2358 int bnx2x_load_cnic(struct bnx2x *bp)
2359 {
2360         int i, rc, port = BP_PORT(bp);
2361
2362         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2363
2364         mutex_init(&bp->cnic_mutex);
2365
2366         if (IS_PF(bp)) {
2367                 rc = bnx2x_alloc_mem_cnic(bp);
2368                 if (rc) {
2369                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2370                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2371                 }
2372         }
2373
2374         rc = bnx2x_alloc_fp_mem_cnic(bp);
2375         if (rc) {
2376                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2377                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2378         }
2379
2380         /* Update the number of queues with the cnic queues */
2381         rc = bnx2x_set_real_num_queues(bp, 1);
2382         if (rc) {
2383                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2384                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2385         }
2386
2387         /* Add all CNIC NAPI objects */
2388         bnx2x_add_all_napi_cnic(bp);
2389         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2390         bnx2x_napi_enable_cnic(bp);
2391
2392         rc = bnx2x_init_hw_func_cnic(bp);
2393         if (rc)
2394                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2395
2396         bnx2x_nic_init_cnic(bp);
2397
2398         if (IS_PF(bp)) {
2399                 /* Enable Timer scan */
2400                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2401
2402                 /* setup cnic queues */
2403                 for_each_cnic_queue(bp, i) {
2404                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2405                         if (rc) {
2406                                 BNX2X_ERR("Queue setup failed\n");
2407                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2408                         }
2409                 }
2410         }
2411
2412         /* Initialize Rx filter. */
2413         netif_addr_lock_bh(bp->dev);
2414         bnx2x_set_rx_mode(bp->dev);
2415         netif_addr_unlock_bh(bp->dev);
2416
2417         /* re-read iscsi info */
2418         bnx2x_get_iscsi_info(bp);
2419         bnx2x_setup_cnic_irq_info(bp);
2420         bnx2x_setup_cnic_info(bp);
2421         bp->cnic_loaded = true;
2422         if (bp->state == BNX2X_STATE_OPEN)
2423                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2424
2425
2426         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2427
2428         return 0;
2429
2430 #ifndef BNX2X_STOP_ON_ERROR
2431 load_error_cnic2:
2432         /* Disable Timer scan */
2433         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2434
2435 load_error_cnic1:
2436         bnx2x_napi_disable_cnic(bp);
2437         /* Update the number of queues without the cnic queues */
2438         rc = bnx2x_set_real_num_queues(bp, 0);
2439         if (rc)
2440                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2441 load_error_cnic0:
2442         BNX2X_ERR("CNIC-related load failed\n");
2443         bnx2x_free_fp_mem_cnic(bp);
2444         bnx2x_free_mem_cnic(bp);
2445         return rc;
2446 #endif /* ! BNX2X_STOP_ON_ERROR */
2447 }
2448
2449 /* must be called with rtnl_lock */
2450 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2451 {
2452         int port = BP_PORT(bp);
2453         int i, rc = 0, load_code = 0;
2454
2455         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2456         DP(NETIF_MSG_IFUP,
2457            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2458
2459 #ifdef BNX2X_STOP_ON_ERROR
2460         if (unlikely(bp->panic)) {
2461                 BNX2X_ERR("Can't load NIC when there is panic\n");
2462                 return -EPERM;
2463         }
2464 #endif
2465
2466         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2467
2468         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2469         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2470                 &bp->last_reported_link.link_report_flags);
2471
2472         if (IS_PF(bp))
2473                 /* must be called before memory allocation and HW init */
2474                 bnx2x_ilt_set_info(bp);
2475
2476         /*
2477          * Zero fastpath structures preserving invariants like napi, which are
2478          * allocated only once, fp index, max_cos, bp pointer.
2479          * Also set fp->disable_tpa and txdata_ptr.
2480          */
2481         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2482         for_each_queue(bp, i)
2483                 bnx2x_bz_fp(bp, i);
2484         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2485                                   bp->num_cnic_queues) *
2486                                   sizeof(struct bnx2x_fp_txdata));
2487
2488         bp->fcoe_init = false;
2489
2490         /* Set the receive queues buffer size */
2491         bnx2x_set_rx_buf_size(bp);
2492
2493         if (IS_PF(bp)) {
2494                 rc = bnx2x_alloc_mem(bp);
2495                 if (rc) {
2496                         BNX2X_ERR("Unable to allocate bp memory\n");
2497                         return rc;
2498                 }
2499         }
2500
2501         /* Allocated memory for FW statistics  */
2502         if (bnx2x_alloc_fw_stats_mem(bp))
2503                 LOAD_ERROR_EXIT(bp, load_error0);
2504
2505         /* need to be done after alloc mem, since it's self adjusting to amount
2506          * of memory available for RSS queues
2507          */
2508         rc = bnx2x_alloc_fp_mem(bp);
2509         if (rc) {
2510                 BNX2X_ERR("Unable to allocate memory for fps\n");
2511                 LOAD_ERROR_EXIT(bp, load_error0);
2512         }
2513
2514         /* request pf to initialize status blocks */
2515         if (IS_VF(bp)) {
2516                 rc = bnx2x_vfpf_init(bp);
2517                 if (rc)
2518                         LOAD_ERROR_EXIT(bp, load_error0);
2519         }
2520
2521         /* As long as bnx2x_alloc_mem() may possibly update
2522          * bp->num_queues, bnx2x_set_real_num_queues() should always
2523          * come after it. At this stage cnic queues are not counted.
2524          */
2525         rc = bnx2x_set_real_num_queues(bp, 0);
2526         if (rc) {
2527                 BNX2X_ERR("Unable to set real_num_queues\n");
2528                 LOAD_ERROR_EXIT(bp, load_error0);
2529         }
2530
2531         /* configure multi cos mappings in kernel.
2532          * this configuration may be overriden by a multi class queue discipline
2533          * or by a dcbx negotiation result.
2534          */
2535         bnx2x_setup_tc(bp->dev, bp->max_cos);
2536
2537         /* Add all NAPI objects */
2538         bnx2x_add_all_napi(bp);
2539         DP(NETIF_MSG_IFUP, "napi added\n");
2540         bnx2x_napi_enable(bp);
2541
2542         if (IS_PF(bp)) {
2543                 /* set pf load just before approaching the MCP */
2544                 bnx2x_set_pf_load(bp);
2545
2546                 /* if mcp exists send load request and analyze response */
2547                 if (!BP_NOMCP(bp)) {
2548                         /* attempt to load pf */
2549                         rc = bnx2x_nic_load_request(bp, &load_code);
2550                         if (rc)
2551                                 LOAD_ERROR_EXIT(bp, load_error1);
2552
2553                         /* what did mcp say? */
2554                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2555                         if (rc) {
2556                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2557                                 LOAD_ERROR_EXIT(bp, load_error2);
2558                         }
2559                 } else {
2560                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2561                 }
2562
2563                 /* mark pmf if applicable */
2564                 bnx2x_nic_load_pmf(bp, load_code);
2565
2566                 /* Init Function state controlling object */
2567                 bnx2x__init_func_obj(bp);
2568
2569                 /* Initialize HW */
2570                 rc = bnx2x_init_hw(bp, load_code);
2571                 if (rc) {
2572                         BNX2X_ERR("HW init failed, aborting\n");
2573                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2574                         LOAD_ERROR_EXIT(bp, load_error2);
2575                 }
2576         }
2577
2578         /* Connect to IRQs */
2579         rc = bnx2x_setup_irqs(bp);
2580         if (rc) {
2581                 BNX2X_ERR("setup irqs failed\n");
2582                 if (IS_PF(bp))
2583                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2584                 LOAD_ERROR_EXIT(bp, load_error2);
2585         }
2586
2587         /* Setup NIC internals and enable interrupts */
2588         bnx2x_nic_init(bp, load_code);
2589
2590         /* Init per-function objects */
2591         if (IS_PF(bp)) {
2592                 bnx2x_init_bp_objs(bp);
2593                 bnx2x_iov_nic_init(bp);
2594
2595                 /* Set AFEX default VLAN tag to an invalid value */
2596                 bp->afex_def_vlan_tag = -1;
2597                 bnx2x_nic_load_afex_dcc(bp, load_code);
2598                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2599                 rc = bnx2x_func_start(bp);
2600                 if (rc) {
2601                         BNX2X_ERR("Function start failed!\n");
2602                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2603
2604                         LOAD_ERROR_EXIT(bp, load_error3);
2605                 }
2606
2607                 /* Send LOAD_DONE command to MCP */
2608                 if (!BP_NOMCP(bp)) {
2609                         load_code = bnx2x_fw_command(bp,
2610                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2611                         if (!load_code) {
2612                                 BNX2X_ERR("MCP response failure, aborting\n");
2613                                 rc = -EBUSY;
2614                                 LOAD_ERROR_EXIT(bp, load_error3);
2615                         }
2616                 }
2617
2618                 /* setup the leading queue */
2619                 rc = bnx2x_setup_leading(bp);
2620                 if (rc) {
2621                         BNX2X_ERR("Setup leading failed!\n");
2622                         LOAD_ERROR_EXIT(bp, load_error3);
2623                 }
2624
2625                 /* set up the rest of the queues */
2626                 for_each_nondefault_eth_queue(bp, i) {
2627                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2628                         if (rc) {
2629                                 BNX2X_ERR("Queue setup failed\n");
2630                                 LOAD_ERROR_EXIT(bp, load_error3);
2631                         }
2632                 }
2633
2634                 /* setup rss */
2635                 rc = bnx2x_init_rss_pf(bp);
2636                 if (rc) {
2637                         BNX2X_ERR("PF RSS init failed\n");
2638                         LOAD_ERROR_EXIT(bp, load_error3);
2639                 }
2640
2641         } else { /* vf */
2642                 for_each_eth_queue(bp, i) {
2643                         rc = bnx2x_vfpf_setup_q(bp, i);
2644                         if (rc) {
2645                                 BNX2X_ERR("Queue setup failed\n");
2646                                 LOAD_ERROR_EXIT(bp, load_error3);
2647                         }
2648                 }
2649         }
2650
2651         /* Now when Clients are configured we are ready to work */
2652         bp->state = BNX2X_STATE_OPEN;
2653
2654         /* Configure a ucast MAC */
2655         if (IS_PF(bp))
2656                 rc = bnx2x_set_eth_mac(bp, true);
2657         else /* vf */
2658                 rc = bnx2x_vfpf_set_mac(bp);
2659         if (rc) {
2660                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2661                 LOAD_ERROR_EXIT(bp, load_error3);
2662         }
2663
2664         if (IS_PF(bp) && bp->pending_max) {
2665                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2666                 bp->pending_max = 0;
2667         }
2668
2669         if (bp->port.pmf) {
2670                 rc = bnx2x_initial_phy_init(bp, load_mode);
2671                 if (rc)
2672                         LOAD_ERROR_EXIT(bp, load_error3);
2673         }
2674         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2675
2676         /* Start fast path */
2677
2678         /* Initialize Rx filter. */
2679         netif_addr_lock_bh(bp->dev);
2680         bnx2x_set_rx_mode(bp->dev);
2681         netif_addr_unlock_bh(bp->dev);
2682
2683         /* Start the Tx */
2684         switch (load_mode) {
2685         case LOAD_NORMAL:
2686                 /* Tx queue should be only reenabled */
2687                 netif_tx_wake_all_queues(bp->dev);
2688                 break;
2689
2690         case LOAD_OPEN:
2691                 netif_tx_start_all_queues(bp->dev);
2692                 smp_mb__after_clear_bit();
2693                 break;
2694
2695         case LOAD_DIAG:
2696         case LOAD_LOOPBACK_EXT:
2697                 bp->state = BNX2X_STATE_DIAG;
2698                 break;
2699
2700         default:
2701                 break;
2702         }
2703
2704         if (bp->port.pmf)
2705                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2706         else
2707                 bnx2x__link_status_update(bp);
2708
2709         /* start the timer */
2710         mod_timer(&bp->timer, jiffies + bp->current_interval);
2711
2712         if (CNIC_ENABLED(bp))
2713                 bnx2x_load_cnic(bp);
2714
2715         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2716                 /* mark driver is loaded in shmem2 */
2717                 u32 val;
2718                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2719                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2720                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2721                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2722         }
2723
2724         /* Wait for all pending SP commands to complete */
2725         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2726                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2727                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2728                 return -EBUSY;
2729         }
2730
2731         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2732         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2733                 bnx2x_dcbx_init(bp, false);
2734
2735         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2736
2737         return 0;
2738
2739 #ifndef BNX2X_STOP_ON_ERROR
2740 load_error3:
2741         if (IS_PF(bp)) {
2742                 bnx2x_int_disable_sync(bp, 1);
2743
2744                 /* Clean queueable objects */
2745                 bnx2x_squeeze_objects(bp);
2746         }
2747
2748         /* Free SKBs, SGEs, TPA pool and driver internals */
2749         bnx2x_free_skbs(bp);
2750         for_each_rx_queue(bp, i)
2751                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2752
2753         /* Release IRQs */
2754         bnx2x_free_irq(bp);
2755 load_error2:
2756         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2757                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2758                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2759         }
2760
2761         bp->port.pmf = 0;
2762 load_error1:
2763         bnx2x_napi_disable(bp);
2764         bnx2x_del_all_napi(bp);
2765
2766         /* clear pf_load status, as it was already set */
2767         if (IS_PF(bp))
2768                 bnx2x_clear_pf_load(bp);
2769 load_error0:
2770         bnx2x_free_fp_mem(bp);
2771         bnx2x_free_fw_stats_mem(bp);
2772         bnx2x_free_mem(bp);
2773
2774         return rc;
2775 #endif /* ! BNX2X_STOP_ON_ERROR */
2776 }
2777
2778 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2779 {
2780         u8 rc = 0, cos, i;
2781
2782         /* Wait until tx fastpath tasks complete */
2783         for_each_tx_queue(bp, i) {
2784                 struct bnx2x_fastpath *fp = &bp->fp[i];
2785
2786                 for_each_cos_in_tx_queue(fp, cos)
2787                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2788                 if (rc)
2789                         return rc;
2790         }
2791         return 0;
2792 }
2793
2794 /* must be called with rtnl_lock */
2795 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2796 {
2797         int i;
2798         bool global = false;
2799
2800         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2801
2802         /* mark driver is unloaded in shmem2 */
2803         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2804                 u32 val;
2805                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2806                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2807                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2808         }
2809
2810         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2811             (bp->state == BNX2X_STATE_CLOSED ||
2812              bp->state == BNX2X_STATE_ERROR)) {
2813                 /* We can get here if the driver has been unloaded
2814                  * during parity error recovery and is either waiting for a
2815                  * leader to complete or for other functions to unload and
2816                  * then ifdown has been issued. In this case we want to
2817                  * unload and let other functions to complete a recovery
2818                  * process.
2819                  */
2820                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2821                 bp->is_leader = 0;
2822                 bnx2x_release_leader_lock(bp);
2823                 smp_mb();
2824
2825                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2826                 BNX2X_ERR("Can't unload in closed or error state\n");
2827                 return -EINVAL;
2828         }
2829
2830         /* Nothing to do during unload if previous bnx2x_nic_load()
2831          * have not completed succesfully - all resourses are released.
2832          *
2833          * we can get here only after unsuccessful ndo_* callback, during which
2834          * dev->IFF_UP flag is still on.
2835          */
2836         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2837                 return 0;
2838
2839         /* It's important to set the bp->state to the value different from
2840          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2841          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2842          */
2843         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2844         smp_mb();
2845
2846         if (CNIC_LOADED(bp))
2847                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2848
2849         /* Stop Tx */
2850         bnx2x_tx_disable(bp);
2851         netdev_reset_tc(bp->dev);
2852
2853         bp->rx_mode = BNX2X_RX_MODE_NONE;
2854
2855         del_timer_sync(&bp->timer);
2856
2857         if (IS_PF(bp)) {
2858                 /* Set ALWAYS_ALIVE bit in shmem */
2859                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2860                 bnx2x_drv_pulse(bp);
2861                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2862                 bnx2x_save_statistics(bp);
2863         }
2864
2865         /* wait till consumers catch up with producers in all queues */
2866         bnx2x_drain_tx_queues(bp);
2867
2868         /* if VF indicate to PF this function is going down (PF will delete sp
2869          * elements and clear initializations
2870          */
2871         if (IS_VF(bp))
2872                 bnx2x_vfpf_close_vf(bp);
2873         else if (unload_mode != UNLOAD_RECOVERY)
2874                 /* if this is a normal/close unload need to clean up chip*/
2875                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2876         else {
2877                 /* Send the UNLOAD_REQUEST to the MCP */
2878                 bnx2x_send_unload_req(bp, unload_mode);
2879
2880                 /*
2881                  * Prevent transactions to host from the functions on the
2882                  * engine that doesn't reset global blocks in case of global
2883                  * attention once gloabl blocks are reset and gates are opened
2884                  * (the engine which leader will perform the recovery
2885                  * last).
2886                  */
2887                 if (!CHIP_IS_E1x(bp))
2888                         bnx2x_pf_disable(bp);
2889
2890                 /* Disable HW interrupts, NAPI */
2891                 bnx2x_netif_stop(bp, 1);
2892                 /* Delete all NAPI objects */
2893                 bnx2x_del_all_napi(bp);
2894                 if (CNIC_LOADED(bp))
2895                         bnx2x_del_all_napi_cnic(bp);
2896                 /* Release IRQs */
2897                 bnx2x_free_irq(bp);
2898
2899                 /* Report UNLOAD_DONE to MCP */
2900                 bnx2x_send_unload_done(bp, false);
2901         }
2902
2903         /*
2904          * At this stage no more interrupts will arrive so we may safly clean
2905          * the queueable objects here in case they failed to get cleaned so far.
2906          */
2907         if (IS_PF(bp))
2908                 bnx2x_squeeze_objects(bp);
2909
2910         /* There should be no more pending SP commands at this stage */
2911         bp->sp_state = 0;
2912
2913         bp->port.pmf = 0;
2914
2915         /* Free SKBs, SGEs, TPA pool and driver internals */
2916         bnx2x_free_skbs(bp);
2917         if (CNIC_LOADED(bp))
2918                 bnx2x_free_skbs_cnic(bp);
2919         for_each_rx_queue(bp, i)
2920                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2921
2922         bnx2x_free_fp_mem(bp);
2923         if (CNIC_LOADED(bp))
2924                 bnx2x_free_fp_mem_cnic(bp);
2925
2926         if (IS_PF(bp)) {
2927                 bnx2x_free_mem(bp);
2928                 if (CNIC_LOADED(bp))
2929                         bnx2x_free_mem_cnic(bp);
2930         }
2931         bp->state = BNX2X_STATE_CLOSED;
2932         bp->cnic_loaded = false;
2933
2934         /* Check if there are pending parity attentions. If there are - set
2935          * RECOVERY_IN_PROGRESS.
2936          */
2937         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2938                 bnx2x_set_reset_in_progress(bp);
2939
2940                 /* Set RESET_IS_GLOBAL if needed */
2941                 if (global)
2942                         bnx2x_set_reset_global(bp);
2943         }
2944
2945
2946         /* The last driver must disable a "close the gate" if there is no
2947          * parity attention or "process kill" pending.
2948          */
2949         if (IS_PF(bp) &&
2950             !bnx2x_clear_pf_load(bp) &&
2951             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2952                 bnx2x_disable_close_the_gate(bp);
2953
2954         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2955
2956         return 0;
2957 }
2958
2959 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2960 {
2961         u16 pmcsr;
2962
2963         /* If there is no power capability, silently succeed */
2964         if (!bp->pm_cap) {
2965                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2966                 return 0;
2967         }
2968
2969         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2970
2971         switch (state) {
2972         case PCI_D0:
2973                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2974                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2975                                        PCI_PM_CTRL_PME_STATUS));
2976
2977                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2978                         /* delay required during transition out of D3hot */
2979                         msleep(20);
2980                 break;
2981
2982         case PCI_D3hot:
2983                 /* If there are other clients above don't
2984                    shut down the power */
2985                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2986                         return 0;
2987                 /* Don't shut down the power for emulation and FPGA */
2988                 if (CHIP_REV_IS_SLOW(bp))
2989                         return 0;
2990
2991                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2992                 pmcsr |= 3;
2993
2994                 if (bp->wol)
2995                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2996
2997                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2998                                       pmcsr);
2999
3000                 /* No more memory access after this point until
3001                 * device is brought back to D0.
3002                 */
3003                 break;
3004
3005         default:
3006                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3007                 return -EINVAL;
3008         }
3009         return 0;
3010 }
3011
3012 /*
3013  * net_device service functions
3014  */
3015 int bnx2x_poll(struct napi_struct *napi, int budget)
3016 {
3017         int work_done = 0;
3018         u8 cos;
3019         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3020                                                  napi);
3021         struct bnx2x *bp = fp->bp;
3022
3023         while (1) {
3024 #ifdef BNX2X_STOP_ON_ERROR
3025                 if (unlikely(bp->panic)) {
3026                         napi_complete(napi);
3027                         return 0;
3028                 }
3029 #endif
3030
3031                 for_each_cos_in_tx_queue(fp, cos)
3032                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3033                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3034
3035                 if (bnx2x_has_rx_work(fp)) {
3036                         work_done += bnx2x_rx_int(fp, budget - work_done);
3037
3038                         /* must not complete if we consumed full budget */
3039                         if (work_done >= budget)
3040                                 break;
3041                 }
3042
3043                 /* Fall out from the NAPI loop if needed */
3044                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3045
3046                         /* No need to update SB for FCoE L2 ring as long as
3047                          * it's connected to the default SB and the SB
3048                          * has been updated when NAPI was scheduled.
3049                          */
3050                         if (IS_FCOE_FP(fp)) {
3051                                 napi_complete(napi);
3052                                 break;
3053                         }
3054                         bnx2x_update_fpsb_idx(fp);
3055                         /* bnx2x_has_rx_work() reads the status block,
3056                          * thus we need to ensure that status block indices
3057                          * have been actually read (bnx2x_update_fpsb_idx)
3058                          * prior to this check (bnx2x_has_rx_work) so that
3059                          * we won't write the "newer" value of the status block
3060                          * to IGU (if there was a DMA right after
3061                          * bnx2x_has_rx_work and if there is no rmb, the memory
3062                          * reading (bnx2x_update_fpsb_idx) may be postponed
3063                          * to right before bnx2x_ack_sb). In this case there
3064                          * will never be another interrupt until there is
3065                          * another update of the status block, while there
3066                          * is still unhandled work.
3067                          */
3068                         rmb();
3069
3070                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3071                                 napi_complete(napi);
3072                                 /* Re-enable interrupts */
3073                                 DP(NETIF_MSG_RX_STATUS,
3074                                    "Update index to %d\n", fp->fp_hc_idx);
3075                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3076                                              le16_to_cpu(fp->fp_hc_idx),
3077                                              IGU_INT_ENABLE, 1);
3078                                 break;
3079                         }
3080                 }
3081         }
3082
3083         return work_done;
3084 }
3085
3086 /* we split the first BD into headers and data BDs
3087  * to ease the pain of our fellow microcode engineers
3088  * we use one mapping for both BDs
3089  */
3090 static u16 bnx2x_tx_split(struct bnx2x *bp,
3091                           struct bnx2x_fp_txdata *txdata,
3092                           struct sw_tx_bd *tx_buf,
3093                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3094                           u16 bd_prod)
3095 {
3096         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3097         struct eth_tx_bd *d_tx_bd;
3098         dma_addr_t mapping;
3099         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3100
3101         /* first fix first BD */
3102         h_tx_bd->nbytes = cpu_to_le16(hlen);
3103
3104         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3105            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3106
3107         /* now get a new data BD
3108          * (after the pbd) and fill it */
3109         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3110         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3111
3112         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3113                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3114
3115         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3116         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3117         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3118
3119         /* this marks the BD as one that has no individual mapping */
3120         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3121
3122         DP(NETIF_MSG_TX_QUEUED,
3123            "TSO split data size is %d (%x:%x)\n",
3124            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3125
3126         /* update tx_bd */
3127         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3128
3129         return bd_prod;
3130 }
3131
3132 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3133 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3134 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3135 {
3136         __sum16 tsum = (__force __sum16) csum;
3137
3138         if (fix > 0)
3139                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3140                                   csum_partial(t_header - fix, fix, 0)));
3141
3142         else if (fix < 0)
3143                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3144                                   csum_partial(t_header, -fix, 0)));
3145
3146         return bswab16(tsum);
3147 }
3148
3149 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3150 {
3151         u32 rc;
3152         __u8 prot = 0;
3153         __be16 protocol;
3154
3155         if (skb->ip_summed != CHECKSUM_PARTIAL)
3156                 return XMIT_PLAIN;
3157
3158         protocol = vlan_get_protocol(skb);
3159         if (protocol == htons(ETH_P_IPV6)) {
3160                 rc = XMIT_CSUM_V6;
3161                 prot = ipv6_hdr(skb)->nexthdr;
3162         } else {
3163                 rc = XMIT_CSUM_V4;
3164                 prot = ip_hdr(skb)->protocol;
3165         }
3166
3167         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3168                 if (inner_ip_hdr(skb)->version == 6) {
3169                         rc |= XMIT_CSUM_ENC_V6;
3170                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3171                                 rc |= XMIT_CSUM_TCP;
3172                 } else {
3173                         rc |= XMIT_CSUM_ENC_V4;
3174                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3175                                 rc |= XMIT_CSUM_TCP;
3176                 }
3177         }
3178         if (prot == IPPROTO_TCP)
3179                 rc |= XMIT_CSUM_TCP;
3180
3181         if (skb_is_gso_v6(skb)) {
3182                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
3183                 if (rc & XMIT_CSUM_ENC)
3184                         rc |= XMIT_GSO_ENC_V6;
3185         } else if (skb_is_gso(skb)) {
3186                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
3187                 if (rc & XMIT_CSUM_ENC)
3188                         rc |= XMIT_GSO_ENC_V4;
3189         }
3190
3191         return rc;
3192 }
3193
3194 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3195 /* check if packet requires linearization (packet is too fragmented)
3196    no need to check fragmentation if page size > 8K (there will be no
3197    violation to FW restrictions) */
3198 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3199                              u32 xmit_type)
3200 {
3201         int to_copy = 0;
3202         int hlen = 0;
3203         int first_bd_sz = 0;
3204
3205         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3206         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3207
3208                 if (xmit_type & XMIT_GSO) {
3209                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3210                         /* Check if LSO packet needs to be copied:
3211                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3212                         int wnd_size = MAX_FETCH_BD - 3;
3213                         /* Number of windows to check */
3214                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3215                         int wnd_idx = 0;
3216                         int frag_idx = 0;
3217                         u32 wnd_sum = 0;
3218
3219                         /* Headers length */
3220                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3221                                 tcp_hdrlen(skb);
3222
3223                         /* Amount of data (w/o headers) on linear part of SKB*/
3224                         first_bd_sz = skb_headlen(skb) - hlen;
3225
3226                         wnd_sum  = first_bd_sz;
3227
3228                         /* Calculate the first sum - it's special */
3229                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3230                                 wnd_sum +=
3231                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3232
3233                         /* If there was data on linear skb data - check it */
3234                         if (first_bd_sz > 0) {
3235                                 if (unlikely(wnd_sum < lso_mss)) {
3236                                         to_copy = 1;
3237                                         goto exit_lbl;
3238                                 }
3239
3240                                 wnd_sum -= first_bd_sz;
3241                         }
3242
3243                         /* Others are easier: run through the frag list and
3244                            check all windows */
3245                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3246                                 wnd_sum +=
3247                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3248
3249                                 if (unlikely(wnd_sum < lso_mss)) {
3250                                         to_copy = 1;
3251                                         break;
3252                                 }
3253                                 wnd_sum -=
3254                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3255                         }
3256                 } else {
3257                         /* in non-LSO too fragmented packet should always
3258                            be linearized */
3259                         to_copy = 1;
3260                 }
3261         }
3262
3263 exit_lbl:
3264         if (unlikely(to_copy))
3265                 DP(NETIF_MSG_TX_QUEUED,
3266                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3267                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3268                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3269
3270         return to_copy;
3271 }
3272 #endif
3273
3274 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3275                                  u32 xmit_type)
3276 {
3277         struct ipv6hdr *ipv6;
3278
3279         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3280                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3281                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3282
3283         if (xmit_type & XMIT_GSO_ENC_V6)
3284                 ipv6 = inner_ipv6_hdr(skb);
3285         else if (xmit_type & XMIT_GSO_V6)
3286                 ipv6 = ipv6_hdr(skb);
3287         else
3288                 ipv6 = NULL;
3289
3290         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3291                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3292 }
3293
3294 /**
3295  * bnx2x_set_pbd_gso - update PBD in GSO case.
3296  *
3297  * @skb:        packet skb
3298  * @pbd:        parse BD
3299  * @xmit_type:  xmit flags
3300  */
3301 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3302                               struct eth_tx_parse_bd_e1x *pbd,
3303                               u32 xmit_type)
3304 {
3305         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3306         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3307         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3308
3309         if (xmit_type & XMIT_GSO_V4) {
3310                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3311                 pbd->tcp_pseudo_csum =
3312                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3313                                                    ip_hdr(skb)->daddr,
3314                                                    0, IPPROTO_TCP, 0));
3315
3316         } else
3317                 pbd->tcp_pseudo_csum =
3318                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3319                                                  &ipv6_hdr(skb)->daddr,
3320                                                  0, IPPROTO_TCP, 0));
3321
3322         pbd->global_data |=
3323                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3324 }
3325
3326 /**
3327  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3328  *
3329  * @bp:                 driver handle
3330  * @skb:                packet skb
3331  * @parsing_data:       data to be updated
3332  * @xmit_type:          xmit flags
3333  *
3334  * 57712/578xx related, when skb has encapsulation
3335  */
3336 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3337                                  u32 *parsing_data, u32 xmit_type)
3338 {
3339         *parsing_data |=
3340                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3341                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3342                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3343
3344         if (xmit_type & XMIT_CSUM_TCP) {
3345                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3346                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3347                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3348
3349                 return skb_inner_transport_header(skb) +
3350                         inner_tcp_hdrlen(skb) - skb->data;
3351         }
3352
3353         /* We support checksum offload for TCP and UDP only.
3354          * No need to pass the UDP header length - it's a constant.
3355          */
3356         return skb_inner_transport_header(skb) +
3357                 sizeof(struct udphdr) - skb->data;
3358 }
3359
3360 /**
3361  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3362  *
3363  * @bp:                 driver handle
3364  * @skb:                packet skb
3365  * @parsing_data:       data to be updated
3366  * @xmit_type:          xmit flags
3367  *
3368  * 57712/578xx related
3369  */
3370 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3371                                 u32 *parsing_data, u32 xmit_type)
3372 {
3373         *parsing_data |=
3374                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3375                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3376                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3377
3378         if (xmit_type & XMIT_CSUM_TCP) {
3379                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3380                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3381                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3382
3383                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3384         }
3385         /* We support checksum offload for TCP and UDP only.
3386          * No need to pass the UDP header length - it's a constant.
3387          */
3388         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3389 }
3390
3391 /* set FW indication according to inner or outer protocols if tunneled */
3392 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3393                                struct eth_tx_start_bd *tx_start_bd,
3394                                u32 xmit_type)
3395 {
3396         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3397
3398         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3399                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3400
3401         if (!(xmit_type & XMIT_CSUM_TCP))
3402                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3403 }
3404
3405 /**
3406  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3407  *
3408  * @bp:         driver handle
3409  * @skb:        packet skb
3410  * @pbd:        parse BD to be updated
3411  * @xmit_type:  xmit flags
3412  */
3413 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3414                              struct eth_tx_parse_bd_e1x *pbd,
3415                              u32 xmit_type)
3416 {
3417         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3418
3419         /* for now NS flag is not used in Linux */
3420         pbd->global_data =
3421                 cpu_to_le16(hlen |
3422                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3423                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3424
3425         pbd->ip_hlen_w = (skb_transport_header(skb) -
3426                         skb_network_header(skb)) >> 1;
3427
3428         hlen += pbd->ip_hlen_w;
3429
3430         /* We support checksum offload for TCP and UDP only */
3431         if (xmit_type & XMIT_CSUM_TCP)
3432                 hlen += tcp_hdrlen(skb) / 2;
3433         else
3434                 hlen += sizeof(struct udphdr) / 2;
3435
3436         pbd->total_hlen_w = cpu_to_le16(hlen);
3437         hlen = hlen*2;
3438
3439         if (xmit_type & XMIT_CSUM_TCP) {
3440                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3441
3442         } else {
3443                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3444
3445                 DP(NETIF_MSG_TX_QUEUED,
3446                    "hlen %d  fix %d  csum before fix %x\n",
3447                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3448
3449                 /* HW bug: fixup the CSUM */
3450                 pbd->tcp_pseudo_csum =
3451                         bnx2x_csum_fix(skb_transport_header(skb),
3452                                        SKB_CS(skb), fix);
3453
3454                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3455                    pbd->tcp_pseudo_csum);
3456         }
3457
3458         return hlen;
3459 }
3460
3461 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3462                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3463                                       struct eth_tx_parse_2nd_bd *pbd2,
3464                                       u16 *global_data,
3465                                       u32 xmit_type)
3466 {
3467         u16 hlen_w = 0;
3468         u8 outerip_off, outerip_len = 0;
3469         /* from outer IP to transport */
3470         hlen_w = (skb_inner_transport_header(skb) -
3471                   skb_network_header(skb)) >> 1;
3472
3473         /* transport len */
3474         if (xmit_type & XMIT_CSUM_TCP)
3475                 hlen_w += inner_tcp_hdrlen(skb) >> 1;
3476         else
3477                 hlen_w += sizeof(struct udphdr) >> 1;
3478
3479         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3480
3481         if (xmit_type & XMIT_CSUM_ENC_V4) {
3482                 struct iphdr *iph = ip_hdr(skb);
3483                 pbd2->fw_ip_csum_wo_len_flags_frag =
3484                         bswab16(csum_fold((~iph->check) -
3485                                           iph->tot_len - iph->frag_off));
3486         } else {
3487                 pbd2->fw_ip_hdr_to_payload_w =
3488                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3489         }
3490
3491         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3492
3493         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3494
3495         if (xmit_type & XMIT_GSO_V4) {
3496                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3497
3498                 pbd_e2->data.tunnel_data.pseudo_csum =
3499                         bswab16(~csum_tcpudp_magic(
3500                                         inner_ip_hdr(skb)->saddr,
3501                                         inner_ip_hdr(skb)->daddr,
3502                                         0, IPPROTO_TCP, 0));
3503
3504                 outerip_len = ip_hdr(skb)->ihl << 1;
3505         } else {
3506                 pbd_e2->data.tunnel_data.pseudo_csum =
3507                         bswab16(~csum_ipv6_magic(
3508                                         &inner_ipv6_hdr(skb)->saddr,
3509                                         &inner_ipv6_hdr(skb)->daddr,
3510                                         0, IPPROTO_TCP, 0));
3511         }
3512
3513         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3514
3515         *global_data |=
3516                 outerip_off |
3517                 (!!(xmit_type & XMIT_CSUM_V6) <<
3518                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3519                 (outerip_len <<
3520                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3521                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3522                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3523 }
3524
3525 /* called with netif_tx_lock
3526  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3527  * netif_wake_queue()
3528  */
3529 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3530 {
3531         struct bnx2x *bp = netdev_priv(dev);
3532
3533         struct netdev_queue *txq;
3534         struct bnx2x_fp_txdata *txdata;
3535         struct sw_tx_bd *tx_buf;
3536         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3537         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3538         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3539         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3540         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3541         u32 pbd_e2_parsing_data = 0;
3542         u16 pkt_prod, bd_prod;
3543         int nbd, txq_index;
3544         dma_addr_t mapping;
3545         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3546         int i;
3547         u8 hlen = 0;
3548         __le16 pkt_size = 0;
3549         struct ethhdr *eth;
3550         u8 mac_type = UNICAST_ADDRESS;
3551
3552 #ifdef BNX2X_STOP_ON_ERROR
3553         if (unlikely(bp->panic))
3554                 return NETDEV_TX_BUSY;
3555 #endif
3556
3557         txq_index = skb_get_queue_mapping(skb);
3558         txq = netdev_get_tx_queue(dev, txq_index);
3559
3560         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3561
3562         txdata = &bp->bnx2x_txq[txq_index];
3563
3564         /* enable this debug print to view the transmission queue being used
3565         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3566            txq_index, fp_index, txdata_index); */
3567
3568         /* enable this debug print to view the tranmission details
3569         DP(NETIF_MSG_TX_QUEUED,
3570            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3571            txdata->cid, fp_index, txdata_index, txdata, fp); */
3572
3573         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3574                         skb_shinfo(skb)->nr_frags +
3575                         BDS_PER_TX_PKT +
3576                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3577                 /* Handle special storage cases separately */
3578                 if (txdata->tx_ring_size == 0) {
3579                         struct bnx2x_eth_q_stats *q_stats =
3580                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3581                         q_stats->driver_filtered_tx_pkt++;
3582                         dev_kfree_skb(skb);
3583                         return NETDEV_TX_OK;
3584                 }
3585                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3586                 netif_tx_stop_queue(txq);
3587                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3588
3589                 return NETDEV_TX_BUSY;
3590         }
3591
3592         DP(NETIF_MSG_TX_QUEUED,
3593            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3594            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3595            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3596            skb->len);
3597
3598         eth = (struct ethhdr *)skb->data;
3599
3600         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3601         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3602                 if (is_broadcast_ether_addr(eth->h_dest))
3603                         mac_type = BROADCAST_ADDRESS;
3604                 else
3605                         mac_type = MULTICAST_ADDRESS;
3606         }
3607
3608 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3609         /* First, check if we need to linearize the skb (due to FW
3610            restrictions). No need to check fragmentation if page size > 8K
3611            (there will be no violation to FW restrictions) */
3612         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3613                 /* Statistics of linearization */
3614                 bp->lin_cnt++;
3615                 if (skb_linearize(skb) != 0) {
3616                         DP(NETIF_MSG_TX_QUEUED,
3617                            "SKB linearization failed - silently dropping this SKB\n");
3618                         dev_kfree_skb_any(skb);
3619                         return NETDEV_TX_OK;
3620                 }
3621         }
3622 #endif
3623         /* Map skb linear data for DMA */
3624         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3625                                  skb_headlen(skb), DMA_TO_DEVICE);
3626         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3627                 DP(NETIF_MSG_TX_QUEUED,
3628                    "SKB mapping failed - silently dropping this SKB\n");
3629                 dev_kfree_skb_any(skb);
3630                 return NETDEV_TX_OK;
3631         }
3632         /*
3633         Please read carefully. First we use one BD which we mark as start,
3634         then we have a parsing info BD (used for TSO or xsum),
3635         and only then we have the rest of the TSO BDs.
3636         (don't forget to mark the last one as last,
3637         and to unmap only AFTER you write to the BD ...)
3638         And above all, all pdb sizes are in words - NOT DWORDS!
3639         */
3640
3641         /* get current pkt produced now - advance it just before sending packet
3642          * since mapping of pages may fail and cause packet to be dropped
3643          */
3644         pkt_prod = txdata->tx_pkt_prod;
3645         bd_prod = TX_BD(txdata->tx_bd_prod);
3646
3647         /* get a tx_buf and first BD
3648          * tx_start_bd may be changed during SPLIT,
3649          * but first_bd will always stay first
3650          */
3651         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3652         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3653         first_bd = tx_start_bd;
3654
3655         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3656
3657         /* header nbd: indirectly zero other flags! */
3658         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3659
3660         /* remember the first BD of the packet */
3661         tx_buf->first_bd = txdata->tx_bd_prod;
3662         tx_buf->skb = skb;
3663         tx_buf->flags = 0;
3664
3665         DP(NETIF_MSG_TX_QUEUED,
3666            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3667            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3668
3669         if (vlan_tx_tag_present(skb)) {
3670                 tx_start_bd->vlan_or_ethertype =
3671                     cpu_to_le16(vlan_tx_tag_get(skb));
3672                 tx_start_bd->bd_flags.as_bitfield |=
3673                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3674         } else {
3675                 /* when transmitting in a vf, start bd must hold the ethertype
3676                  * for fw to enforce it
3677                  */
3678                 if (IS_VF(bp))
3679                         tx_start_bd->vlan_or_ethertype =
3680                                 cpu_to_le16(ntohs(eth->h_proto));
3681                 else
3682                         /* used by FW for packet accounting */
3683                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3684         }
3685
3686         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3687
3688         /* turn on parsing and get a BD */
3689         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3690
3691         if (xmit_type & XMIT_CSUM)
3692                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3693
3694         if (!CHIP_IS_E1x(bp)) {
3695                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3696                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3697
3698                 if (xmit_type & XMIT_CSUM_ENC) {
3699                         u16 global_data = 0;
3700
3701                         /* Set PBD in enc checksum offload case */
3702                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3703                                                       &pbd_e2_parsing_data,
3704                                                       xmit_type);
3705
3706                         /* turn on 2nd parsing and get a BD */
3707                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3708
3709                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3710
3711                         memset(pbd2, 0, sizeof(*pbd2));
3712
3713                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3714                                 (skb_inner_network_header(skb) -
3715                                  skb->data) >> 1;
3716
3717                         if (xmit_type & XMIT_GSO_ENC)
3718                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3719                                                           &global_data,
3720                                                           xmit_type);
3721
3722                         pbd2->global_data = cpu_to_le16(global_data);
3723
3724                         /* add addition parse BD indication to start BD */
3725                         SET_FLAG(tx_start_bd->general_data,
3726                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3727                         /* set encapsulation flag in start BD */
3728                         SET_FLAG(tx_start_bd->general_data,
3729                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3730                         nbd++;
3731                 } else if (xmit_type & XMIT_CSUM) {
3732                         /* Set PBD in checksum offload case w/o encapsulation */
3733                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3734                                                      &pbd_e2_parsing_data,
3735                                                      xmit_type);
3736                 }
3737
3738                 /* Add the macs to the parsing BD this is a vf */
3739                 if (IS_VF(bp)) {
3740                         /* override GRE parameters in BD */
3741                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3742                                               &pbd_e2->data.mac_addr.src_mid,
3743                                               &pbd_e2->data.mac_addr.src_lo,
3744                                               eth->h_source);
3745
3746                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3747                                               &pbd_e2->data.mac_addr.dst_mid,
3748                                               &pbd_e2->data.mac_addr.dst_lo,
3749                                               eth->h_dest);
3750                 }
3751
3752                 SET_FLAG(pbd_e2_parsing_data,
3753                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3754         } else {
3755                 u16 global_data = 0;
3756                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3757                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3758                 /* Set PBD in checksum offload case */
3759                 if (xmit_type & XMIT_CSUM)
3760                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3761
3762                 SET_FLAG(global_data,
3763                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3764                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3765         }
3766
3767         /* Setup the data pointer of the first BD of the packet */
3768         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3769         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3770         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3771         pkt_size = tx_start_bd->nbytes;
3772
3773         DP(NETIF_MSG_TX_QUEUED,
3774            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3775            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3776            le16_to_cpu(tx_start_bd->nbytes),
3777            tx_start_bd->bd_flags.as_bitfield,
3778            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3779
3780         if (xmit_type & XMIT_GSO) {
3781
3782                 DP(NETIF_MSG_TX_QUEUED,
3783                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3784                    skb->len, hlen, skb_headlen(skb),
3785                    skb_shinfo(skb)->gso_size);
3786
3787                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3788
3789                 if (unlikely(skb_headlen(skb) > hlen)) {
3790                         nbd++;
3791                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3792                                                  &tx_start_bd, hlen,
3793                                                  bd_prod);
3794                 }
3795                 if (!CHIP_IS_E1x(bp))
3796                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3797                                              xmit_type);
3798                 else
3799                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3800         }
3801
3802         /* Set the PBD's parsing_data field if not zero
3803          * (for the chips newer than 57711).
3804          */
3805         if (pbd_e2_parsing_data)
3806                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3807
3808         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3809
3810         /* Handle fragmented skb */
3811         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3812                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3813
3814                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3815                                            skb_frag_size(frag), DMA_TO_DEVICE);
3816                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3817                         unsigned int pkts_compl = 0, bytes_compl = 0;
3818
3819                         DP(NETIF_MSG_TX_QUEUED,
3820                            "Unable to map page - dropping packet...\n");
3821
3822                         /* we need unmap all buffers already mapped
3823                          * for this SKB;
3824                          * first_bd->nbd need to be properly updated
3825                          * before call to bnx2x_free_tx_pkt
3826                          */
3827                         first_bd->nbd = cpu_to_le16(nbd);
3828                         bnx2x_free_tx_pkt(bp, txdata,
3829                                           TX_BD(txdata->tx_pkt_prod),
3830                                           &pkts_compl, &bytes_compl);
3831                         return NETDEV_TX_OK;
3832                 }
3833
3834                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3835                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3836                 if (total_pkt_bd == NULL)
3837                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3838
3839                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3840                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3841                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3842                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3843                 nbd++;
3844
3845                 DP(NETIF_MSG_TX_QUEUED,
3846                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3847                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3848                    le16_to_cpu(tx_data_bd->nbytes));
3849         }
3850
3851         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3852
3853         /* update with actual num BDs */
3854         first_bd->nbd = cpu_to_le16(nbd);
3855
3856         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3857
3858         /* now send a tx doorbell, counting the next BD
3859          * if the packet contains or ends with it
3860          */
3861         if (TX_BD_POFF(bd_prod) < nbd)
3862                 nbd++;
3863
3864         /* total_pkt_bytes should be set on the first data BD if
3865          * it's not an LSO packet and there is more than one
3866          * data BD. In this case pkt_size is limited by an MTU value.
3867          * However we prefer to set it for an LSO packet (while we don't
3868          * have to) in order to save some CPU cycles in a none-LSO
3869          * case, when we much more care about them.
3870          */
3871         if (total_pkt_bd != NULL)
3872                 total_pkt_bd->total_pkt_bytes = pkt_size;
3873
3874         if (pbd_e1x)
3875                 DP(NETIF_MSG_TX_QUEUED,
3876                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3877                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3878                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3879                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3880                     le16_to_cpu(pbd_e1x->total_hlen_w));
3881         if (pbd_e2)
3882                 DP(NETIF_MSG_TX_QUEUED,
3883                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3884                    pbd_e2,
3885                    pbd_e2->data.mac_addr.dst_hi,
3886                    pbd_e2->data.mac_addr.dst_mid,
3887                    pbd_e2->data.mac_addr.dst_lo,
3888                    pbd_e2->data.mac_addr.src_hi,
3889                    pbd_e2->data.mac_addr.src_mid,
3890                    pbd_e2->data.mac_addr.src_lo,
3891                    pbd_e2->parsing_data);
3892         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3893
3894         netdev_tx_sent_queue(txq, skb->len);
3895
3896         skb_tx_timestamp(skb);
3897
3898         txdata->tx_pkt_prod++;
3899         /*
3900          * Make sure that the BD data is updated before updating the producer
3901          * since FW might read the BD right after the producer is updated.
3902          * This is only applicable for weak-ordered memory model archs such
3903          * as IA-64. The following barrier is also mandatory since FW will
3904          * assumes packets must have BDs.
3905          */
3906         wmb();
3907
3908         txdata->tx_db.data.prod += nbd;
3909         barrier();
3910
3911         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3912
3913         mmiowb();
3914
3915         txdata->tx_bd_prod += nbd;
3916
3917         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3918                 netif_tx_stop_queue(txq);
3919
3920                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3921                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3922                  * fp->bd_tx_cons */
3923                 smp_mb();
3924
3925                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3926                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3927                         netif_tx_wake_queue(txq);
3928         }
3929         txdata->tx_pkt++;
3930
3931         return NETDEV_TX_OK;
3932 }
3933
3934 /**
3935  * bnx2x_setup_tc - routine to configure net_device for multi tc
3936  *
3937  * @netdev: net device to configure
3938  * @tc: number of traffic classes to enable
3939  *
3940  * callback connected to the ndo_setup_tc function pointer
3941  */
3942 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3943 {
3944         int cos, prio, count, offset;
3945         struct bnx2x *bp = netdev_priv(dev);
3946
3947         /* setup tc must be called under rtnl lock */
3948         ASSERT_RTNL();
3949
3950         /* no traffic classes requested. aborting */
3951         if (!num_tc) {
3952                 netdev_reset_tc(dev);
3953                 return 0;
3954         }
3955
3956         /* requested to support too many traffic classes */
3957         if (num_tc > bp->max_cos) {
3958                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3959                           num_tc, bp->max_cos);
3960                 return -EINVAL;
3961         }
3962
3963         /* declare amount of supported traffic classes */
3964         if (netdev_set_num_tc(dev, num_tc)) {
3965                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3966                 return -EINVAL;
3967         }
3968
3969         /* configure priority to traffic class mapping */
3970         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3971                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3972                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3973                    "mapping priority %d to tc %d\n",
3974                    prio, bp->prio_to_cos[prio]);
3975         }
3976
3977
3978         /* Use this configuration to diffrentiate tc0 from other COSes
3979            This can be used for ets or pfc, and save the effort of setting
3980            up a multio class queue disc or negotiating DCBX with a switch
3981         netdev_set_prio_tc_map(dev, 0, 0);
3982         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3983         for (prio = 1; prio < 16; prio++) {
3984                 netdev_set_prio_tc_map(dev, prio, 1);
3985                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3986         } */
3987
3988         /* configure traffic class to transmission queue mapping */
3989         for (cos = 0; cos < bp->max_cos; cos++) {
3990                 count = BNX2X_NUM_ETH_QUEUES(bp);
3991                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
3992                 netdev_set_tc_queue(dev, cos, count, offset);
3993                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3994                    "mapping tc %d to offset %d count %d\n",
3995                    cos, offset, count);
3996         }
3997
3998         return 0;
3999 }
4000
4001 /* called with rtnl_lock */
4002 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4003 {
4004         struct sockaddr *addr = p;
4005         struct bnx2x *bp = netdev_priv(dev);
4006         int rc = 0;
4007
4008         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4009                 BNX2X_ERR("Requested MAC address is not valid\n");
4010                 return -EINVAL;
4011         }
4012
4013         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4014             !is_zero_ether_addr(addr->sa_data)) {
4015                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4016                 return -EINVAL;
4017         }
4018
4019         if (netif_running(dev))  {
4020                 rc = bnx2x_set_eth_mac(bp, false);
4021                 if (rc)
4022                         return rc;
4023         }
4024
4025         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4026
4027         if (netif_running(dev))
4028                 rc = bnx2x_set_eth_mac(bp, true);
4029
4030         return rc;
4031 }
4032
4033 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4034 {
4035         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4036         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4037         u8 cos;
4038
4039         /* Common */
4040
4041         if (IS_FCOE_IDX(fp_index)) {
4042                 memset(sb, 0, sizeof(union host_hc_status_block));
4043                 fp->status_blk_mapping = 0;
4044         } else {
4045                 /* status blocks */
4046                 if (!CHIP_IS_E1x(bp))
4047                         BNX2X_PCI_FREE(sb->e2_sb,
4048                                        bnx2x_fp(bp, fp_index,
4049                                                 status_blk_mapping),
4050                                        sizeof(struct host_hc_status_block_e2));
4051                 else
4052                         BNX2X_PCI_FREE(sb->e1x_sb,
4053                                        bnx2x_fp(bp, fp_index,
4054                                                 status_blk_mapping),
4055                                        sizeof(struct host_hc_status_block_e1x));
4056         }
4057
4058         /* Rx */
4059         if (!skip_rx_queue(bp, fp_index)) {
4060                 bnx2x_free_rx_bds(fp);
4061
4062                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4063                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4064                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4065                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4066                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4067
4068                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4069                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4070                                sizeof(struct eth_fast_path_rx_cqe) *
4071                                NUM_RCQ_BD);
4072
4073                 /* SGE ring */
4074                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4075                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4076                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4077                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4078         }
4079
4080         /* Tx */
4081         if (!skip_tx_queue(bp, fp_index)) {
4082                 /* fastpath tx rings: tx_buf tx_desc */
4083                 for_each_cos_in_tx_queue(fp, cos) {
4084                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4085
4086                         DP(NETIF_MSG_IFDOWN,
4087                            "freeing tx memory of fp %d cos %d cid %d\n",
4088                            fp_index, cos, txdata->cid);
4089
4090                         BNX2X_FREE(txdata->tx_buf_ring);
4091                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4092                                 txdata->tx_desc_mapping,
4093                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4094                 }
4095         }
4096         /* end of fastpath */
4097 }
4098
4099 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4100 {
4101         int i;
4102         for_each_cnic_queue(bp, i)
4103                 bnx2x_free_fp_mem_at(bp, i);
4104 }
4105
4106 void bnx2x_free_fp_mem(struct bnx2x *bp)
4107 {
4108         int i;
4109         for_each_eth_queue(bp, i)
4110                 bnx2x_free_fp_mem_at(bp, i);
4111 }
4112
4113 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4114 {
4115         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4116         if (!CHIP_IS_E1x(bp)) {
4117                 bnx2x_fp(bp, index, sb_index_values) =
4118                         (__le16 *)status_blk.e2_sb->sb.index_values;
4119                 bnx2x_fp(bp, index, sb_running_index) =
4120                         (__le16 *)status_blk.e2_sb->sb.running_index;
4121         } else {
4122                 bnx2x_fp(bp, index, sb_index_values) =
4123                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4124                 bnx2x_fp(bp, index, sb_running_index) =
4125                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4126         }
4127 }
4128
4129 /* Returns the number of actually allocated BDs */
4130 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4131                               int rx_ring_size)
4132 {
4133         struct bnx2x *bp = fp->bp;
4134         u16 ring_prod, cqe_ring_prod;
4135         int i, failure_cnt = 0;
4136
4137         fp->rx_comp_cons = 0;
4138         cqe_ring_prod = ring_prod = 0;
4139
4140         /* This routine is called only during fo init so
4141          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4142          */
4143         for (i = 0; i < rx_ring_size; i++) {
4144                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
4145                         failure_cnt++;
4146                         continue;
4147                 }
4148                 ring_prod = NEXT_RX_IDX(ring_prod);
4149                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4150                 WARN_ON(ring_prod <= (i - failure_cnt));
4151         }
4152
4153         if (failure_cnt)
4154                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4155                           i - failure_cnt, fp->index);
4156
4157         fp->rx_bd_prod = ring_prod;
4158         /* Limit the CQE producer by the CQE ring size */
4159         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4160                                cqe_ring_prod);
4161         fp->rx_pkt = fp->rx_calls = 0;
4162
4163         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4164
4165         return i - failure_cnt;
4166 }
4167
4168 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4169 {
4170         int i;
4171
4172         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4173                 struct eth_rx_cqe_next_page *nextpg;
4174
4175                 nextpg = (struct eth_rx_cqe_next_page *)
4176                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4177                 nextpg->addr_hi =
4178                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4179                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4180                 nextpg->addr_lo =
4181                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4182                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4183         }
4184 }
4185
4186 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4187 {
4188         union host_hc_status_block *sb;
4189         struct bnx2x_fastpath *fp = &bp->fp[index];
4190         int ring_size = 0;
4191         u8 cos;
4192         int rx_ring_size = 0;
4193
4194         if (!bp->rx_ring_size &&
4195             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4196                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4197                 bp->rx_ring_size = rx_ring_size;
4198         } else if (!bp->rx_ring_size) {
4199                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4200
4201                 if (CHIP_IS_E3(bp)) {
4202                         u32 cfg = SHMEM_RD(bp,
4203                                            dev_info.port_hw_config[BP_PORT(bp)].
4204                                            default_cfg);
4205
4206                         /* Decrease ring size for 1G functions */
4207                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4208                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4209                                 rx_ring_size /= 10;
4210                 }
4211
4212                 /* allocate at least number of buffers required by FW */
4213                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4214                                      MIN_RX_SIZE_TPA, rx_ring_size);
4215
4216                 bp->rx_ring_size = rx_ring_size;
4217         } else /* if rx_ring_size specified - use it */
4218                 rx_ring_size = bp->rx_ring_size;
4219
4220         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4221
4222         /* Common */
4223         sb = &bnx2x_fp(bp, index, status_blk);
4224
4225         if (!IS_FCOE_IDX(index)) {
4226                 /* status blocks */
4227                 if (!CHIP_IS_E1x(bp))
4228                         BNX2X_PCI_ALLOC(sb->e2_sb,
4229                                 &bnx2x_fp(bp, index, status_blk_mapping),
4230                                 sizeof(struct host_hc_status_block_e2));
4231                 else
4232                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4233                                 &bnx2x_fp(bp, index, status_blk_mapping),
4234                             sizeof(struct host_hc_status_block_e1x));
4235         }
4236
4237         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4238          * set shortcuts for it.
4239          */
4240         if (!IS_FCOE_IDX(index))
4241                 set_sb_shortcuts(bp, index);
4242
4243         /* Tx */
4244         if (!skip_tx_queue(bp, index)) {
4245                 /* fastpath tx rings: tx_buf tx_desc */
4246                 for_each_cos_in_tx_queue(fp, cos) {
4247                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4248
4249                         DP(NETIF_MSG_IFUP,
4250                            "allocating tx memory of fp %d cos %d\n",
4251                            index, cos);
4252
4253                         BNX2X_ALLOC(txdata->tx_buf_ring,
4254                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4255                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4256                                 &txdata->tx_desc_mapping,
4257                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4258                 }
4259         }
4260
4261         /* Rx */
4262         if (!skip_rx_queue(bp, index)) {
4263                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4264                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4265                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4266                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4267                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4268                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4269
4270                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4271                                 &bnx2x_fp(bp, index, rx_comp_mapping),
4272                                 sizeof(struct eth_fast_path_rx_cqe) *
4273                                 NUM_RCQ_BD);
4274
4275                 /* SGE ring */
4276                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4277                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4278                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4279                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4280                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4281                 /* RX BD ring */
4282                 bnx2x_set_next_page_rx_bd(fp);
4283
4284                 /* CQ ring */
4285                 bnx2x_set_next_page_rx_cq(fp);
4286
4287                 /* BDs */
4288                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4289                 if (ring_size < rx_ring_size)
4290                         goto alloc_mem_err;
4291         }
4292
4293         return 0;
4294
4295 /* handles low memory cases */
4296 alloc_mem_err:
4297         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4298                                                 index, ring_size);
4299         /* FW will drop all packets if queue is not big enough,
4300          * In these cases we disable the queue
4301          * Min size is different for OOO, TPA and non-TPA queues
4302          */
4303         if (ring_size < (fp->disable_tpa ?
4304                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4305                         /* release memory allocated for this queue */
4306                         bnx2x_free_fp_mem_at(bp, index);
4307                         return -ENOMEM;
4308         }
4309         return 0;
4310 }
4311
4312 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4313 {
4314         if (!NO_FCOE(bp))
4315                 /* FCoE */
4316                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4317                         /* we will fail load process instead of mark
4318                          * NO_FCOE_FLAG
4319                          */
4320                         return -ENOMEM;
4321
4322         return 0;
4323 }
4324
4325 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4326 {
4327         int i;
4328
4329         /* 1. Allocate FP for leading - fatal if error
4330          * 2. Allocate RSS - fix number of queues if error
4331          */
4332
4333         /* leading */
4334         if (bnx2x_alloc_fp_mem_at(bp, 0))
4335                 return -ENOMEM;
4336
4337         /* RSS */
4338         for_each_nondefault_eth_queue(bp, i)
4339                 if (bnx2x_alloc_fp_mem_at(bp, i))
4340                         break;
4341
4342         /* handle memory failures */
4343         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4344                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4345
4346                 WARN_ON(delta < 0);
4347                 bnx2x_shrink_eth_fp(bp, delta);
4348                 if (CNIC_SUPPORT(bp))
4349                         /* move non eth FPs next to last eth FP
4350                          * must be done in that order
4351                          * FCOE_IDX < FWD_IDX < OOO_IDX
4352                          */
4353
4354                         /* move FCoE fp even NO_FCOE_FLAG is on */
4355                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4356                 bp->num_ethernet_queues -= delta;
4357                 bp->num_queues = bp->num_ethernet_queues +
4358                                  bp->num_cnic_queues;
4359                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4360                           bp->num_queues + delta, bp->num_queues);
4361         }
4362
4363         return 0;
4364 }
4365
4366 void bnx2x_free_mem_bp(struct bnx2x *bp)
4367 {
4368         int i;
4369
4370         for (i = 0; i < bp->fp_array_size; i++)
4371                 kfree(bp->fp[i].tpa_info);
4372         kfree(bp->fp);
4373         kfree(bp->sp_objs);
4374         kfree(bp->fp_stats);
4375         kfree(bp->bnx2x_txq);
4376         kfree(bp->msix_table);
4377         kfree(bp->ilt);
4378 }
4379
4380 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4381 {
4382         struct bnx2x_fastpath *fp;
4383         struct msix_entry *tbl;
4384         struct bnx2x_ilt *ilt;
4385         int msix_table_size = 0;
4386         int fp_array_size, txq_array_size;
4387         int i;
4388
4389         /*
4390          * The biggest MSI-X table we might need is as a maximum number of fast
4391          * path IGU SBs plus default SB (for PF only).
4392          */
4393         msix_table_size = bp->igu_sb_cnt;
4394         if (IS_PF(bp))
4395                 msix_table_size++;
4396         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4397
4398         /* fp array: RSS plus CNIC related L2 queues */
4399         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4400         bp->fp_array_size = fp_array_size;
4401         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4402
4403         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4404         if (!fp)
4405                 goto alloc_err;
4406         for (i = 0; i < bp->fp_array_size; i++) {
4407                 fp[i].tpa_info =
4408                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4409                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4410                 if (!(fp[i].tpa_info))
4411                         goto alloc_err;
4412         }
4413
4414         bp->fp = fp;
4415
4416         /* allocate sp objs */
4417         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4418                               GFP_KERNEL);
4419         if (!bp->sp_objs)
4420                 goto alloc_err;
4421
4422         /* allocate fp_stats */
4423         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4424                                GFP_KERNEL);
4425         if (!bp->fp_stats)
4426                 goto alloc_err;
4427
4428         /* Allocate memory for the transmission queues array */
4429         txq_array_size =
4430                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4431         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4432
4433         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4434                                 GFP_KERNEL);
4435         if (!bp->bnx2x_txq)
4436                 goto alloc_err;
4437
4438         /* msix table */
4439         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4440         if (!tbl)
4441                 goto alloc_err;
4442         bp->msix_table = tbl;
4443
4444         /* ilt */
4445         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4446         if (!ilt)
4447                 goto alloc_err;
4448         bp->ilt = ilt;
4449
4450         return 0;
4451 alloc_err:
4452         bnx2x_free_mem_bp(bp);
4453         return -ENOMEM;
4454
4455 }
4456
4457 int bnx2x_reload_if_running(struct net_device *dev)
4458 {
4459         struct bnx2x *bp = netdev_priv(dev);
4460
4461         if (unlikely(!netif_running(dev)))
4462                 return 0;
4463
4464         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4465         return bnx2x_nic_load(bp, LOAD_NORMAL);
4466 }
4467
4468 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4469 {
4470         u32 sel_phy_idx = 0;
4471         if (bp->link_params.num_phys <= 1)
4472                 return INT_PHY;
4473
4474         if (bp->link_vars.link_up) {
4475                 sel_phy_idx = EXT_PHY1;
4476                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4477                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4478                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4479                         sel_phy_idx = EXT_PHY2;
4480         } else {
4481
4482                 switch (bnx2x_phy_selection(&bp->link_params)) {
4483                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4484                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4485                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4486                        sel_phy_idx = EXT_PHY1;
4487                        break;
4488                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4489                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4490                        sel_phy_idx = EXT_PHY2;
4491                        break;
4492                 }
4493         }
4494
4495         return sel_phy_idx;
4496
4497 }
4498 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4499 {
4500         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4501         /*
4502          * The selected activated PHY is always after swapping (in case PHY
4503          * swapping is enabled). So when swapping is enabled, we need to reverse
4504          * the configuration
4505          */
4506
4507         if (bp->link_params.multi_phy_config &
4508             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4509                 if (sel_phy_idx == EXT_PHY1)
4510                         sel_phy_idx = EXT_PHY2;
4511                 else if (sel_phy_idx == EXT_PHY2)
4512                         sel_phy_idx = EXT_PHY1;
4513         }
4514         return LINK_CONFIG_IDX(sel_phy_idx);
4515 }
4516
4517 #ifdef NETDEV_FCOE_WWNN
4518 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4519 {
4520         struct bnx2x *bp = netdev_priv(dev);
4521         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4522
4523         switch (type) {
4524         case NETDEV_FCOE_WWNN:
4525                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4526                                 cp->fcoe_wwn_node_name_lo);
4527                 break;
4528         case NETDEV_FCOE_WWPN:
4529                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4530                                 cp->fcoe_wwn_port_name_lo);
4531                 break;
4532         default:
4533                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4534                 return -EINVAL;
4535         }
4536
4537         return 0;
4538 }
4539 #endif
4540
4541 /* called with rtnl_lock */
4542 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4543 {
4544         struct bnx2x *bp = netdev_priv(dev);
4545
4546         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4547                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4548                 return -EAGAIN;
4549         }
4550
4551         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4552             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4553                 BNX2X_ERR("Can't support requested MTU size\n");
4554                 return -EINVAL;
4555         }
4556
4557         /* This does not race with packet allocation
4558          * because the actual alloc size is
4559          * only updated as part of load
4560          */
4561         dev->mtu = new_mtu;
4562
4563         return bnx2x_reload_if_running(dev);
4564 }
4565
4566 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4567                                      netdev_features_t features)
4568 {
4569         struct bnx2x *bp = netdev_priv(dev);
4570
4571         /* TPA requires Rx CSUM offloading */
4572         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4573                 features &= ~NETIF_F_LRO;
4574                 features &= ~NETIF_F_GRO;
4575         }
4576
4577         return features;
4578 }
4579
4580 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4581 {
4582         struct bnx2x *bp = netdev_priv(dev);
4583         u32 flags = bp->flags;
4584         bool bnx2x_reload = false;
4585
4586         if (features & NETIF_F_LRO)
4587                 flags |= TPA_ENABLE_FLAG;
4588         else
4589                 flags &= ~TPA_ENABLE_FLAG;
4590
4591         if (features & NETIF_F_GRO)
4592                 flags |= GRO_ENABLE_FLAG;
4593         else
4594                 flags &= ~GRO_ENABLE_FLAG;
4595
4596         if (features & NETIF_F_LOOPBACK) {
4597                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4598                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4599                         bnx2x_reload = true;
4600                 }
4601         } else {
4602                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4603                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4604                         bnx2x_reload = true;
4605                 }
4606         }
4607
4608         if (flags ^ bp->flags) {
4609                 bp->flags = flags;
4610                 bnx2x_reload = true;
4611         }
4612
4613         if (bnx2x_reload) {
4614                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4615                         return bnx2x_reload_if_running(dev);
4616                 /* else: bnx2x_nic_load() will be called at end of recovery */
4617         }
4618
4619         return 0;
4620 }
4621
4622 void bnx2x_tx_timeout(struct net_device *dev)
4623 {
4624         struct bnx2x *bp = netdev_priv(dev);
4625
4626 #ifdef BNX2X_STOP_ON_ERROR
4627         if (!bp->panic)
4628                 bnx2x_panic();
4629 #endif
4630
4631         smp_mb__before_clear_bit();
4632         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4633         smp_mb__after_clear_bit();
4634
4635         /* This allows the netif to be shutdown gracefully before resetting */
4636         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4637 }
4638
4639 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4640 {
4641         struct net_device *dev = pci_get_drvdata(pdev);
4642         struct bnx2x *bp;
4643
4644         if (!dev) {
4645                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4646                 return -ENODEV;
4647         }
4648         bp = netdev_priv(dev);
4649
4650         rtnl_lock();
4651
4652         pci_save_state(pdev);
4653
4654         if (!netif_running(dev)) {
4655                 rtnl_unlock();
4656                 return 0;
4657         }
4658
4659         netif_device_detach(dev);
4660
4661         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4662
4663         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4664
4665         rtnl_unlock();
4666
4667         return 0;
4668 }
4669
4670 int bnx2x_resume(struct pci_dev *pdev)
4671 {
4672         struct net_device *dev = pci_get_drvdata(pdev);
4673         struct bnx2x *bp;
4674         int rc;
4675
4676         if (!dev) {
4677                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4678                 return -ENODEV;
4679         }
4680         bp = netdev_priv(dev);
4681
4682         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4683                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4684                 return -EAGAIN;
4685         }
4686
4687         rtnl_lock();
4688
4689         pci_restore_state(pdev);
4690
4691         if (!netif_running(dev)) {
4692                 rtnl_unlock();
4693                 return 0;
4694         }
4695
4696         bnx2x_set_power_state(bp, PCI_D0);
4697         netif_device_attach(dev);
4698
4699         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4700
4701         rtnl_unlock();
4702
4703         return rc;
4704 }
4705
4706
4707 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4708                               u32 cid)
4709 {
4710         /* ustorm cxt validation */
4711         cxt->ustorm_ag_context.cdu_usage =
4712                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4713                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4714         /* xcontext validation */
4715         cxt->xstorm_ag_context.cdu_reserved =
4716                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4717                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4718 }
4719
4720 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4721                                     u8 fw_sb_id, u8 sb_index,
4722                                     u8 ticks)
4723 {
4724
4725         u32 addr = BAR_CSTRORM_INTMEM +
4726                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4727         REG_WR8(bp, addr, ticks);
4728         DP(NETIF_MSG_IFUP,
4729            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4730            port, fw_sb_id, sb_index, ticks);
4731 }
4732
4733 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4734                                     u16 fw_sb_id, u8 sb_index,
4735                                     u8 disable)
4736 {
4737         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4738         u32 addr = BAR_CSTRORM_INTMEM +
4739                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4740         u16 flags = REG_RD16(bp, addr);
4741         /* clear and set */
4742         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4743         flags |= enable_flag;
4744         REG_WR16(bp, addr, flags);
4745         DP(NETIF_MSG_IFUP,
4746            "port %x fw_sb_id %d sb_index %d disable %d\n",
4747            port, fw_sb_id, sb_index, disable);
4748 }
4749
4750 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4751                                     u8 sb_index, u8 disable, u16 usec)
4752 {
4753         int port = BP_PORT(bp);
4754         u8 ticks = usec / BNX2X_BTR;
4755
4756         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4757
4758         disable = disable ? 1 : (usec ? 0 : 1);
4759         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4760 }