drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2013 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/tcp.h>
  25 #include <net/ipv6.h>
  26 #include <net/ip6_checksum.h>
  27 #include <net/busy_poll.h>
  28 #include <linux/prefetch.h>
  29 #include "bnx2x_cmn.h"
  30 #include "bnx2x_init.h"
  31 #include "bnx2x_sp.h"
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
  57
  58         /* Copy the NAPI object as it has been already initialized */
  59         from_fp->napi = to_fp->napi;
  60
  61         /* Move bnx2x_fastpath contents */
  62         memcpy(to_fp, from_fp, sizeof(*to_fp));
  63         to_fp->index = to;
  64
  65         /* Retain the tpa_info of the original `to' version as we don't want
  66          * 2 FPs to contain the same tpa_info pointer.
  67          */
  68         to_fp->tpa_info = old_tpa_info;
  69
  70         /* move sp_objs contents as well, as their indices match fp ones */
  71         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  72
  73         /* move fp_stats contents as well, as their indices match fp ones */
  74         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  75
  76         /* Update txdata pointers in fp and move txdata content accordingly:
  77          * Each fp consumes 'max_cos' txdata structures, so the index should be
  78          * decremented by max_cos x delta.
  79          */
  80
  81         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  82         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  83                                 (bp)->max_cos;
  84         if (from == FCOE_IDX(bp)) {
  85                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  86                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  87         }
  88
  89         memcpy(&bp->bnx2x_txq[new_txdata_index],
  90                &bp->bnx2x_txq[old_txdata_index],
  91                sizeof(struct bnx2x_fp_txdata));
  92         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  93 }
  94
  95 /**
  96  * bnx2x_fill_fw_str - Fill buffer with FW version string.
  97  *
  98  * @bp:        driver handle
  99  * @buf:       character buffer to fill with the fw name
 100  * @buf_len:   length of the above buffer
 101  *
 102  */
 103 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 104 {
 105         if (IS_PF(bp)) {
 106                 u8 phy_fw_ver[PHY_FW_VER_LEN];
 107
 108                 phy_fw_ver[0] = '\0';
 109                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
 110                                              phy_fw_ver, PHY_FW_VER_LEN);
 111                 strlcpy(buf, bp->fw_ver, buf_len);
 112                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 113                          "bc %d.%d.%d%s%s",
 114                          (bp->common.bc_ver & 0xff0000) >> 16,
 115                          (bp->common.bc_ver & 0xff00) >> 8,
 116                          (bp->common.bc_ver & 0xff),
 117                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
 118         } else {
 119                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
 120         }
 121 }
 122
 123 /**
 124  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
 125  *
 126  * @bp: driver handle
 127  * @delta:      number of eth queues which were not allocated
 128  */
 129 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
 130 {
 131         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
 132
 133         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
 134          * backward along the array could cause memory to be overridden
 135          */
 136         for (cos = 1; cos < bp->max_cos; cos++) {
 137                 for (i = 0; i < old_eth_num - delta; i++) {
 138                         struct bnx2x_fastpath *fp = &bp->fp[i];
 139                         int new_idx = cos * (old_eth_num - delta) + i;
 140
 141                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
 142                                sizeof(struct bnx2x_fp_txdata));
 143                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
 144                 }
 145         }
 146 }
 147
 148 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
 149
 150 /* free skb in the packet ring at pos idx
 151  * return idx of last bd freed
 152  */
 153 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 154                              u16 idx, unsigned int *pkts_compl,
 155                              unsigned int *bytes_compl)
 156 {
 157         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 158         struct eth_tx_start_bd *tx_start_bd;
 159         struct eth_tx_bd *tx_data_bd;
 160         struct sk_buff *skb = tx_buf->skb;
 161         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 162         int nbd;
 163
 164         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 165         prefetch(&skb->end);
 166
 167         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 168            txdata->txq_index, idx, tx_buf, skb);
 169
 170         /* unmap first bd */
 171         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 172         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 173                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 174
 175         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 176 #ifdef BNX2X_STOP_ON_ERROR
 177         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 178                 BNX2X_ERR("BAD nbd!\n");
 179                 bnx2x_panic();
 180         }
 181 #endif
 182         new_cons = nbd + tx_buf->first_bd;
 183
 184         /* Get the next bd */
 185         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 186
 187         /* Skip a parse bd... */
 188         --nbd;
 189         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 190
 191         /* ...and the TSO split header bd since they have no mapping */
 192         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 193                 --nbd;
 194                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 195         }
 196
 197         /* now free frags */
 198         while (nbd > 0) {
 199
 200                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 201                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 202                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 203                 if (--nbd)
 204                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 205         }
 206
 207         /* release skb */
 208         WARN_ON(!skb);
 209         if (likely(skb)) {
 210                 (*pkts_compl)++;
 211                 (*bytes_compl) += skb->len;
 212         }
 213
 214         dev_kfree_skb_any(skb);
 215         tx_buf->first_bd = 0;
 216         tx_buf->skb = NULL;
 217
 218         return new_cons;
 219 }
 220
 221 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 222 {
 223         struct netdev_queue *txq;
 224         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 225         unsigned int pkts_compl = 0, bytes_compl = 0;
 226
 227 #ifdef BNX2X_STOP_ON_ERROR
 228         if (unlikely(bp->panic))
 229                 return -1;
 230 #endif
 231
 232         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 233         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 234         sw_cons = txdata->tx_pkt_cons;
 235
 236         while (sw_cons != hw_cons) {
 237                 u16 pkt_cons;
 238
 239                 pkt_cons = TX_BD(sw_cons);
 240
 241                 DP(NETIF_MSG_TX_DONE,
 242                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 243                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 244
 245                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 246                                             &pkts_compl, &bytes_compl);
 247
 248                 sw_cons++;
 249         }
 250
 251         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 252
 253         txdata->tx_pkt_cons = sw_cons;
 254         txdata->tx_bd_cons = bd_cons;
 255
 256         /* Need to make the tx_bd_cons update visible to start_xmit()
 257          * before checking for netif_tx_queue_stopped().  Without the
 258          * memory barrier, there is a small possibility that
 259          * start_xmit() will miss it and cause the queue to be stopped
 260          * forever.
 261          * On the other hand we need an rmb() here to ensure the proper
 262          * ordering of bit testing in the following
 263          * netif_tx_queue_stopped(txq) call.
 264          */
 265         smp_mb();
 266
 267         if (unlikely(netif_tx_queue_stopped(txq))) {
 268                 /* Taking tx_lock() is needed to prevent re-enabling the queue
 269                  * while it's empty. This could have happen if rx_action() gets
 270                  * suspended in bnx2x_tx_int() after the condition before
 271                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 272                  *
 273                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 274                  * sends some packets consuming the whole queue again->
 275                  * stops the queue
 276                  */
 277
 278                 __netif_tx_lock(txq, smp_processor_id());
 279
 280                 if ((netif_tx_queue_stopped(txq)) &&
 281                     (bp->state == BNX2X_STATE_OPEN) &&
 282                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 283                         netif_tx_wake_queue(txq);
 284
 285                 __netif_tx_unlock(txq);
 286         }
 287         return 0;
 288 }
 289
 290 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 291                                              u16 idx)
 292 {
 293         u16 last_max = fp->last_max_sge;
 294
 295         if (SUB_S16(idx, last_max) > 0)
 296                 fp->last_max_sge = idx;
 297 }
 298
 299 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 300                                          u16 sge_len,
 301                                          struct eth_end_agg_rx_cqe *cqe)
 302 {
 303         struct bnx2x *bp = fp->bp;
 304         u16 last_max, last_elem, first_elem;
 305         u16 delta = 0;
 306         u16 i;
 307
 308         if (!sge_len)
 309                 return;
 310
 311         /* First mark all used pages */
 312         for (i = 0; i < sge_len; i++)
 313                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 314                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 315
 316         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 317            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 318
 319         /* Here we assume that the last SGE index is the biggest */
 320         prefetch((void *)(fp->sge_mask));
 321         bnx2x_update_last_max_sge(fp,
 322                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 323
 324         last_max = RX_SGE(fp->last_max_sge);
 325         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 326         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 327
 328         /* If ring is not full */
 329         if (last_elem + 1 != first_elem)
 330                 last_elem++;
 331
 332         /* Now update the prod */
 333         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 334                 if (likely(fp->sge_mask[i]))
 335                         break;
 336
 337                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 338                 delta += BIT_VEC64_ELEM_SZ;
 339         }
 340
 341         if (delta > 0) {
 342                 fp->rx_sge_prod += delta;
 343                 /* clear page-end entries */
 344                 bnx2x_clear_sge_mask_next_elems(fp);
 345         }
 346
 347         DP(NETIF_MSG_RX_STATUS,
 348            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 349            fp->last_max_sge, fp->rx_sge_prod);
 350 }
 351
 352 /* Get Toeplitz hash value in the skb using the value from the
 353  * CQE (calculated by HW).
 354  */
 355 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 356                             const struct eth_fast_path_rx_cqe *cqe,
 357                             bool *l4_rxhash)
 358 {
 359         /* Get Toeplitz hash from CQE */
 360         if ((bp->dev->features & NETIF_F_RXHASH) &&
 361             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 362                 enum eth_rss_hash_type htype;
 363
 364                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 365                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 366                              (htype == TCP_IPV6_HASH_TYPE);
 367                 return le32_to_cpu(cqe->rss_hash_result);
 368         }
 369         *l4_rxhash = false;
 370         return 0;
 371 }
 372
 373 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 374                             u16 cons, u16 prod,
 375                             struct eth_fast_path_rx_cqe *cqe)
 376 {
 377         struct bnx2x *bp = fp->bp;
 378         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 379         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 380         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 381         dma_addr_t mapping;
 382         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 383         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 384
 385         /* print error if current state != stop */
 386         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 387                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 388
 389         /* Try to map an empty data buffer from the aggregation info  */
 390         mapping = dma_map_single(&bp->pdev->dev,
 391                                  first_buf->data + NET_SKB_PAD,
 392                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 393         /*
 394          *  ...if it fails - move the skb from the consumer to the producer
 395          *  and set the current aggregation state as ERROR to drop it
 396          *  when TPA_STOP arrives.
 397          */
 398
 399         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 400                 /* Move the BD from the consumer to the producer */
 401                 bnx2x_reuse_rx_data(fp, cons, prod);
 402                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 403                 return;
 404         }
 405
 406         /* move empty data from pool to prod */
 407         prod_rx_buf->data = first_buf->data;
 408         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 409         /* point prod_bd to new data */
 410         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 411         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 412
 413         /* move partial skb from cons to pool (don't unmap yet) */
 414         *first_buf = *cons_rx_buf;
 415
 416         /* mark bin state as START */
 417         tpa_info->parsing_flags =
 418                 le16_to_cpu(cqe->pars_flags.flags);
 419         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 420         tpa_info->tpa_state = BNX2X_TPA_START;
 421         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 422         tpa_info->placement_offset = cqe->placement_offset;
 423         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 424         if (fp->mode == TPA_MODE_GRO) {
 425                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 426                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
 427                 tpa_info->gro_size = gro_size;
 428         }
 429
 430 #ifdef BNX2X_STOP_ON_ERROR
 431         fp->tpa_queue_used |= (1 << queue);
 432 #ifdef _ASM_GENERIC_INT_L64_H
 433         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 434 #else
 435         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 436 #endif
 437            fp->tpa_queue_used);
 438 #endif
 439 }
 440
 441 /* Timestamp option length allowed for TPA aggregation:
 442  *
 443  *              nop nop kind length echo val
 444  */
 445 #define TPA_TSTAMP_OPT_LEN      12
 446 /**
 447  * bnx2x_set_gro_params - compute GRO values
 448  *
 449  * @skb:                packet skb
 450  * @parsing_flags:      parsing flags from the START CQE
 451  * @len_on_bd:          total length of the first packet for the
 452  *                      aggregation.
 453  * @pkt_len:            length of all segments
 454  *
 455  * Approximate value of the MSS for this aggregation calculated using
 456  * the first packet of it.
 457  * Compute number of aggregated segments, and gso_type.
 458  */
 459 static void bnx2x_set_gro_params(struct sk_buff *skb, u16 parsing_flags,
 460                                  u16 len_on_bd, unsigned int pkt_len,
 461                                  u16 num_of_coalesced_segs)
 462 {
 463         /* TPA aggregation won't have either IP options or TCP options
 464          * other than timestamp or IPv6 extension headers.
 465          */
 466         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 467
 468         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 469             PRS_FLAG_OVERETH_IPV6) {
 470                 hdrs_len += sizeof(struct ipv6hdr);
 471                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 472         } else {
 473                 hdrs_len += sizeof(struct iphdr);
 474                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
 475         }
 476
 477         /* Check if there was a TCP timestamp, if there is it's will
 478          * always be 12 bytes length: nop nop kind length echo val.
 479          *
 480          * Otherwise FW would close the aggregation.
 481          */
 482         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 483                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 484
 485         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
 486
 487         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
 488          * to skb_shinfo(skb)->gso_segs
 489          */
 490         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
 491 }
 492
 493 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 494                               u16 index, gfp_t gfp_mask)
 495 {
 496         struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
 497         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 498         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 499         dma_addr_t mapping;
 500
 501         if (unlikely(page == NULL)) {
 502                 BNX2X_ERR("Can't alloc sge\n");
 503                 return -ENOMEM;
 504         }
 505
 506         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 507                                SGE_PAGES, DMA_FROM_DEVICE);
 508         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 509                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 510                 BNX2X_ERR("Can't map sge\n");
 511                 return -ENOMEM;
 512         }
 513
 514         sw_buf->page = page;
 515         dma_unmap_addr_set(sw_buf, mapping, mapping);
 516
 517         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 518         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 519
 520         return 0;
 521 }
 522
 523 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 524                                struct bnx2x_agg_info *tpa_info,
 525                                u16 pages,
 526                                struct sk_buff *skb,
 527                                struct eth_end_agg_rx_cqe *cqe,
 528                                u16 cqe_idx)
 529 {
 530         struct sw_rx_page *rx_pg, old_rx_pg;
 531         u32 i, frag_len, frag_size;
 532         int err, j, frag_id = 0;
 533         u16 len_on_bd = tpa_info->len_on_bd;
 534         u16 full_page = 0, gro_size = 0;
 535
 536         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 537
 538         if (fp->mode == TPA_MODE_GRO) {
 539                 gro_size = tpa_info->gro_size;
 540                 full_page = tpa_info->full_page;
 541         }
 542
 543         /* This is needed in order to enable forwarding support */
 544         if (frag_size)
 545                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
 546                                      le16_to_cpu(cqe->pkt_len),
 547                                      le16_to_cpu(cqe->num_of_coalesced_segs));
 548
 549 #ifdef BNX2X_STOP_ON_ERROR
 550         if (pages > min_t(u32, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
 551                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 552                           pages, cqe_idx);
 553                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 554                 bnx2x_panic();
 555                 return -EINVAL;
 556         }
 557 #endif
 558
 559         /* Run through the SGL and compose the fragmented skb */
 560         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 561                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 562
 563                 /* FW gives the indices of the SGE as if the ring is an array
 564                    (meaning that "next" element will consume 2 indices) */
 565                 if (fp->mode == TPA_MODE_GRO)
 566                         frag_len = min_t(u32, frag_size, (u32)full_page);
 567                 else /* LRO */
 568                         frag_len = min_t(u32, frag_size, (u32)SGE_PAGES);
 569
 570                 rx_pg = &fp->rx_page_ring[sge_idx];
 571                 old_rx_pg = *rx_pg;
 572
 573                 /* If we fail to allocate a substitute page, we simply stop
 574                    where we are and drop the whole packet */
 575                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
 576                 if (unlikely(err)) {
 577                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 578                         return err;
 579                 }
 580
 581                 /* Unmap the page as we're going to pass it to the stack */
 582                 dma_unmap_page(&bp->pdev->dev,
 583                                dma_unmap_addr(&old_rx_pg, mapping),
 584                                SGE_PAGES, DMA_FROM_DEVICE);
 585                 /* Add one frag and update the appropriate fields in the skb */
 586                 if (fp->mode == TPA_MODE_LRO)
 587                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 588                 else { /* GRO */
 589                         int rem;
 590                         int offset = 0;
 591                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 592                                 int len = rem > gro_size ? gro_size : rem;
 593                                 skb_fill_page_desc(skb, frag_id++,
 594                                                    old_rx_pg.page, offset, len);
 595                                 if (offset)
 596                                         get_page(old_rx_pg.page);
 597                                 offset += len;
 598                         }
 599                 }
 600
 601                 skb->data_len += frag_len;
 602                 skb->truesize += SGE_PAGES;
 603                 skb->len += frag_len;
 604
 605                 frag_size -= frag_len;
 606         }
 607
 608         return 0;
 609 }
 610
 611 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 612 {
 613         if (fp->rx_frag_size)
 614                 put_page(virt_to_head_page(data));
 615         else
 616                 kfree(data);
 617 }
 618
 619 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
 620 {
 621         if (fp->rx_frag_size) {
 622                 /* GFP_KERNEL allocations are used only during initialization */
 623                 if (unlikely(gfp_mask & __GFP_WAIT))
 624                         return (void *)__get_free_page(gfp_mask);
 625
 626                 return netdev_alloc_frag(fp->rx_frag_size);
 627         }
 628
 629         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
 630 }
 631
 632 #ifdef CONFIG_INET
 633 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
 634 {
 635         const struct iphdr *iph = ip_hdr(skb);
 636         struct tcphdr *th;
 637
 638         skb_set_transport_header(skb, sizeof(struct iphdr));
 639         th = tcp_hdr(skb);
 640
 641         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
 642                                   iph->saddr, iph->daddr, 0);
 643 }
 644
 645 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
 646 {
 647         struct ipv6hdr *iph = ipv6_hdr(skb);
 648         struct tcphdr *th;
 649
 650         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
 651         th = tcp_hdr(skb);
 652
 653         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
 654                                   &iph->saddr, &iph->daddr, 0);
 655 }
 656
 657 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
 658                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
 659 {
 660         skb_set_network_header(skb, 0);
 661         gro_func(bp, skb);
 662         tcp_gro_complete(skb);
 663 }
 664 #endif
 665
 666 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 667                                struct sk_buff *skb)
 668 {
 669 #ifdef CONFIG_INET
 670         if (skb_shinfo(skb)->gso_size) {
 671                 switch (be16_to_cpu(skb->protocol)) {
 672                 case ETH_P_IP:
 673                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
 674                         break;
 675                 case ETH_P_IPV6:
 676                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 677                         break;
 678                 default:
 679                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
 680                                   be16_to_cpu(skb->protocol));
 681                 }
 682         }
 683 #endif
 684         napi_gro_receive(&fp->napi, skb);
 685 }
 686
 687 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 688                            struct bnx2x_agg_info *tpa_info,
 689                            u16 pages,
 690                            struct eth_end_agg_rx_cqe *cqe,
 691                            u16 cqe_idx)
 692 {
 693         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 694         u8 pad = tpa_info->placement_offset;
 695         u16 len = tpa_info->len_on_bd;
 696         struct sk_buff *skb = NULL;
 697         u8 *new_data, *data = rx_buf->data;
 698         u8 old_tpa_state = tpa_info->tpa_state;
 699
 700         tpa_info->tpa_state = BNX2X_TPA_STOP;
 701
 702         /* If we there was an error during the handling of the TPA_START -
 703          * drop this aggregation.
 704          */
 705         if (old_tpa_state == BNX2X_TPA_ERROR)
 706                 goto drop;
 707
 708         /* Try to allocate the new data */
 709         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
 710         /* Unmap skb in the pool anyway, as we are going to change
 711            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 712            fails. */
 713         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 714                          fp->rx_buf_size, DMA_FROM_DEVICE);
 715         if (likely(new_data))
 716                 skb = build_skb(data, fp->rx_frag_size);
 717
 718         if (likely(skb)) {
 719 #ifdef BNX2X_STOP_ON_ERROR
 720                 if (pad + len > fp->rx_buf_size) {
 721                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 722                                   pad, len, fp->rx_buf_size);
 723                         bnx2x_panic();
 724                         return;
 725                 }
 726 #endif
 727
 728                 skb_reserve(skb, pad + NET_SKB_PAD);
 729                 skb_put(skb, len);
 730                 skb->rxhash = tpa_info->rxhash;
 731                 skb->l4_rxhash = tpa_info->l4_rxhash;
 732
 733                 skb->protocol = eth_type_trans(skb, bp->dev);
 734                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 735
 736                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 737                                          skb, cqe, cqe_idx)) {
 738                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 739                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
 740                         bnx2x_gro_receive(bp, fp, skb);
 741                 } else {
 742                         DP(NETIF_MSG_RX_STATUS,
 743                            "Failed to allocate new pages - dropping packet!\n");
 744                         dev_kfree_skb_any(skb);
 745                 }
 746
 747                 /* put new data in bin */
 748                 rx_buf->data = new_data;
 749
 750                 return;
 751         }
 752         bnx2x_frag_free(fp, new_data);
 753 drop:
 754         /* drop the packet and keep the buffer in the bin */
 755         DP(NETIF_MSG_RX_STATUS,
 756            "Failed to allocate or map a new skb - dropping packet!\n");
 757         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 758 }
 759
 760 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 761                                u16 index, gfp_t gfp_mask)
 762 {
 763         u8 *data;
 764         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 765         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 766         dma_addr_t mapping;
 767
 768         data = bnx2x_frag_alloc(fp, gfp_mask);
 769         if (unlikely(data == NULL))
 770                 return -ENOMEM;
 771
 772         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 773                                  fp->rx_buf_size,
 774                                  DMA_FROM_DEVICE);
 775         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 776                 bnx2x_frag_free(fp, data);
 777                 BNX2X_ERR("Can't map rx data\n");
 778                 return -ENOMEM;
 779         }
 780
 781         rx_buf->data = data;
 782         dma_unmap_addr_set(rx_buf, mapping, mapping);
 783
 784         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 785         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 786
 787         return 0;
 788 }
 789
 790 static
 791 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 792                                  struct bnx2x_fastpath *fp,
 793                                  struct bnx2x_eth_q_stats *qstats)
 794 {
 795         /* Do nothing if no L4 csum validation was done.
 796          * We do not check whether IP csum was validated. For IPv4 we assume
 797          * that if the card got as far as validating the L4 csum, it also
 798          * validated the IP csum. IPv6 has no IP csum.
 799          */
 800         if (cqe->fast_path_cqe.status_flags &
 801             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 802                 return;
 803
 804         /* If L4 validation was done, check if an error was found. */
 805
 806         if (cqe->fast_path_cqe.type_error_flags &
 807             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 808              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 809                 qstats->hw_csum_err++;
 810         else
 811                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 812 }
 813
 814 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 815 {
 816         struct bnx2x *bp = fp->bp;
 817         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 818         u16 sw_comp_cons, sw_comp_prod;
 819         int rx_pkt = 0;
 820         union eth_rx_cqe *cqe;
 821         struct eth_fast_path_rx_cqe *cqe_fp;
 822
 823 #ifdef BNX2X_STOP_ON_ERROR
 824         if (unlikely(bp->panic))
 825                 return 0;
 826 #endif
 827
 828         bd_cons = fp->rx_bd_cons;
 829         bd_prod = fp->rx_bd_prod;
 830         bd_prod_fw = bd_prod;
 831         sw_comp_cons = fp->rx_comp_cons;
 832         sw_comp_prod = fp->rx_comp_prod;
 833
 834         comp_ring_cons = RCQ_BD(sw_comp_cons);
 835         cqe = &fp->rx_comp_ring[comp_ring_cons];
 836         cqe_fp = &cqe->fast_path_cqe;
 837
 838         DP(NETIF_MSG_RX_STATUS,
 839            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
 840
 841         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
 842                 struct sw_rx_bd *rx_buf = NULL;
 843                 struct sk_buff *skb;
 844                 u8 cqe_fp_flags;
 845                 enum eth_rx_cqe_type cqe_fp_type;
 846                 u16 len, pad, queue;
 847                 u8 *data;
 848                 bool l4_rxhash;
 849
 850 #ifdef BNX2X_STOP_ON_ERROR
 851                 if (unlikely(bp->panic))
 852                         return 0;
 853 #endif
 854
 855                 bd_prod = RX_BD(bd_prod);
 856                 bd_cons = RX_BD(bd_cons);
 857
 858                 cqe_fp_flags = cqe_fp->type_error_flags;
 859                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 860
 861                 DP(NETIF_MSG_RX_STATUS,
 862                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 863                    CQE_TYPE(cqe_fp_flags),
 864                    cqe_fp_flags, cqe_fp->status_flags,
 865                    le32_to_cpu(cqe_fp->rss_hash_result),
 866                    le16_to_cpu(cqe_fp->vlan_tag),
 867                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 868
 869                 /* is this a slowpath msg? */
 870                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 871                         bnx2x_sp_event(fp, cqe);
 872                         goto next_cqe;
 873                 }
 874
 875                 rx_buf = &fp->rx_buf_ring[bd_cons];
 876                 data = rx_buf->data;
 877
 878                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 879                         struct bnx2x_agg_info *tpa_info;
 880                         u16 frag_size, pages;
 881 #ifdef BNX2X_STOP_ON_ERROR
 882                         /* sanity check */
 883                         if (fp->disable_tpa &&
 884                             (CQE_TYPE_START(cqe_fp_type) ||
 885                              CQE_TYPE_STOP(cqe_fp_type)))
 886                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 887                                           CQE_TYPE(cqe_fp_type));
 888 #endif
 889
 890                         if (CQE_TYPE_START(cqe_fp_type)) {
 891                                 u16 queue = cqe_fp->queue_index;
 892                                 DP(NETIF_MSG_RX_STATUS,
 893                                    "calling tpa_start on queue %d\n",
 894                                    queue);
 895
 896                                 bnx2x_tpa_start(fp, queue,
 897                                                 bd_cons, bd_prod,
 898                                                 cqe_fp);
 899
 900                                 goto next_rx;
 901                         }
 902                         queue = cqe->end_agg_cqe.queue_index;
 903                         tpa_info = &fp->tpa_info[queue];
 904                         DP(NETIF_MSG_RX_STATUS,
 905                            "calling tpa_stop on queue %d\n",
 906                            queue);
 907
 908                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 909                                     tpa_info->len_on_bd;
 910
 911                         if (fp->mode == TPA_MODE_GRO)
 912                                 pages = (frag_size + tpa_info->full_page - 1) /
 913                                          tpa_info->full_page;
 914                         else
 915                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 916                                         SGE_PAGE_SHIFT;
 917
 918                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 919                                        &cqe->end_agg_cqe, comp_ring_cons);
 920 #ifdef BNX2X_STOP_ON_ERROR
 921                         if (bp->panic)
 922                                 return 0;
 923 #endif
 924
 925                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 926                         goto next_cqe;
 927                 }
 928                 /* non TPA */
 929                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 930                 pad = cqe_fp->placement_offset;
 931                 dma_sync_single_for_cpu(&bp->pdev->dev,
 932                                         dma_unmap_addr(rx_buf, mapping),
 933                                         pad + RX_COPY_THRESH,
 934                                         DMA_FROM_DEVICE);
 935                 pad += NET_SKB_PAD;
 936                 prefetch(data + pad); /* speedup eth_type_trans() */
 937                 /* is this an error packet? */
 938                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 939                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 940                            "ERROR  flags %x  rx packet %u\n",
 941                            cqe_fp_flags, sw_comp_cons);
 942                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 943                         goto reuse_rx;
 944                 }
 945
 946                 /* Since we don't have a jumbo ring
 947                  * copy small packets if mtu > 1500
 948                  */
 949                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 950                     (len <= RX_COPY_THRESH)) {
 951                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 952                         if (skb == NULL) {
 953                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 954                                    "ERROR  packet dropped because of alloc failure\n");
 955                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 956                                 goto reuse_rx;
 957                         }
 958                         memcpy(skb->data, data + pad, len);
 959                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 960                 } else {
 961                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
 962                                                        GFP_ATOMIC) == 0)) {
 963                                 dma_unmap_single(&bp->pdev->dev,
 964                                                  dma_unmap_addr(rx_buf, mapping),
 965                                                  fp->rx_buf_size,
 966                                                  DMA_FROM_DEVICE);
 967                                 skb = build_skb(data, fp->rx_frag_size);
 968                                 if (unlikely(!skb)) {
 969                                         bnx2x_frag_free(fp, data);
 970                                         bnx2x_fp_qstats(bp, fp)->
 971                                                         rx_skb_alloc_failed++;
 972                                         goto next_rx;
 973                                 }
 974                                 skb_reserve(skb, pad);
 975                         } else {
 976                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 977                                    "ERROR  packet dropped because of alloc failure\n");
 978                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 979 reuse_rx:
 980                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 981                                 goto next_rx;
 982                         }
 983                 }
 984
 985                 skb_put(skb, len);
 986                 skb->protocol = eth_type_trans(skb, bp->dev);
 987
 988                 /* Set Toeplitz hash for a none-LRO skb */
 989                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 990                 skb->l4_rxhash = l4_rxhash;
 991
 992                 skb_checksum_none_assert(skb);
 993
 994                 if (bp->dev->features & NETIF_F_RXCSUM)
 995                         bnx2x_csum_validate(skb, cqe, fp,
 996                                             bnx2x_fp_qstats(bp, fp));
 997
 998                 skb_record_rx_queue(skb, fp->rx_queue);
 999
1000                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1001                     PARSING_FLAGS_VLAN)
1002                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1003                                                le16_to_cpu(cqe_fp->vlan_tag));
1004
1005                 skb_mark_napi_id(skb, &fp->napi);
1006
1007                 if (bnx2x_fp_ll_polling(fp))
1008                         netif_receive_skb(skb);
1009                 else
1010                         napi_gro_receive(&fp->napi, skb);
1011 next_rx:
1012                 rx_buf->data = NULL;
1013
1014                 bd_cons = NEXT_RX_IDX(bd_cons);
1015                 bd_prod = NEXT_RX_IDX(bd_prod);
1016                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1017                 rx_pkt++;
1018 next_cqe:
1019                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1020                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1021
1022                 /* mark CQE as free */
1023                 BNX2X_SEED_CQE(cqe_fp);
1024
1025                 if (rx_pkt == budget)
1026                         break;
1027
1028                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1029                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1030                 cqe_fp = &cqe->fast_path_cqe;
1031         } /* while */
1032
1033         fp->rx_bd_cons = bd_cons;
1034         fp->rx_bd_prod = bd_prod_fw;
1035         fp->rx_comp_cons = sw_comp_cons;
1036         fp->rx_comp_prod = sw_comp_prod;
1037
1038         /* Update producers */
1039         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1040                              fp->rx_sge_prod);
1041
1042         fp->rx_pkt += rx_pkt;
1043         fp->rx_calls++;
1044
1045         return rx_pkt;
1046 }
1047
1048 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1049 {
1050         struct bnx2x_fastpath *fp = fp_cookie;
1051         struct bnx2x *bp = fp->bp;
1052         u8 cos;
1053
1054         DP(NETIF_MSG_INTR,
1055            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1056            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1057
1058         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1059
1060 #ifdef BNX2X_STOP_ON_ERROR
1061         if (unlikely(bp->panic))
1062                 return IRQ_HANDLED;
1063 #endif
1064
1065         /* Handle Rx and Tx according to MSI-X vector */
1066         for_each_cos_in_tx_queue(fp, cos)
1067                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1068
1069         prefetch(&fp->sb_running_index[SM_RX_ID]);
1070         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
1071
1072         return IRQ_HANDLED;
1073 }
1074
1075 /* HW Lock for shared dual port PHYs */
1076 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1077 {
1078         mutex_lock(&bp->port.phy_mutex);
1079
1080         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1081 }
1082
1083 void bnx2x_release_phy_lock(struct bnx2x *bp)
1084 {
1085         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1086
1087         mutex_unlock(&bp->port.phy_mutex);
1088 }
1089
1090 /* calculates MF speed according to current linespeed and MF configuration */
1091 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
1092 {
1093         u16 line_speed = bp->link_vars.line_speed;
1094         if (IS_MF(bp)) {
1095                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
1096                                                    bp->mf_config[BP_VN(bp)]);
1097
1098                 /* Calculate the current MAX line speed limit for the MF
1099                  * devices
1100                  */
1101                 if (IS_MF_SI(bp))
1102                         line_speed = (line_speed * maxCfg) / 100;
1103                 else { /* SD mode */
1104                         u16 vn_max_rate = maxCfg * 100;
1105
1106                         if (vn_max_rate < line_speed)
1107                                 line_speed = vn_max_rate;
1108                 }
1109         }
1110
1111         return line_speed;
1112 }
1113
1114 /**
1115  * bnx2x_fill_report_data - fill link report data to report
1116  *
1117  * @bp:         driver handle
1118  * @data:       link state to update
1119  *
1120  * It uses a none-atomic bit operations because is called under the mutex.
1121  */
1122 static void bnx2x_fill_report_data(struct bnx2x *bp,
1123                                    struct bnx2x_link_report_data *data)
1124 {
1125         u16 line_speed = bnx2x_get_mf_speed(bp);
1126
1127         memset(data, 0, sizeof(*data));
1128
1129         /* Fill the report data: effective line speed */
1130         data->line_speed = line_speed;
1131
1132         /* Link is down */
1133         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1134                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1135                           &data->link_report_flags);
1136
1137         /* Full DUPLEX */
1138         if (bp->link_vars.duplex == DUPLEX_FULL)
1139                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1140
1141         /* Rx Flow Control is ON */
1142         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1143                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1144
1145         /* Tx Flow Control is ON */
1146         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1147                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1148 }
1149
1150 /**
1151  * bnx2x_link_report - report link status to OS.
1152  *
1153  * @bp:         driver handle
1154  *
1155  * Calls the __bnx2x_link_report() under the same locking scheme
1156  * as a link/PHY state managing code to ensure a consistent link
1157  * reporting.
1158  */
1159
1160 void bnx2x_link_report(struct bnx2x *bp)
1161 {
1162         bnx2x_acquire_phy_lock(bp);
1163         __bnx2x_link_report(bp);
1164         bnx2x_release_phy_lock(bp);
1165 }
1166
1167 /**
1168  * __bnx2x_link_report - report link status to OS.
1169  *
1170  * @bp:         driver handle
1171  *
1172  * None atomic implementation.
1173  * Should be called under the phy_lock.
1174  */
1175 void __bnx2x_link_report(struct bnx2x *bp)
1176 {
1177         struct bnx2x_link_report_data cur_data;
1178
1179         /* reread mf_cfg */
1180         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1181                 bnx2x_read_mf_cfg(bp);
1182
1183         /* Read the current link report info */
1184         bnx2x_fill_report_data(bp, &cur_data);
1185
1186         /* Don't report link down or exactly the same link status twice */
1187         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1188             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1189                       &bp->last_reported_link.link_report_flags) &&
1190              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1191                       &cur_data.link_report_flags)))
1192                 return;
1193
1194         bp->link_cnt++;
1195
1196         /* We are going to report a new link parameters now -
1197          * remember the current data for the next time.
1198          */
1199         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1200
1201         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1202                      &cur_data.link_report_flags)) {
1203                 netif_carrier_off(bp->dev);
1204                 netdev_err(bp->dev, "NIC Link is Down\n");
1205                 return;
1206         } else {
1207                 const char *duplex;
1208                 const char *flow;
1209
1210                 netif_carrier_on(bp->dev);
1211
1212                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1213                                        &cur_data.link_report_flags))
1214                         duplex = "full";
1215                 else
1216                         duplex = "half";
1217
1218                 /* Handle the FC at the end so that only these flags would be
1219                  * possibly set. This way we may easily check if there is no FC
1220                  * enabled.
1221                  */
1222                 if (cur_data.link_report_flags) {
1223                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1224                                      &cur_data.link_report_flags)) {
1225                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1226                                      &cur_data.link_report_flags))
1227                                         flow = "ON - receive & transmit";
1228                                 else
1229                                         flow = "ON - receive";
1230                         } else {
1231                                 flow = "ON - transmit";
1232                         }
1233                 } else {
1234                         flow = "none";
1235                 }
1236                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1237                             cur_data.line_speed, duplex, flow);
1238         }
1239 }
1240
1241 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1242 {
1243         int i;
1244
1245         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1246                 struct eth_rx_sge *sge;
1247
1248                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1249                 sge->addr_hi =
1250                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1251                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1252
1253                 sge->addr_lo =
1254                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1255                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1256         }
1257 }
1258
1259 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1260                                 struct bnx2x_fastpath *fp, int last)
1261 {
1262         int i;
1263
1264         for (i = 0; i < last; i++) {
1265                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1266                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1267                 u8 *data = first_buf->data;
1268
1269                 if (data == NULL) {
1270                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1271                         continue;
1272                 }
1273                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1274                         dma_unmap_single(&bp->pdev->dev,
1275                                          dma_unmap_addr(first_buf, mapping),
1276                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1277                 bnx2x_frag_free(fp, data);
1278                 first_buf->data = NULL;
1279         }
1280 }
1281
1282 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1283 {
1284         int j;
1285
1286         for_each_rx_queue_cnic(bp, j) {
1287                 struct bnx2x_fastpath *fp = &bp->fp[j];
1288
1289                 fp->rx_bd_cons = 0;
1290
1291                 /* Activate BD ring */
1292                 /* Warning!
1293                  * this will generate an interrupt (to the TSTORM)
1294                  * must only be done after chip is initialized
1295                  */
1296                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1297                                      fp->rx_sge_prod);
1298         }
1299 }
1300
1301 void bnx2x_init_rx_rings(struct bnx2x *bp)
1302 {
1303         int func = BP_FUNC(bp);
1304         u16 ring_prod;
1305         int i, j;
1306
1307         /* Allocate TPA resources */
1308         for_each_eth_queue(bp, j) {
1309                 struct bnx2x_fastpath *fp = &bp->fp[j];
1310
1311                 DP(NETIF_MSG_IFUP,
1312                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1313
1314                 if (!fp->disable_tpa) {
1315                         /* Fill the per-aggregation pool */
1316                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1317                                 struct bnx2x_agg_info *tpa_info =
1318                                         &fp->tpa_info[i];
1319                                 struct sw_rx_bd *first_buf =
1320                                         &tpa_info->first_buf;
1321
1322                                 first_buf->data =
1323                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1324                                 if (!first_buf->data) {
1325                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1326                                                   j);
1327                                         bnx2x_free_tpa_pool(bp, fp, i);
1328                                         fp->disable_tpa = 1;
1329                                         break;
1330                                 }
1331                                 dma_unmap_addr_set(first_buf, mapping, 0);
1332                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1333                         }
1334
1335                         /* "next page" elements initialization */
1336                         bnx2x_set_next_page_sgl(fp);
1337
1338                         /* set SGEs bit mask */
1339                         bnx2x_init_sge_ring_bit_mask(fp);
1340
1341                         /* Allocate SGEs and initialize the ring elements */
1342                         for (i = 0, ring_prod = 0;
1343                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1344
1345                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1346                                                        GFP_KERNEL) < 0) {
1347                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1348                                                   i);
1349                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1350                                                   j);
1351                                         /* Cleanup already allocated elements */
1352                                         bnx2x_free_rx_sge_range(bp, fp,
1353                                                                 ring_prod);
1354                                         bnx2x_free_tpa_pool(bp, fp,
1355                                                             MAX_AGG_QS(bp));
1356                                         fp->disable_tpa = 1;
1357                                         ring_prod = 0;
1358                                         break;
1359                                 }
1360                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1361                         }
1362
1363                         fp->rx_sge_prod = ring_prod;
1364                 }
1365         }
1366
1367         for_each_eth_queue(bp, j) {
1368                 struct bnx2x_fastpath *fp = &bp->fp[j];
1369
1370                 fp->rx_bd_cons = 0;
1371
1372                 /* Activate BD ring */
1373                 /* Warning!
1374                  * this will generate an interrupt (to the TSTORM)
1375                  * must only be done after chip is initialized
1376                  */
1377                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1378                                      fp->rx_sge_prod);
1379
1380                 if (j != 0)
1381                         continue;
1382
1383                 if (CHIP_IS_E1(bp)) {
1384                         REG_WR(bp, BAR_USTRORM_INTMEM +
1385                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1386                                U64_LO(fp->rx_comp_mapping));
1387                         REG_WR(bp, BAR_USTRORM_INTMEM +
1388                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1389                                U64_HI(fp->rx_comp_mapping));
1390                 }
1391         }
1392 }
1393
1394 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1395 {
1396         u8 cos;
1397         struct bnx2x *bp = fp->bp;
1398
1399         for_each_cos_in_tx_queue(fp, cos) {
1400                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1401                 unsigned pkts_compl = 0, bytes_compl = 0;
1402
1403                 u16 sw_prod = txdata->tx_pkt_prod;
1404                 u16 sw_cons = txdata->tx_pkt_cons;
1405
1406                 while (sw_cons != sw_prod) {
1407                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1408                                           &pkts_compl, &bytes_compl);
1409                         sw_cons++;
1410                 }
1411
1412                 netdev_tx_reset_queue(
1413                         netdev_get_tx_queue(bp->dev,
1414                                             txdata->txq_index));
1415         }
1416 }
1417
1418 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1419 {
1420         int i;
1421
1422         for_each_tx_queue_cnic(bp, i) {
1423                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1424         }
1425 }
1426
1427 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1428 {
1429         int i;
1430
1431         for_each_eth_queue(bp, i) {
1432                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1433         }
1434 }
1435
1436 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1437 {
1438         struct bnx2x *bp = fp->bp;
1439         int i;
1440
1441         /* ring wasn't allocated */
1442         if (fp->rx_buf_ring == NULL)
1443                 return;
1444
1445         for (i = 0; i < NUM_RX_BD; i++) {
1446                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1447                 u8 *data = rx_buf->data;
1448
1449                 if (data == NULL)
1450                         continue;
1451                 dma_unmap_single(&bp->pdev->dev,
1452                                  dma_unmap_addr(rx_buf, mapping),
1453                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1454
1455                 rx_buf->data = NULL;
1456                 bnx2x_frag_free(fp, data);
1457         }
1458 }
1459
1460 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1461 {
1462         int j;
1463
1464         for_each_rx_queue_cnic(bp, j) {
1465                 bnx2x_free_rx_bds(&bp->fp[j]);
1466         }
1467 }
1468
1469 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1470 {
1471         int j;
1472
1473         for_each_eth_queue(bp, j) {
1474                 struct bnx2x_fastpath *fp = &bp->fp[j];
1475
1476                 bnx2x_free_rx_bds(fp);
1477
1478                 if (!fp->disable_tpa)
1479                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1480         }
1481 }
1482
1483 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1484 {
1485         bnx2x_free_tx_skbs_cnic(bp);
1486         bnx2x_free_rx_skbs_cnic(bp);
1487 }
1488
1489 void bnx2x_free_skbs(struct bnx2x *bp)
1490 {
1491         bnx2x_free_tx_skbs(bp);
1492         bnx2x_free_rx_skbs(bp);
1493 }
1494
1495 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1496 {
1497         /* load old values */
1498         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1499
1500         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1501                 /* leave all but MAX value */
1502                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1503
1504                 /* set new MAX value */
1505                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1506                                 & FUNC_MF_CFG_MAX_BW_MASK;
1507
1508                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1509         }
1510 }
1511
1512 /**
1513  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1514  *
1515  * @bp:         driver handle
1516  * @nvecs:      number of vectors to be released
1517  */
1518 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1519 {
1520         int i, offset = 0;
1521
1522         if (nvecs == offset)
1523                 return;
1524
1525         /* VFs don't have a default SB */
1526         if (IS_PF(bp)) {
1527                 free_irq(bp->msix_table[offset].vector, bp->dev);
1528                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1529                    bp->msix_table[offset].vector);
1530                 offset++;
1531         }
1532
1533         if (CNIC_SUPPORT(bp)) {
1534                 if (nvecs == offset)
1535                         return;
1536                 offset++;
1537         }
1538
1539         for_each_eth_queue(bp, i) {
1540                 if (nvecs == offset)
1541                         return;
1542                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1543                    i, bp->msix_table[offset].vector);
1544
1545                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1546         }
1547 }
1548
1549 void bnx2x_free_irq(struct bnx2x *bp)
1550 {
1551         if (bp->flags & USING_MSIX_FLAG &&
1552             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1553                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1554
1555                 /* vfs don't have a default status block */
1556                 if (IS_PF(bp))
1557                         nvecs++;
1558
1559                 bnx2x_free_msix_irqs(bp, nvecs);
1560         } else {
1561                 free_irq(bp->dev->irq, bp->dev);
1562         }
1563 }
1564
1565 int bnx2x_enable_msix(struct bnx2x *bp)
1566 {
1567         int msix_vec = 0, i, rc;
1568
1569         /* VFs don't have a default status block */
1570         if (IS_PF(bp)) {
1571                 bp->msix_table[msix_vec].entry = msix_vec;
1572                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1573                                bp->msix_table[0].entry);
1574                 msix_vec++;
1575         }
1576
1577         /* Cnic requires an msix vector for itself */
1578         if (CNIC_SUPPORT(bp)) {
1579                 bp->msix_table[msix_vec].entry = msix_vec;
1580                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1581                                msix_vec, bp->msix_table[msix_vec].entry);
1582                 msix_vec++;
1583         }
1584
1585         /* We need separate vectors for ETH queues only (not FCoE) */
1586         for_each_eth_queue(bp, i) {
1587                 bp->msix_table[msix_vec].entry = msix_vec;
1588                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1589                                msix_vec, msix_vec, i);
1590                 msix_vec++;
1591         }
1592
1593         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1594            msix_vec);
1595
1596         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], msix_vec);
1597
1598         /*
1599          * reconfigure number of tx/rx queues according to available
1600          * MSI-X vectors
1601          */
1602         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1603                 /* how less vectors we will have? */
1604                 int diff = msix_vec - rc;
1605
1606                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1607
1608                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1609
1610                 if (rc) {
1611                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1612                         goto no_msix;
1613                 }
1614                 /*
1615                  * decrease number of queues by number of unallocated entries
1616                  */
1617                 bp->num_ethernet_queues -= diff;
1618                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1619
1620                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1621                                bp->num_queues);
1622         } else if (rc > 0) {
1623                 /* Get by with single vector */
1624                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1625                 if (rc) {
1626                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1627                                        rc);
1628                         goto no_msix;
1629                 }
1630
1631                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1632                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1633
1634                 BNX2X_DEV_INFO("set number of queues to 1\n");
1635                 bp->num_ethernet_queues = 1;
1636                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1637         } else if (rc < 0) {
1638                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1639                 goto no_msix;
1640         }
1641
1642         bp->flags |= USING_MSIX_FLAG;
1643
1644         return 0;
1645
1646 no_msix:
1647         /* fall to INTx if not enough memory */
1648         if (rc == -ENOMEM)
1649                 bp->flags |= DISABLE_MSI_FLAG;
1650
1651         return rc;
1652 }
1653
1654 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1655 {
1656         int i, rc, offset = 0;
1657
1658         /* no default status block for vf */
1659         if (IS_PF(bp)) {
1660                 rc = request_irq(bp->msix_table[offset++].vector,
1661                                  bnx2x_msix_sp_int, 0,
1662                                  bp->dev->name, bp->dev);
1663                 if (rc) {
1664                         BNX2X_ERR("request sp irq failed\n");
1665                         return -EBUSY;
1666                 }
1667         }
1668
1669         if (CNIC_SUPPORT(bp))
1670                 offset++;
1671
1672         for_each_eth_queue(bp, i) {
1673                 struct bnx2x_fastpath *fp = &bp->fp[i];
1674                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1675                          bp->dev->name, i);
1676
1677                 rc = request_irq(bp->msix_table[offset].vector,
1678                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1679                 if (rc) {
1680                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1681                               bp->msix_table[offset].vector, rc);
1682                         bnx2x_free_msix_irqs(bp, offset);
1683                         return -EBUSY;
1684                 }
1685
1686                 offset++;
1687         }
1688
1689         i = BNX2X_NUM_ETH_QUEUES(bp);
1690         if (IS_PF(bp)) {
1691                 offset = 1 + CNIC_SUPPORT(bp);
1692                 netdev_info(bp->dev,
1693                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1694                             bp->msix_table[0].vector,
1695                             0, bp->msix_table[offset].vector,
1696                             i - 1, bp->msix_table[offset + i - 1].vector);
1697         } else {
1698                 offset = CNIC_SUPPORT(bp);
1699                 netdev_info(bp->dev,
1700                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1701                             0, bp->msix_table[offset].vector,
1702                             i - 1, bp->msix_table[offset + i - 1].vector);
1703         }
1704         return 0;
1705 }
1706
1707 int bnx2x_enable_msi(struct bnx2x *bp)
1708 {
1709         int rc;
1710
1711         rc = pci_enable_msi(bp->pdev);
1712         if (rc) {
1713                 BNX2X_DEV_INFO("MSI is not attainable\n");
1714                 return -1;
1715         }
1716         bp->flags |= USING_MSI_FLAG;
1717
1718         return 0;
1719 }
1720
1721 static int bnx2x_req_irq(struct bnx2x *bp)
1722 {
1723         unsigned long flags;
1724         unsigned int irq;
1725
1726         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1727                 flags = 0;
1728         else
1729                 flags = IRQF_SHARED;
1730
1731         if (bp->flags & USING_MSIX_FLAG)
1732                 irq = bp->msix_table[0].vector;
1733         else
1734                 irq = bp->pdev->irq;
1735
1736         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1737 }
1738
1739 static int bnx2x_setup_irqs(struct bnx2x *bp)
1740 {
1741         int rc = 0;
1742         if (bp->flags & USING_MSIX_FLAG &&
1743             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1744                 rc = bnx2x_req_msix_irqs(bp);
1745                 if (rc)
1746                         return rc;
1747         } else {
1748                 rc = bnx2x_req_irq(bp);
1749                 if (rc) {
1750                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1751                         return rc;
1752                 }
1753                 if (bp->flags & USING_MSI_FLAG) {
1754                         bp->dev->irq = bp->pdev->irq;
1755                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1756                                     bp->dev->irq);
1757                 }
1758                 if (bp->flags & USING_MSIX_FLAG) {
1759                         bp->dev->irq = bp->msix_table[0].vector;
1760                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1761                                     bp->dev->irq);
1762                 }
1763         }
1764
1765         return 0;
1766 }
1767
1768 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1769 {
1770         int i;
1771
1772         for_each_rx_queue_cnic(bp, i) {
1773                 bnx2x_fp_init_lock(&bp->fp[i]);
1774                 napi_enable(&bnx2x_fp(bp, i, napi));
1775         }
1776 }
1777
1778 static void bnx2x_napi_enable(struct bnx2x *bp)
1779 {
1780         int i;
1781
1782         for_each_eth_queue(bp, i) {
1783                 bnx2x_fp_init_lock(&bp->fp[i]);
1784                 napi_enable(&bnx2x_fp(bp, i, napi));
1785         }
1786 }
1787
1788 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1789 {
1790         int i;
1791
1792         local_bh_disable();
1793         for_each_rx_queue_cnic(bp, i) {
1794                 napi_disable(&bnx2x_fp(bp, i, napi));
1795                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1796                         mdelay(1);
1797         }
1798         local_bh_enable();
1799 }
1800
1801 static void bnx2x_napi_disable(struct bnx2x *bp)
1802 {
1803         int i;
1804
1805         local_bh_disable();
1806         for_each_eth_queue(bp, i) {
1807                 napi_disable(&bnx2x_fp(bp, i, napi));
1808                 while (!bnx2x_fp_lock_napi(&bp->fp[i]))
1809                         mdelay(1);
1810         }
1811         local_bh_enable();
1812 }
1813
1814 void bnx2x_netif_start(struct bnx2x *bp)
1815 {
1816         if (netif_running(bp->dev)) {
1817                 bnx2x_napi_enable(bp);
1818                 if (CNIC_LOADED(bp))
1819                         bnx2x_napi_enable_cnic(bp);
1820                 bnx2x_int_enable(bp);
1821                 if (bp->state == BNX2X_STATE_OPEN)
1822                         netif_tx_wake_all_queues(bp->dev);
1823         }
1824 }
1825
1826 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1827 {
1828         bnx2x_int_disable_sync(bp, disable_hw);
1829         bnx2x_napi_disable(bp);
1830         if (CNIC_LOADED(bp))
1831                 bnx2x_napi_disable_cnic(bp);
1832 }
1833
1834 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1835 {
1836         struct bnx2x *bp = netdev_priv(dev);
1837
1838         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1839                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1840                 u16 ether_type = ntohs(hdr->h_proto);
1841
1842                 /* Skip VLAN tag if present */
1843                 if (ether_type == ETH_P_8021Q) {
1844                         struct vlan_ethhdr *vhdr =
1845                                 (struct vlan_ethhdr *)skb->data;
1846
1847                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1848                 }
1849
1850                 /* If ethertype is FCoE or FIP - use FCoE ring */
1851                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1852                         return bnx2x_fcoe_tx(bp, txq_index);
1853         }
1854
1855         /* select a non-FCoE queue */
1856         return __netdev_pick_tx(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1857 }
1858
1859 void bnx2x_set_num_queues(struct bnx2x *bp)
1860 {
1861         /* RSS queues */
1862         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1863
1864         /* override in STORAGE SD modes */
1865         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1866                 bp->num_ethernet_queues = 1;
1867
1868         /* Add special queues */
1869         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1870         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1871
1872         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1873 }
1874
1875 /**
1876  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1877  *
1878  * @bp:         Driver handle
1879  *
1880  * We currently support for at most 16 Tx queues for each CoS thus we will
1881  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1882  * bp->max_cos.
1883  *
1884  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1885  * index after all ETH L2 indices.
1886  *
1887  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1888  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1889  * 16..31,...) with indices that are not coupled with any real Tx queue.
1890  *
1891  * The proper configuration of skb->queue_mapping is handled by
1892  * bnx2x_select_queue() and __skb_tx_hash().
1893  *
1894  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1895  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1896  */
1897 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1898 {
1899         int rc, tx, rx;
1900
1901         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1902         rx = BNX2X_NUM_ETH_QUEUES(bp);
1903
1904 /* account for fcoe queue */
1905         if (include_cnic && !NO_FCOE(bp)) {
1906                 rx++;
1907                 tx++;
1908         }
1909
1910         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1911         if (rc) {
1912                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1913                 return rc;
1914         }
1915         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1916         if (rc) {
1917                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1918                 return rc;
1919         }
1920
1921         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1922                           tx, rx);
1923
1924         return rc;
1925 }
1926
1927 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1928 {
1929         int i;
1930
1931         for_each_queue(bp, i) {
1932                 struct bnx2x_fastpath *fp = &bp->fp[i];
1933                 u32 mtu;
1934
1935                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1936                 if (IS_FCOE_IDX(i))
1937                         /*
1938                          * Although there are no IP frames expected to arrive to
1939                          * this ring we still want to add an
1940                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1941                          * overrun attack.
1942                          */
1943                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1944                 else
1945                         mtu = bp->dev->mtu;
1946                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1947                                   IP_HEADER_ALIGNMENT_PADDING +
1948                                   ETH_OVREHEAD +
1949                                   mtu +
1950                                   BNX2X_FW_RX_ALIGN_END;
1951                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
1952                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1953                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1954                 else
1955                         fp->rx_frag_size = 0;
1956         }
1957 }
1958
1959 static int bnx2x_init_rss(struct bnx2x *bp)
1960 {
1961         int i;
1962         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1963
1964         /* Prepare the initial contents for the indirection table if RSS is
1965          * enabled
1966          */
1967         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1968                 bp->rss_conf_obj.ind_table[i] =
1969                         bp->fp->cl_id +
1970                         ethtool_rxfh_indir_default(i, num_eth_queues);
1971
1972         /*
1973          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1974          * per-port, so if explicit configuration is needed , do it only
1975          * for a PMF.
1976          *
1977          * For 57712 and newer on the other hand it's a per-function
1978          * configuration.
1979          */
1980         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1981 }
1982
1983 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1984               bool config_hash, bool enable)
1985 {
1986         struct bnx2x_config_rss_params params = {NULL};
1987
1988         /* Although RSS is meaningless when there is a single HW queue we
1989          * still need it enabled in order to have HW Rx hash generated.
1990          *
1991          * if (!is_eth_multi(bp))
1992          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1993          */
1994
1995         params.rss_obj = rss_obj;
1996
1997         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1998
1999         if (enable) {
2000                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2001
2002                 /* RSS configuration */
2003                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2004                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2005                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2006                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2007                 if (rss_obj->udp_rss_v4)
2008                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2009                 if (rss_obj->udp_rss_v6)
2010                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2011         } else {
2012                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2013         }
2014
2015         /* Hash bits */
2016         params.rss_result_mask = MULTI_MASK;
2017
2018         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2019
2020         if (config_hash) {
2021                 /* RSS keys */
2022                 prandom_bytes(params.rss_key, T_ETH_RSS_KEY * 4);
2023                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2024         }
2025
2026         if (IS_PF(bp))
2027                 return bnx2x_config_rss(bp, &params);
2028         else
2029                 return bnx2x_vfpf_config_rss(bp, &params);
2030 }
2031
2032 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
2033 {
2034         struct bnx2x_func_state_params func_params = {NULL};
2035
2036         /* Prepare parameters for function state transitions */
2037         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2038
2039         func_params.f_obj = &bp->func_obj;
2040         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2041
2042         func_params.params.hw_init.load_phase = load_code;
2043
2044         return bnx2x_func_state_change(bp, &func_params);
2045 }
2046
2047 /*
2048  * Cleans the object that have internal lists without sending
2049  * ramrods. Should be run when interrupts are disabled.
2050  */
2051 void bnx2x_squeeze_objects(struct bnx2x *bp)
2052 {
2053         int rc;
2054         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2055         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2056         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2057
2058         /***************** Cleanup MACs' object first *************************/
2059
2060         /* Wait for completion of requested */
2061         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2062         /* Perform a dry cleanup */
2063         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2064
2065         /* Clean ETH primary MAC */
2066         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2067         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2068                                  &ramrod_flags);
2069         if (rc != 0)
2070                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2071
2072         /* Cleanup UC list */
2073         vlan_mac_flags = 0;
2074         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2075         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2076                                  &ramrod_flags);
2077         if (rc != 0)
2078                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2079
2080         /***************** Now clean mcast object *****************************/
2081         rparam.mcast_obj = &bp->mcast_obj;
2082         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2083
2084         /* Add a DEL command... - Since we're doing a driver cleanup only,
2085          * we take a lock surrounding both the initial send and the CONTs,
2086          * as we don't want a true completion to disrupt us in the middle.
2087          */
2088         netif_addr_lock_bh(bp->dev);
2089         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2090         if (rc < 0)
2091                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2092                           rc);
2093
2094         /* ...and wait until all pending commands are cleared */
2095         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2096         while (rc != 0) {
2097                 if (rc < 0) {
2098                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2099                                   rc);
2100                         netif_addr_unlock_bh(bp->dev);
2101                         return;
2102                 }
2103
2104                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2105         }
2106         netif_addr_unlock_bh(bp->dev);
2107 }
2108
2109 #ifndef BNX2X_STOP_ON_ERROR
2110 #define LOAD_ERROR_EXIT(bp, label) \
2111         do { \
2112                 (bp)->state = BNX2X_STATE_ERROR; \
2113                 goto label; \
2114         } while (0)
2115
2116 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2117         do { \
2118                 bp->cnic_loaded = false; \
2119                 goto label; \
2120         } while (0)
2121 #else /*BNX2X_STOP_ON_ERROR*/
2122 #define LOAD_ERROR_EXIT(bp, label) \
2123         do { \
2124                 (bp)->state = BNX2X_STATE_ERROR; \
2125                 (bp)->panic = 1; \
2126                 return -EBUSY; \
2127         } while (0)
2128 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2129         do { \
2130                 bp->cnic_loaded = false; \
2131                 (bp)->panic = 1; \
2132                 return -EBUSY; \
2133         } while (0)
2134 #endif /*BNX2X_STOP_ON_ERROR*/
2135
2136 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2137 {
2138         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2139                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2140         return;
2141 }
2142
2143 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2144 {
2145         int num_groups, vf_headroom = 0;
2146         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2147
2148         /* number of queues for statistics is number of eth queues + FCoE */
2149         u8 num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2150
2151         /* Total number of FW statistics requests =
2152          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2153          * and fcoe l2 queue) stats + num of queues (which includes another 1
2154          * for fcoe l2 queue if applicable)
2155          */
2156         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2157
2158         /* vf stats appear in the request list, but their data is allocated by
2159          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2160          * it is used to determine where to place the vf stats queries in the
2161          * request struct
2162          */
2163         if (IS_SRIOV(bp))
2164                 vf_headroom = bnx2x_vf_headroom(bp);
2165
2166         /* Request is built from stats_query_header and an array of
2167          * stats_query_cmd_group each of which contains
2168          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2169          * configured in the stats_query_header.
2170          */
2171         num_groups =
2172                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2173                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2174                  1 : 0));
2175
2176         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2177            bp->fw_stats_num, vf_headroom, num_groups);
2178         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2179                 num_groups * sizeof(struct stats_query_cmd_group);
2180
2181         /* Data for statistics requests + stats_counter
2182          * stats_counter holds per-STORM counters that are incremented
2183          * when STORM has finished with the current request.
2184          * memory for FCoE offloaded statistics are counted anyway,
2185          * even if they will not be sent.
2186          * VF stats are not accounted for here as the data of VF stats is stored
2187          * in memory allocated by the VF, not here.
2188          */
2189         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2190                 sizeof(struct per_pf_stats) +
2191                 sizeof(struct fcoe_statistics_params) +
2192                 sizeof(struct per_queue_stats) * num_queue_stats +
2193                 sizeof(struct stats_counter);
2194
2195         BNX2X_PCI_ALLOC(bp->fw_stats, &bp->fw_stats_mapping,
2196                         bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2197
2198         /* Set shortcuts */
2199         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2200         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2201         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2202                 ((u8 *)bp->fw_stats + bp->fw_stats_req_sz);
2203         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2204                 bp->fw_stats_req_sz;
2205
2206         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2207            U64_HI(bp->fw_stats_req_mapping),
2208            U64_LO(bp->fw_stats_req_mapping));
2209         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2210            U64_HI(bp->fw_stats_data_mapping),
2211            U64_LO(bp->fw_stats_data_mapping));
2212         return 0;
2213
2214 alloc_mem_err:
2215         bnx2x_free_fw_stats_mem(bp);
2216         BNX2X_ERR("Can't allocate FW stats memory\n");
2217         return -ENOMEM;
2218 }
2219
2220 /* send load request to mcp and analyze response */
2221 static int bnx2x_nic_load_request(struct bnx2x *bp, u32 *load_code)
2222 {
2223         u32 param;
2224
2225         /* init fw_seq */
2226         bp->fw_seq =
2227                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2228                  DRV_MSG_SEQ_NUMBER_MASK);
2229         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2230
2231         /* Get current FW pulse sequence */
2232         bp->fw_drv_pulse_wr_seq =
2233                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2234                  DRV_PULSE_SEQ_MASK);
2235         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2236
2237         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2238
2239         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2240                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2241
2242         /* load request */
2243         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2244
2245         /* if mcp fails to respond we must abort */
2246         if (!(*load_code)) {
2247                 BNX2X_ERR("MCP response failure, aborting\n");
2248                 return -EBUSY;
2249         }
2250
2251         /* If mcp refused (e.g. other port is in diagnostic mode) we
2252          * must abort
2253          */
2254         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2255                 BNX2X_ERR("MCP refused load request, aborting\n");
2256                 return -EBUSY;
2257         }
2258         return 0;
2259 }
2260
2261 /* check whether another PF has already loaded FW to chip. In
2262  * virtualized environments a pf from another VM may have already
2263  * initialized the device including loading FW
2264  */
2265 int bnx2x_nic_load_analyze_req(struct bnx2x *bp, u32 load_code)
2266 {
2267         /* is another pf loaded on this engine? */
2268         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2269             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2270                 /* build my FW version dword */
2271                 u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2272                         (BCM_5710_FW_MINOR_VERSION << 8) +
2273                         (BCM_5710_FW_REVISION_VERSION << 16) +
2274                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2275
2276                 /* read loaded FW from chip */
2277                 u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2278
2279                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2280                    loaded_fw, my_fw);
2281
2282                 /* abort nic load if version mismatch */
2283                 if (my_fw != loaded_fw) {
2284                         BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2285                                   loaded_fw, my_fw);
2286                         return -EBUSY;
2287                 }
2288         }
2289         return 0;
2290 }
2291
2292 /* returns the "mcp load_code" according to global load_count array */
2293 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2294 {
2295         int path = BP_PATH(bp);
2296
2297         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2298            path, load_count[path][0], load_count[path][1],
2299            load_count[path][2]);
2300         load_count[path][0]++;
2301         load_count[path][1 + port]++;
2302         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2303            path, load_count[path][0], load_count[path][1],
2304            load_count[path][2]);
2305         if (load_count[path][0] == 1)
2306                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2307         else if (load_count[path][1 + port] == 1)
2308                 return FW_MSG_CODE_DRV_LOAD_PORT;
2309         else
2310                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2311 }
2312
2313 /* mark PMF if applicable */
2314 static void bnx2x_nic_load_pmf(struct bnx2x *bp, u32 load_code)
2315 {
2316         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2317             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2318             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2319                 bp->port.pmf = 1;
2320                 /* We need the barrier to ensure the ordering between the
2321                  * writing to bp->port.pmf here and reading it from the
2322                  * bnx2x_periodic_task().
2323                  */
2324                 smp_mb();
2325         } else {
2326                 bp->port.pmf = 0;
2327         }
2328
2329         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2330 }
2331
2332 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2333 {
2334         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2335              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2336             (bp->common.shmem2_base)) {
2337                 if (SHMEM2_HAS(bp, dcc_support))
2338                         SHMEM2_WR(bp, dcc_support,
2339                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2340                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2341                 if (SHMEM2_HAS(bp, afex_driver_support))
2342                         SHMEM2_WR(bp, afex_driver_support,
2343                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2344         }
2345
2346         /* Set AFEX default VLAN tag to an invalid value */
2347         bp->afex_def_vlan_tag = -1;
2348 }
2349
2350 /**
2351  * bnx2x_bz_fp - zero content of the fastpath structure.
2352  *
2353  * @bp:         driver handle
2354  * @index:      fastpath index to be zeroed
2355  *
2356  * Makes sure the contents of the bp->fp[index].napi is kept
2357  * intact.
2358  */
2359 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2360 {
2361         struct bnx2x_fastpath *fp = &bp->fp[index];
2362         int cos;
2363         struct napi_struct orig_napi = fp->napi;
2364         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2365
2366         /* bzero bnx2x_fastpath contents */
2367         if (fp->tpa_info)
2368                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2369                        sizeof(struct bnx2x_agg_info));
2370         memset(fp, 0, sizeof(*fp));
2371
2372         /* Restore the NAPI object as it has been already initialized */
2373         fp->napi = orig_napi;
2374         fp->tpa_info = orig_tpa_info;
2375         fp->bp = bp;
2376         fp->index = index;
2377         if (IS_ETH_FP(fp))
2378                 fp->max_cos = bp->max_cos;
2379         else
2380                 /* Special queues support only one CoS */
2381                 fp->max_cos = 1;
2382
2383         /* Init txdata pointers */
2384         if (IS_FCOE_FP(fp))
2385                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2386         if (IS_ETH_FP(fp))
2387                 for_each_cos_in_tx_queue(fp, cos)
2388                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2389                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2390
2391         /* set the tpa flag for each queue. The tpa flag determines the queue
2392          * minimal size so it must be set prior to queue memory allocation
2393          */
2394         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2395                                   (bp->flags & GRO_ENABLE_FLAG &&
2396                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2397         if (bp->flags & TPA_ENABLE_FLAG)
2398                 fp->mode = TPA_MODE_LRO;
2399         else if (bp->flags & GRO_ENABLE_FLAG)
2400                 fp->mode = TPA_MODE_GRO;
2401
2402         /* We don't want TPA on an FCoE L2 ring */
2403         if (IS_FCOE_FP(fp))
2404                 fp->disable_tpa = 1;
2405 }
2406
2407 int bnx2x_load_cnic(struct bnx2x *bp)
2408 {
2409         int i, rc, port = BP_PORT(bp);
2410
2411         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2412
2413         mutex_init(&bp->cnic_mutex);
2414
2415         if (IS_PF(bp)) {
2416                 rc = bnx2x_alloc_mem_cnic(bp);
2417                 if (rc) {
2418                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2419                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2420                 }
2421         }
2422
2423         rc = bnx2x_alloc_fp_mem_cnic(bp);
2424         if (rc) {
2425                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2426                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2427         }
2428
2429         /* Update the number of queues with the cnic queues */
2430         rc = bnx2x_set_real_num_queues(bp, 1);
2431         if (rc) {
2432                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2433                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2434         }
2435
2436         /* Add all CNIC NAPI objects */
2437         bnx2x_add_all_napi_cnic(bp);
2438         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2439         bnx2x_napi_enable_cnic(bp);
2440
2441         rc = bnx2x_init_hw_func_cnic(bp);
2442         if (rc)
2443                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2444
2445         bnx2x_nic_init_cnic(bp);
2446
2447         if (IS_PF(bp)) {
2448                 /* Enable Timer scan */
2449                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2450
2451                 /* setup cnic queues */
2452                 for_each_cnic_queue(bp, i) {
2453                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2454                         if (rc) {
2455                                 BNX2X_ERR("Queue setup failed\n");
2456                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2457                         }
2458                 }
2459         }
2460
2461         /* Initialize Rx filter. */
2462         bnx2x_set_rx_mode_inner(bp);
2463
2464         /* re-read iscsi info */
2465         bnx2x_get_iscsi_info(bp);
2466         bnx2x_setup_cnic_irq_info(bp);
2467         bnx2x_setup_cnic_info(bp);
2468         bp->cnic_loaded = true;
2469         if (bp->state == BNX2X_STATE_OPEN)
2470                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2471
2472         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2473
2474         return 0;
2475
2476 #ifndef BNX2X_STOP_ON_ERROR
2477 load_error_cnic2:
2478         /* Disable Timer scan */
2479         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2480
2481 load_error_cnic1:
2482         bnx2x_napi_disable_cnic(bp);
2483         /* Update the number of queues without the cnic queues */
2484         if (bnx2x_set_real_num_queues(bp, 0))
2485                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2486 load_error_cnic0:
2487         BNX2X_ERR("CNIC-related load failed\n");
2488         bnx2x_free_fp_mem_cnic(bp);
2489         bnx2x_free_mem_cnic(bp);
2490         return rc;
2491 #endif /* ! BNX2X_STOP_ON_ERROR */
2492 }
2493
2494 /* must be called with rtnl_lock */
2495 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2496 {
2497         int port = BP_PORT(bp);
2498         int i, rc = 0, load_code = 0;
2499
2500         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2501         DP(NETIF_MSG_IFUP,
2502            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2503
2504 #ifdef BNX2X_STOP_ON_ERROR
2505         if (unlikely(bp->panic)) {
2506                 BNX2X_ERR("Can't load NIC when there is panic\n");
2507                 return -EPERM;
2508         }
2509 #endif
2510
2511         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2512
2513         /* zero the structure w/o any lock, before SP handler is initialized */
2514         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2515         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2516                 &bp->last_reported_link.link_report_flags);
2517
2518         if (IS_PF(bp))
2519                 /* must be called before memory allocation and HW init */
2520                 bnx2x_ilt_set_info(bp);
2521
2522         /*
2523          * Zero fastpath structures preserving invariants like napi, which are
2524          * allocated only once, fp index, max_cos, bp pointer.
2525          * Also set fp->disable_tpa and txdata_ptr.
2526          */
2527         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2528         for_each_queue(bp, i)
2529                 bnx2x_bz_fp(bp, i);
2530         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2531                                   bp->num_cnic_queues) *
2532                                   sizeof(struct bnx2x_fp_txdata));
2533
2534         bp->fcoe_init = false;
2535
2536         /* Set the receive queues buffer size */
2537         bnx2x_set_rx_buf_size(bp);
2538
2539         if (IS_PF(bp)) {
2540                 rc = bnx2x_alloc_mem(bp);
2541                 if (rc) {
2542                         BNX2X_ERR("Unable to allocate bp memory\n");
2543                         return rc;
2544                 }
2545         }
2546
2547         /* Allocated memory for FW statistics  */
2548         if (bnx2x_alloc_fw_stats_mem(bp))
2549                 LOAD_ERROR_EXIT(bp, load_error0);
2550
2551         /* need to be done after alloc mem, since it's self adjusting to amount
2552          * of memory available for RSS queues
2553          */
2554         rc = bnx2x_alloc_fp_mem(bp);
2555         if (rc) {
2556                 BNX2X_ERR("Unable to allocate memory for fps\n");
2557                 LOAD_ERROR_EXIT(bp, load_error0);
2558         }
2559
2560         /* request pf to initialize status blocks */
2561         if (IS_VF(bp)) {
2562                 rc = bnx2x_vfpf_init(bp);
2563                 if (rc)
2564                         LOAD_ERROR_EXIT(bp, load_error0);
2565         }
2566
2567         /* As long as bnx2x_alloc_mem() may possibly update
2568          * bp->num_queues, bnx2x_set_real_num_queues() should always
2569          * come after it. At this stage cnic queues are not counted.
2570          */
2571         rc = bnx2x_set_real_num_queues(bp, 0);
2572         if (rc) {
2573                 BNX2X_ERR("Unable to set real_num_queues\n");
2574                 LOAD_ERROR_EXIT(bp, load_error0);
2575         }
2576
2577         /* configure multi cos mappings in kernel.
2578          * this configuration may be overridden by a multi class queue
2579          * discipline or by a dcbx negotiation result.
2580          */
2581         bnx2x_setup_tc(bp->dev, bp->max_cos);
2582
2583         /* Add all NAPI objects */
2584         bnx2x_add_all_napi(bp);
2585         DP(NETIF_MSG_IFUP, "napi added\n");
2586         bnx2x_napi_enable(bp);
2587
2588         if (IS_PF(bp)) {
2589                 /* set pf load just before approaching the MCP */
2590                 bnx2x_set_pf_load(bp);
2591
2592                 /* if mcp exists send load request and analyze response */
2593                 if (!BP_NOMCP(bp)) {
2594                         /* attempt to load pf */
2595                         rc = bnx2x_nic_load_request(bp, &load_code);
2596                         if (rc)
2597                                 LOAD_ERROR_EXIT(bp, load_error1);
2598
2599                         /* what did mcp say? */
2600                         rc = bnx2x_nic_load_analyze_req(bp, load_code);
2601                         if (rc) {
2602                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2603                                 LOAD_ERROR_EXIT(bp, load_error2);
2604                         }
2605                 } else {
2606                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2607                 }
2608
2609                 /* mark pmf if applicable */
2610                 bnx2x_nic_load_pmf(bp, load_code);
2611
2612                 /* Init Function state controlling object */
2613                 bnx2x__init_func_obj(bp);
2614
2615                 /* Initialize HW */
2616                 rc = bnx2x_init_hw(bp, load_code);
2617                 if (rc) {
2618                         BNX2X_ERR("HW init failed, aborting\n");
2619                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2620                         LOAD_ERROR_EXIT(bp, load_error2);
2621                 }
2622         }
2623
2624         bnx2x_pre_irq_nic_init(bp);
2625
2626         /* Connect to IRQs */
2627         rc = bnx2x_setup_irqs(bp);
2628         if (rc) {
2629                 BNX2X_ERR("setup irqs failed\n");
2630                 if (IS_PF(bp))
2631                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2632                 LOAD_ERROR_EXIT(bp, load_error2);
2633         }
2634
2635         /* Init per-function objects */
2636         if (IS_PF(bp)) {
2637                 /* Setup NIC internals and enable interrupts */
2638                 bnx2x_post_irq_nic_init(bp, load_code);
2639
2640                 bnx2x_init_bp_objs(bp);
2641                 bnx2x_iov_nic_init(bp);
2642
2643                 /* Set AFEX default VLAN tag to an invalid value */
2644                 bp->afex_def_vlan_tag = -1;
2645                 bnx2x_nic_load_afex_dcc(bp, load_code);
2646                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2647                 rc = bnx2x_func_start(bp);
2648                 if (rc) {
2649                         BNX2X_ERR("Function start failed!\n");
2650                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2651
2652                         LOAD_ERROR_EXIT(bp, load_error3);
2653                 }
2654
2655                 /* Send LOAD_DONE command to MCP */
2656                 if (!BP_NOMCP(bp)) {
2657                         load_code = bnx2x_fw_command(bp,
2658                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2659                         if (!load_code) {
2660                                 BNX2X_ERR("MCP response failure, aborting\n");
2661                                 rc = -EBUSY;
2662                                 LOAD_ERROR_EXIT(bp, load_error3);
2663                         }
2664                 }
2665
2666                 /* initialize FW coalescing state machines in RAM */
2667                 bnx2x_update_coalesce(bp);
2668         }
2669
2670         /* setup the leading queue */
2671         rc = bnx2x_setup_leading(bp);
2672         if (rc) {
2673                 BNX2X_ERR("Setup leading failed!\n");
2674                 LOAD_ERROR_EXIT(bp, load_error3);
2675         }
2676
2677         /* set up the rest of the queues */
2678         for_each_nondefault_eth_queue(bp, i) {
2679                 if (IS_PF(bp))
2680                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2681                 else /* VF */
2682                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2683                 if (rc) {
2684                         BNX2X_ERR("Queue %d setup failed\n", i);
2685                         LOAD_ERROR_EXIT(bp, load_error3);
2686                 }
2687         }
2688
2689         /* setup rss */
2690         rc = bnx2x_init_rss(bp);
2691         if (rc) {
2692                 BNX2X_ERR("PF RSS init failed\n");
2693                 LOAD_ERROR_EXIT(bp, load_error3);
2694         }
2695
2696         /* Now when Clients are configured we are ready to work */
2697         bp->state = BNX2X_STATE_OPEN;
2698
2699         /* Configure a ucast MAC */
2700         if (IS_PF(bp))
2701                 rc = bnx2x_set_eth_mac(bp, true);
2702         else /* vf */
2703                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2704                                            true);
2705         if (rc) {
2706                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2707                 LOAD_ERROR_EXIT(bp, load_error3);
2708         }
2709
2710         if (IS_PF(bp) && bp->pending_max) {
2711                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2712                 bp->pending_max = 0;
2713         }
2714
2715         if (bp->port.pmf) {
2716                 rc = bnx2x_initial_phy_init(bp, load_mode);
2717                 if (rc)
2718                         LOAD_ERROR_EXIT(bp, load_error3);
2719         }
2720         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2721
2722         /* Start fast path */
2723
2724         /* Initialize Rx filter. */
2725         bnx2x_set_rx_mode_inner(bp);
2726
2727         /* Start the Tx */
2728         switch (load_mode) {
2729         case LOAD_NORMAL:
2730                 /* Tx queue should be only re-enabled */
2731                 netif_tx_wake_all_queues(bp->dev);
2732                 break;
2733
2734         case LOAD_OPEN:
2735                 netif_tx_start_all_queues(bp->dev);
2736                 smp_mb__after_clear_bit();
2737                 break;
2738
2739         case LOAD_DIAG:
2740         case LOAD_LOOPBACK_EXT:
2741                 bp->state = BNX2X_STATE_DIAG;
2742                 break;
2743
2744         default:
2745                 break;
2746         }
2747
2748         if (bp->port.pmf)
2749                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2750         else
2751                 bnx2x__link_status_update(bp);
2752
2753         /* start the timer */
2754         mod_timer(&bp->timer, jiffies + bp->current_interval);
2755
2756         if (CNIC_ENABLED(bp))
2757                 bnx2x_load_cnic(bp);
2758
2759         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2760                 /* mark driver is loaded in shmem2 */
2761                 u32 val;
2762                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2763                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2764                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2765                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2766         }
2767
2768         /* Wait for all pending SP commands to complete */
2769         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2770                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2771                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2772                 return -EBUSY;
2773         }
2774
2775         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2776         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2777                 bnx2x_dcbx_init(bp, false);
2778
2779         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2780
2781         return 0;
2782
2783 #ifndef BNX2X_STOP_ON_ERROR
2784 load_error3:
2785         if (IS_PF(bp)) {
2786                 bnx2x_int_disable_sync(bp, 1);
2787
2788                 /* Clean queueable objects */
2789                 bnx2x_squeeze_objects(bp);
2790         }
2791
2792         /* Free SKBs, SGEs, TPA pool and driver internals */
2793         bnx2x_free_skbs(bp);
2794         for_each_rx_queue(bp, i)
2795                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2796
2797         /* Release IRQs */
2798         bnx2x_free_irq(bp);
2799 load_error2:
2800         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2801                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2802                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2803         }
2804
2805         bp->port.pmf = 0;
2806 load_error1:
2807         bnx2x_napi_disable(bp);
2808         bnx2x_del_all_napi(bp);
2809
2810         /* clear pf_load status, as it was already set */
2811         if (IS_PF(bp))
2812                 bnx2x_clear_pf_load(bp);
2813 load_error0:
2814         bnx2x_free_fp_mem(bp);
2815         bnx2x_free_fw_stats_mem(bp);
2816         bnx2x_free_mem(bp);
2817
2818         return rc;
2819 #endif /* ! BNX2X_STOP_ON_ERROR */
2820 }
2821
2822 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2823 {
2824         u8 rc = 0, cos, i;
2825
2826         /* Wait until tx fastpath tasks complete */
2827         for_each_tx_queue(bp, i) {
2828                 struct bnx2x_fastpath *fp = &bp->fp[i];
2829
2830                 for_each_cos_in_tx_queue(fp, cos)
2831                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2832                 if (rc)
2833                         return rc;
2834         }
2835         return 0;
2836 }
2837
2838 /* must be called with rtnl_lock */
2839 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2840 {
2841         int i;
2842         bool global = false;
2843
2844         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2845
2846         /* mark driver is unloaded in shmem2 */
2847         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2848                 u32 val;
2849                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2850                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2851                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2852         }
2853
2854         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2855             (bp->state == BNX2X_STATE_CLOSED ||
2856              bp->state == BNX2X_STATE_ERROR)) {
2857                 /* We can get here if the driver has been unloaded
2858                  * during parity error recovery and is either waiting for a
2859                  * leader to complete or for other functions to unload and
2860                  * then ifdown has been issued. In this case we want to
2861                  * unload and let other functions to complete a recovery
2862                  * process.
2863                  */
2864                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2865                 bp->is_leader = 0;
2866                 bnx2x_release_leader_lock(bp);
2867                 smp_mb();
2868
2869                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2870                 BNX2X_ERR("Can't unload in closed or error state\n");
2871                 return -EINVAL;
2872         }
2873
2874         /* Nothing to do during unload if previous bnx2x_nic_load()
2875          * have not completed successfully - all resources are released.
2876          *
2877          * we can get here only after unsuccessful ndo_* callback, during which
2878          * dev->IFF_UP flag is still on.
2879          */
2880         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2881                 return 0;
2882
2883         /* It's important to set the bp->state to the value different from
2884          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2885          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2886          */
2887         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2888         smp_mb();
2889
2890         /* indicate to VFs that the PF is going down */
2891         bnx2x_iov_channel_down(bp);
2892
2893         if (CNIC_LOADED(bp))
2894                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2895
2896         /* Stop Tx */
2897         bnx2x_tx_disable(bp);
2898         netdev_reset_tc(bp->dev);
2899
2900         bp->rx_mode = BNX2X_RX_MODE_NONE;
2901
2902         del_timer_sync(&bp->timer);
2903
2904         if (IS_PF(bp)) {
2905                 /* Set ALWAYS_ALIVE bit in shmem */
2906                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2907                 bnx2x_drv_pulse(bp);
2908                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2909                 bnx2x_save_statistics(bp);
2910         }
2911
2912         /* wait till consumers catch up with producers in all queues */
2913         bnx2x_drain_tx_queues(bp);
2914
2915         /* if VF indicate to PF this function is going down (PF will delete sp
2916          * elements and clear initializations
2917          */
2918         if (IS_VF(bp))
2919                 bnx2x_vfpf_close_vf(bp);
2920         else if (unload_mode != UNLOAD_RECOVERY)
2921                 /* if this is a normal/close unload need to clean up chip*/
2922                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2923         else {
2924                 /* Send the UNLOAD_REQUEST to the MCP */
2925                 bnx2x_send_unload_req(bp, unload_mode);
2926
2927                 /* Prevent transactions to host from the functions on the
2928                  * engine that doesn't reset global blocks in case of global
2929                  * attention once global blocks are reset and gates are opened
2930                  * (the engine which leader will perform the recovery
2931                  * last).
2932                  */
2933                 if (!CHIP_IS_E1x(bp))
2934                         bnx2x_pf_disable(bp);
2935
2936                 /* Disable HW interrupts, NAPI */
2937                 bnx2x_netif_stop(bp, 1);
2938                 /* Delete all NAPI objects */
2939                 bnx2x_del_all_napi(bp);
2940                 if (CNIC_LOADED(bp))
2941                         bnx2x_del_all_napi_cnic(bp);
2942                 /* Release IRQs */
2943                 bnx2x_free_irq(bp);
2944
2945                 /* Report UNLOAD_DONE to MCP */
2946                 bnx2x_send_unload_done(bp, false);
2947         }
2948
2949         /*
2950          * At this stage no more interrupts will arrive so we may safely clean
2951          * the queueable objects here in case they failed to get cleaned so far.
2952          */
2953         if (IS_PF(bp))
2954                 bnx2x_squeeze_objects(bp);
2955
2956         /* There should be no more pending SP commands at this stage */
2957         bp->sp_state = 0;
2958
2959         bp->port.pmf = 0;
2960
2961         /* Free SKBs, SGEs, TPA pool and driver internals */
2962         bnx2x_free_skbs(bp);
2963         if (CNIC_LOADED(bp))
2964                 bnx2x_free_skbs_cnic(bp);
2965         for_each_rx_queue(bp, i)
2966                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2967
2968         bnx2x_free_fp_mem(bp);
2969         if (CNIC_LOADED(bp))
2970                 bnx2x_free_fp_mem_cnic(bp);
2971
2972         if (IS_PF(bp)) {
2973                 if (CNIC_LOADED(bp))
2974                         bnx2x_free_mem_cnic(bp);
2975         }
2976         bnx2x_free_mem(bp);
2977
2978         bp->state = BNX2X_STATE_CLOSED;
2979         bp->cnic_loaded = false;
2980
2981         /* Check if there are pending parity attentions. If there are - set
2982          * RECOVERY_IN_PROGRESS.
2983          */
2984         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
2985                 bnx2x_set_reset_in_progress(bp);
2986
2987                 /* Set RESET_IS_GLOBAL if needed */
2988                 if (global)
2989                         bnx2x_set_reset_global(bp);
2990         }
2991
2992         /* The last driver must disable a "close the gate" if there is no
2993          * parity attention or "process kill" pending.
2994          */
2995         if (IS_PF(bp) &&
2996             !bnx2x_clear_pf_load(bp) &&
2997             bnx2x_reset_is_done(bp, BP_PATH(bp)))
2998                 bnx2x_disable_close_the_gate(bp);
2999
3000         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3001
3002         return 0;
3003 }
3004
3005 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3006 {
3007         u16 pmcsr;
3008
3009         /* If there is no power capability, silently succeed */
3010         if (!bp->pdev->pm_cap) {
3011                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3012                 return 0;
3013         }
3014
3015         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3016
3017         switch (state) {
3018         case PCI_D0:
3019                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3020                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3021                                        PCI_PM_CTRL_PME_STATUS));
3022
3023                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3024                         /* delay required during transition out of D3hot */
3025                         msleep(20);
3026                 break;
3027
3028         case PCI_D3hot:
3029                 /* If there are other clients above don't
3030                    shut down the power */
3031                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3032                         return 0;
3033                 /* Don't shut down the power for emulation and FPGA */
3034                 if (CHIP_REV_IS_SLOW(bp))
3035                         return 0;
3036
3037                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3038                 pmcsr |= 3;
3039
3040                 if (bp->wol)
3041                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3042
3043                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3044                                       pmcsr);
3045
3046                 /* No more memory access after this point until
3047                 * device is brought back to D0.
3048                 */
3049                 break;
3050
3051         default:
3052                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3053                 return -EINVAL;
3054         }
3055         return 0;
3056 }
3057
3058 /*
3059  * net_device service functions
3060  */
3061 int bnx2x_poll(struct napi_struct *napi, int budget)
3062 {
3063         int work_done = 0;
3064         u8 cos;
3065         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3066                                                  napi);
3067         struct bnx2x *bp = fp->bp;
3068
3069         while (1) {
3070 #ifdef BNX2X_STOP_ON_ERROR
3071                 if (unlikely(bp->panic)) {
3072                         napi_complete(napi);
3073                         return 0;
3074                 }
3075 #endif
3076                 if (!bnx2x_fp_lock_napi(fp))
3077                         return work_done;
3078
3079                 for_each_cos_in_tx_queue(fp, cos)
3080                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3081                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3082
3083                 if (bnx2x_has_rx_work(fp)) {
3084                         work_done += bnx2x_rx_int(fp, budget - work_done);
3085
3086                         /* must not complete if we consumed full budget */
3087                         if (work_done >= budget) {
3088                                 bnx2x_fp_unlock_napi(fp);
3089                                 break;
3090                         }
3091                 }
3092
3093                 /* Fall out from the NAPI loop if needed */
3094                 if (!bnx2x_fp_unlock_napi(fp) &&
3095                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3096
3097                         /* No need to update SB for FCoE L2 ring as long as
3098                          * it's connected to the default SB and the SB
3099                          * has been updated when NAPI was scheduled.
3100                          */
3101                         if (IS_FCOE_FP(fp)) {
3102                                 napi_complete(napi);
3103                                 break;
3104                         }
3105                         bnx2x_update_fpsb_idx(fp);
3106                         /* bnx2x_has_rx_work() reads the status block,
3107                          * thus we need to ensure that status block indices
3108                          * have been actually read (bnx2x_update_fpsb_idx)
3109                          * prior to this check (bnx2x_has_rx_work) so that
3110                          * we won't write the "newer" value of the status block
3111                          * to IGU (if there was a DMA right after
3112                          * bnx2x_has_rx_work and if there is no rmb, the memory
3113                          * reading (bnx2x_update_fpsb_idx) may be postponed
3114                          * to right before bnx2x_ack_sb). In this case there
3115                          * will never be another interrupt until there is
3116                          * another update of the status block, while there
3117                          * is still unhandled work.
3118                          */
3119                         rmb();
3120
3121                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3122                                 napi_complete(napi);
3123                                 /* Re-enable interrupts */
3124                                 DP(NETIF_MSG_RX_STATUS,
3125                                    "Update index to %d\n", fp->fp_hc_idx);
3126                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3127                                              le16_to_cpu(fp->fp_hc_idx),
3128                                              IGU_INT_ENABLE, 1);
3129                                 break;
3130                         }
3131                 }
3132         }
3133
3134         return work_done;
3135 }
3136
3137 #ifdef CONFIG_NET_RX_BUSY_POLL
3138 /* must be called with local_bh_disable()d */
3139 int bnx2x_low_latency_recv(struct napi_struct *napi)
3140 {
3141         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3142                                                  napi);
3143         struct bnx2x *bp = fp->bp;
3144         int found = 0;
3145
3146         if ((bp->state == BNX2X_STATE_CLOSED) ||
3147             (bp->state == BNX2X_STATE_ERROR) ||
3148             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3149                 return LL_FLUSH_FAILED;
3150
3151         if (!bnx2x_fp_lock_poll(fp))
3152                 return LL_FLUSH_BUSY;
3153
3154         if (bnx2x_has_rx_work(fp))
3155                 found = bnx2x_rx_int(fp, 4);
3156
3157         bnx2x_fp_unlock_poll(fp);
3158
3159         return found;
3160 }
3161 #endif
3162
3163 /* we split the first BD into headers and data BDs
3164  * to ease the pain of our fellow microcode engineers
3165  * we use one mapping for both BDs
3166  */
3167 static u16 bnx2x_tx_split(struct bnx2x *bp,
3168                           struct bnx2x_fp_txdata *txdata,
3169                           struct sw_tx_bd *tx_buf,
3170                           struct eth_tx_start_bd **tx_bd, u16 hlen,
3171                           u16 bd_prod)
3172 {
3173         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3174         struct eth_tx_bd *d_tx_bd;
3175         dma_addr_t mapping;
3176         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3177
3178         /* first fix first BD */
3179         h_tx_bd->nbytes = cpu_to_le16(hlen);
3180
3181         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3182            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3183
3184         /* now get a new data BD
3185          * (after the pbd) and fill it */
3186         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3187         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3188
3189         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3190                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3191
3192         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3193         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3194         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3195
3196         /* this marks the BD as one that has no individual mapping */
3197         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3198
3199         DP(NETIF_MSG_TX_QUEUED,
3200            "TSO split data size is %d (%x:%x)\n",
3201            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3202
3203         /* update tx_bd */
3204         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3205
3206         return bd_prod;
3207 }
3208
3209 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3210 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3211 static __le16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
3212 {
3213         __sum16 tsum = (__force __sum16) csum;
3214
3215         if (fix > 0)
3216                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3217                                   csum_partial(t_header - fix, fix, 0)));
3218
3219         else if (fix < 0)
3220                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3221                                   csum_partial(t_header, -fix, 0)));
3222
3223         return bswab16(tsum);
3224 }
3225
3226 static u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3227 {
3228         u32 rc;
3229         __u8 prot = 0;
3230         __be16 protocol;
3231
3232         if (skb->ip_summed != CHECKSUM_PARTIAL)
3233                 return XMIT_PLAIN;
3234
3235         protocol = vlan_get_protocol(skb);
3236         if (protocol == htons(ETH_P_IPV6)) {
3237                 rc = XMIT_CSUM_V6;
3238                 prot = ipv6_hdr(skb)->nexthdr;
3239         } else {
3240                 rc = XMIT_CSUM_V4;
3241                 prot = ip_hdr(skb)->protocol;
3242         }
3243
3244         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3245                 if (inner_ip_hdr(skb)->version == 6) {
3246                         rc |= XMIT_CSUM_ENC_V6;
3247                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3248                                 rc |= XMIT_CSUM_TCP;
3249                 } else {
3250                         rc |= XMIT_CSUM_ENC_V4;
3251                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3252                                 rc |= XMIT_CSUM_TCP;
3253                 }
3254         }
3255         if (prot == IPPROTO_TCP)
3256                 rc |= XMIT_CSUM_TCP;
3257
3258         if (skb_is_gso(skb)) {
3259                 if (skb_is_gso_v6(skb)) {
3260                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3261                         if (rc & XMIT_CSUM_ENC)
3262                                 rc |= XMIT_GSO_ENC_V6;
3263                 } else {
3264                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3265                         if (rc & XMIT_CSUM_ENC)
3266                                 rc |= XMIT_GSO_ENC_V4;
3267                 }
3268         }
3269
3270         return rc;
3271 }
3272
3273 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3274 /* check if packet requires linearization (packet is too fragmented)
3275    no need to check fragmentation if page size > 8K (there will be no
3276    violation to FW restrictions) */
3277 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3278                              u32 xmit_type)
3279 {
3280         int to_copy = 0;
3281         int hlen = 0;
3282         int first_bd_sz = 0;
3283
3284         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3285         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3286
3287                 if (xmit_type & XMIT_GSO) {
3288                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3289                         /* Check if LSO packet needs to be copied:
3290                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3291                         int wnd_size = MAX_FETCH_BD - 3;
3292                         /* Number of windows to check */
3293                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3294                         int wnd_idx = 0;
3295                         int frag_idx = 0;
3296                         u32 wnd_sum = 0;
3297
3298                         /* Headers length */
3299                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3300                                 tcp_hdrlen(skb);
3301
3302                         /* Amount of data (w/o headers) on linear part of SKB*/
3303                         first_bd_sz = skb_headlen(skb) - hlen;
3304
3305                         wnd_sum  = first_bd_sz;
3306
3307                         /* Calculate the first sum - it's special */
3308                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3309                                 wnd_sum +=
3310                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3311
3312                         /* If there was data on linear skb data - check it */
3313                         if (first_bd_sz > 0) {
3314                                 if (unlikely(wnd_sum < lso_mss)) {
3315                                         to_copy = 1;
3316                                         goto exit_lbl;
3317                                 }
3318
3319                                 wnd_sum -= first_bd_sz;
3320                         }
3321
3322                         /* Others are easier: run through the frag list and
3323                            check all windows */
3324                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3325                                 wnd_sum +=
3326                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3327
3328                                 if (unlikely(wnd_sum < lso_mss)) {
3329                                         to_copy = 1;
3330                                         break;
3331                                 }
3332                                 wnd_sum -=
3333                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3334                         }
3335                 } else {
3336                         /* in non-LSO too fragmented packet should always
3337                            be linearized */
3338                         to_copy = 1;
3339                 }
3340         }
3341
3342 exit_lbl:
3343         if (unlikely(to_copy))
3344                 DP(NETIF_MSG_TX_QUEUED,
3345                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3346                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3347                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3348
3349         return to_copy;
3350 }
3351 #endif
3352
3353 static void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
3354                                  u32 xmit_type)
3355 {
3356         struct ipv6hdr *ipv6;
3357
3358         *parsing_data |= (skb_shinfo(skb)->gso_size <<
3359                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
3360                               ETH_TX_PARSE_BD_E2_LSO_MSS;
3361
3362         if (xmit_type & XMIT_GSO_ENC_V6)
3363                 ipv6 = inner_ipv6_hdr(skb);
3364         else if (xmit_type & XMIT_GSO_V6)
3365                 ipv6 = ipv6_hdr(skb);
3366         else
3367                 ipv6 = NULL;
3368
3369         if (ipv6 && ipv6->nexthdr == NEXTHDR_IPV6)
3370                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
3371 }
3372
3373 /**
3374  * bnx2x_set_pbd_gso - update PBD in GSO case.
3375  *
3376  * @skb:        packet skb
3377  * @pbd:        parse BD
3378  * @xmit_type:  xmit flags
3379  */
3380 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3381                               struct eth_tx_parse_bd_e1x *pbd,
3382                               struct eth_tx_start_bd *tx_start_bd,
3383                               u32 xmit_type)
3384 {
3385         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3386         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3387         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3388
3389         if (xmit_type & XMIT_GSO_V4) {
3390                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3391                 pbd->tcp_pseudo_csum =
3392                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3393                                                    ip_hdr(skb)->daddr,
3394                                                    0, IPPROTO_TCP, 0));
3395
3396                 /* GSO on 57710/57711 needs FW to calculate IP checksum */
3397                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IP_CSUM;
3398         } else {
3399                 pbd->tcp_pseudo_csum =
3400                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3401                                                  &ipv6_hdr(skb)->daddr,
3402                                                  0, IPPROTO_TCP, 0));
3403         }
3404
3405         pbd->global_data |=
3406                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3407 }
3408
3409 /**
3410  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3411  *
3412  * @bp:                 driver handle
3413  * @skb:                packet skb
3414  * @parsing_data:       data to be updated
3415  * @xmit_type:          xmit flags
3416  *
3417  * 57712/578xx related, when skb has encapsulation
3418  */
3419 static u8 bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3420                                  u32 *parsing_data, u32 xmit_type)
3421 {
3422         *parsing_data |=
3423                 ((((u8 *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3424                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3425                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3426
3427         if (xmit_type & XMIT_CSUM_TCP) {
3428                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3429                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3430                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3431
3432                 return skb_inner_transport_header(skb) +
3433                         inner_tcp_hdrlen(skb) - skb->data;
3434         }
3435
3436         /* We support checksum offload for TCP and UDP only.
3437          * No need to pass the UDP header length - it's a constant.
3438          */
3439         return skb_inner_transport_header(skb) +
3440                 sizeof(struct udphdr) - skb->data;
3441 }
3442
3443 /**
3444  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3445  *
3446  * @bp:                 driver handle
3447  * @skb:                packet skb
3448  * @parsing_data:       data to be updated
3449  * @xmit_type:          xmit flags
3450  *
3451  * 57712/578xx related
3452  */
3453 static u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3454                                 u32 *parsing_data, u32 xmit_type)
3455 {
3456         *parsing_data |=
3457                 ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3458                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3459                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3460
3461         if (xmit_type & XMIT_CSUM_TCP) {
3462                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3463                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3464                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3465
3466                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3467         }
3468         /* We support checksum offload for TCP and UDP only.
3469          * No need to pass the UDP header length - it's a constant.
3470          */
3471         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3472 }
3473
3474 /* set FW indication according to inner or outer protocols if tunneled */
3475 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3476                                struct eth_tx_start_bd *tx_start_bd,
3477                                u32 xmit_type)
3478 {
3479         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3480
3481         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3482                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3483
3484         if (!(xmit_type & XMIT_CSUM_TCP))
3485                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3486 }
3487
3488 /**
3489  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3490  *
3491  * @bp:         driver handle
3492  * @skb:        packet skb
3493  * @pbd:        parse BD to be updated
3494  * @xmit_type:  xmit flags
3495  */
3496 static u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3497                              struct eth_tx_parse_bd_e1x *pbd,
3498                              u32 xmit_type)
3499 {
3500         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3501
3502         /* for now NS flag is not used in Linux */
3503         pbd->global_data =
3504                 cpu_to_le16(hlen |
3505                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3506                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3507
3508         pbd->ip_hlen_w = (skb_transport_header(skb) -
3509                         skb_network_header(skb)) >> 1;
3510
3511         hlen += pbd->ip_hlen_w;
3512
3513         /* We support checksum offload for TCP and UDP only */
3514         if (xmit_type & XMIT_CSUM_TCP)
3515                 hlen += tcp_hdrlen(skb) / 2;
3516         else
3517                 hlen += sizeof(struct udphdr) / 2;
3518
3519         pbd->total_hlen_w = cpu_to_le16(hlen);
3520         hlen = hlen*2;
3521
3522         if (xmit_type & XMIT_CSUM_TCP) {
3523                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3524
3525         } else {
3526                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3527
3528                 DP(NETIF_MSG_TX_QUEUED,
3529                    "hlen %d  fix %d  csum before fix %x\n",
3530                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3531
3532                 /* HW bug: fixup the CSUM */
3533                 pbd->tcp_pseudo_csum =
3534                         bnx2x_csum_fix(skb_transport_header(skb),
3535                                        SKB_CS(skb), fix);
3536
3537                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3538                    pbd->tcp_pseudo_csum);
3539         }
3540
3541         return hlen;
3542 }
3543
3544 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3545                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3546                                       struct eth_tx_parse_2nd_bd *pbd2,
3547                                       u16 *global_data,
3548                                       u32 xmit_type)
3549 {
3550         u16 hlen_w = 0;
3551         u8 outerip_off, outerip_len = 0;
3552
3553         /* from outer IP to transport */
3554         hlen_w = (skb_inner_transport_header(skb) -
3555                   skb_network_header(skb)) >> 1;
3556
3557         /* transport len */
3558         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3559
3560         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3561
3562         /* outer IP header info */
3563         if (xmit_type & XMIT_CSUM_V4) {
3564                 struct iphdr *iph = ip_hdr(skb);
3565                 u32 csum = (__force u32)(~iph->check) -
3566                            (__force u32)iph->tot_len -
3567                            (__force u32)iph->frag_off;
3568
3569                 pbd2->fw_ip_csum_wo_len_flags_frag =
3570                         bswab16(csum_fold((__force __wsum)csum));
3571         } else {
3572                 pbd2->fw_ip_hdr_to_payload_w =
3573                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3574         }
3575
3576         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);
3577
3578         pbd2->tcp_flags = pbd_tcp_flags(inner_tcp_hdr(skb));
3579
3580         if (xmit_type & XMIT_GSO_V4) {
3581                 pbd2->hw_ip_id = bswab16(inner_ip_hdr(skb)->id);
3582
3583                 pbd_e2->data.tunnel_data.pseudo_csum =
3584                         bswab16(~csum_tcpudp_magic(
3585                                         inner_ip_hdr(skb)->saddr,
3586                                         inner_ip_hdr(skb)->daddr,
3587                                         0, IPPROTO_TCP, 0));
3588
3589                 outerip_len = ip_hdr(skb)->ihl << 1;
3590         } else {
3591                 pbd_e2->data.tunnel_data.pseudo_csum =
3592                         bswab16(~csum_ipv6_magic(
3593                                         &inner_ipv6_hdr(skb)->saddr,
3594                                         &inner_ipv6_hdr(skb)->daddr,
3595                                         0, IPPROTO_TCP, 0));
3596         }
3597
3598         outerip_off = (skb_network_header(skb) - skb->data) >> 1;
3599
3600         *global_data |=
3601                 outerip_off |
3602                 (!!(xmit_type & XMIT_CSUM_V6) <<
3603                         ETH_TX_PARSE_2ND_BD_IP_HDR_TYPE_OUTER_SHIFT) |
3604                 (outerip_len <<
3605                         ETH_TX_PARSE_2ND_BD_IP_HDR_LEN_OUTER_W_SHIFT) |
3606                 ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3607                         ETH_TX_PARSE_2ND_BD_LLC_SNAP_EN_SHIFT);
3608
3609         if (ip_hdr(skb)->protocol == IPPROTO_UDP) {
3610                 SET_FLAG(*global_data, ETH_TX_PARSE_2ND_BD_TUNNEL_UDP_EXIST, 1);
3611                 pbd2->tunnel_udp_hdr_start_w = skb_transport_offset(skb) >> 1;
3612         }
3613 }
3614
3615 /* called with netif_tx_lock
3616  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3617  * netif_wake_queue()
3618  */
3619 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3620 {
3621         struct bnx2x *bp = netdev_priv(dev);
3622
3623         struct netdev_queue *txq;
3624         struct bnx2x_fp_txdata *txdata;
3625         struct sw_tx_bd *tx_buf;
3626         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3627         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3628         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3629         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3630         struct eth_tx_parse_2nd_bd *pbd2 = NULL;
3631         u32 pbd_e2_parsing_data = 0;
3632         u16 pkt_prod, bd_prod;
3633         int nbd, txq_index;
3634         dma_addr_t mapping;
3635         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3636         int i;
3637         u8 hlen = 0;
3638         __le16 pkt_size = 0;
3639         struct ethhdr *eth;
3640         u8 mac_type = UNICAST_ADDRESS;
3641
3642 #ifdef BNX2X_STOP_ON_ERROR
3643         if (unlikely(bp->panic))
3644                 return NETDEV_TX_BUSY;
3645 #endif
3646
3647         txq_index = skb_get_queue_mapping(skb);
3648         txq = netdev_get_tx_queue(dev, txq_index);
3649
3650         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3651
3652         txdata = &bp->bnx2x_txq[txq_index];
3653
3654         /* enable this debug print to view the transmission queue being used
3655         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3656            txq_index, fp_index, txdata_index); */
3657
3658         /* enable this debug print to view the transmission details
3659         DP(NETIF_MSG_TX_QUEUED,
3660            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3661            txdata->cid, fp_index, txdata_index, txdata, fp); */
3662
3663         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3664                         skb_shinfo(skb)->nr_frags +
3665                         BDS_PER_TX_PKT +
3666                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3667                 /* Handle special storage cases separately */
3668                 if (txdata->tx_ring_size == 0) {
3669                         struct bnx2x_eth_q_stats *q_stats =
3670                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3671                         q_stats->driver_filtered_tx_pkt++;
3672                         dev_kfree_skb(skb);
3673                         return NETDEV_TX_OK;
3674                 }
3675                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3676                 netif_tx_stop_queue(txq);
3677                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3678
3679                 return NETDEV_TX_BUSY;
3680         }
3681
3682         DP(NETIF_MSG_TX_QUEUED,
3683            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x len %d\n",
3684            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3685            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type,
3686            skb->len);
3687
3688         eth = (struct ethhdr *)skb->data;
3689
3690         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3691         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3692                 if (is_broadcast_ether_addr(eth->h_dest))
3693                         mac_type = BROADCAST_ADDRESS;
3694                 else
3695                         mac_type = MULTICAST_ADDRESS;
3696         }
3697
3698 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - BDS_PER_TX_PKT)
3699         /* First, check if we need to linearize the skb (due to FW
3700            restrictions). No need to check fragmentation if page size > 8K
3701            (there will be no violation to FW restrictions) */
3702         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3703                 /* Statistics of linearization */
3704                 bp->lin_cnt++;
3705                 if (skb_linearize(skb) != 0) {
3706                         DP(NETIF_MSG_TX_QUEUED,
3707                            "SKB linearization failed - silently dropping this SKB\n");
3708                         dev_kfree_skb_any(skb);
3709                         return NETDEV_TX_OK;
3710                 }
3711         }
3712 #endif
3713         /* Map skb linear data for DMA */
3714         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3715                                  skb_headlen(skb), DMA_TO_DEVICE);
3716         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3717                 DP(NETIF_MSG_TX_QUEUED,
3718                    "SKB mapping failed - silently dropping this SKB\n");
3719                 dev_kfree_skb_any(skb);
3720                 return NETDEV_TX_OK;
3721         }
3722         /*
3723         Please read carefully. First we use one BD which we mark as start,
3724         then we have a parsing info BD (used for TSO or xsum),
3725         and only then we have the rest of the TSO BDs.
3726         (don't forget to mark the last one as last,
3727         and to unmap only AFTER you write to the BD ...)
3728         And above all, all pdb sizes are in words - NOT DWORDS!
3729         */
3730
3731         /* get current pkt produced now - advance it just before sending packet
3732          * since mapping of pages may fail and cause packet to be dropped
3733          */
3734         pkt_prod = txdata->tx_pkt_prod;
3735         bd_prod = TX_BD(txdata->tx_bd_prod);
3736
3737         /* get a tx_buf and first BD
3738          * tx_start_bd may be changed during SPLIT,
3739          * but first_bd will always stay first
3740          */
3741         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3742         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3743         first_bd = tx_start_bd;
3744
3745         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3746
3747         /* header nbd: indirectly zero other flags! */
3748         tx_start_bd->general_data = 1 << ETH_TX_START_BD_HDR_NBDS_SHIFT;
3749
3750         /* remember the first BD of the packet */
3751         tx_buf->first_bd = txdata->tx_bd_prod;
3752         tx_buf->skb = skb;
3753         tx_buf->flags = 0;
3754
3755         DP(NETIF_MSG_TX_QUEUED,
3756            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3757            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3758
3759         if (vlan_tx_tag_present(skb)) {
3760                 tx_start_bd->vlan_or_ethertype =
3761                     cpu_to_le16(vlan_tx_tag_get(skb));
3762                 tx_start_bd->bd_flags.as_bitfield |=
3763                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3764         } else {
3765                 /* when transmitting in a vf, start bd must hold the ethertype
3766                  * for fw to enforce it
3767                  */
3768                 if (IS_VF(bp))
3769                         tx_start_bd->vlan_or_ethertype =
3770                                 cpu_to_le16(ntohs(eth->h_proto));
3771                 else
3772                         /* used by FW for packet accounting */
3773                         tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3774         }
3775
3776         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3777
3778         /* turn on parsing and get a BD */
3779         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3780
3781         if (xmit_type & XMIT_CSUM)
3782                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3783
3784         if (!CHIP_IS_E1x(bp)) {
3785                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3786                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3787
3788                 if (xmit_type & XMIT_CSUM_ENC) {
3789                         u16 global_data = 0;
3790
3791                         /* Set PBD in enc checksum offload case */
3792                         hlen = bnx2x_set_pbd_csum_enc(bp, skb,
3793                                                       &pbd_e2_parsing_data,
3794                                                       xmit_type);
3795
3796                         /* turn on 2nd parsing and get a BD */
3797                         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3798
3799                         pbd2 = &txdata->tx_desc_ring[bd_prod].parse_2nd_bd;
3800
3801                         memset(pbd2, 0, sizeof(*pbd2));
3802
3803                         pbd_e2->data.tunnel_data.ip_hdr_start_inner_w =
3804                                 (skb_inner_network_header(skb) -
3805                                  skb->data) >> 1;
3806
3807                         if (xmit_type & XMIT_GSO_ENC)
3808                                 bnx2x_update_pbds_gso_enc(skb, pbd_e2, pbd2,
3809                                                           &global_data,
3810                                                           xmit_type);
3811
3812                         pbd2->global_data = cpu_to_le16(global_data);
3813
3814                         /* add addition parse BD indication to start BD */
3815                         SET_FLAG(tx_start_bd->general_data,
3816                                  ETH_TX_START_BD_PARSE_NBDS, 1);
3817                         /* set encapsulation flag in start BD */
3818                         SET_FLAG(tx_start_bd->general_data,
3819                                  ETH_TX_START_BD_TUNNEL_EXIST, 1);
3820                         nbd++;
3821                 } else if (xmit_type & XMIT_CSUM) {
3822                         /* Set PBD in checksum offload case w/o encapsulation */
3823                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3824                                                      &pbd_e2_parsing_data,
3825                                                      xmit_type);
3826                 }
3827
3828                 /* Add the macs to the parsing BD this is a vf */
3829                 if (IS_VF(bp)) {
3830                         /* override GRE parameters in BD */
3831                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.src_hi,
3832                                               &pbd_e2->data.mac_addr.src_mid,
3833                                               &pbd_e2->data.mac_addr.src_lo,
3834                                               eth->h_source);
3835
3836                         bnx2x_set_fw_mac_addr(&pbd_e2->data.mac_addr.dst_hi,
3837                                               &pbd_e2->data.mac_addr.dst_mid,
3838                                               &pbd_e2->data.mac_addr.dst_lo,
3839                                               eth->h_dest);
3840                 }
3841
3842                 SET_FLAG(pbd_e2_parsing_data,
3843                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3844         } else {
3845                 u16 global_data = 0;
3846                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3847                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3848                 /* Set PBD in checksum offload case */
3849                 if (xmit_type & XMIT_CSUM)
3850                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3851
3852                 SET_FLAG(global_data,
3853                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3854                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3855         }
3856
3857         /* Setup the data pointer of the first BD of the packet */
3858         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3859         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3860         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3861         pkt_size = tx_start_bd->nbytes;
3862
3863         DP(NETIF_MSG_TX_QUEUED,
3864            "first bd @%p  addr (%x:%x)  nbytes %d  flags %x  vlan %x\n",
3865            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3866            le16_to_cpu(tx_start_bd->nbytes),
3867            tx_start_bd->bd_flags.as_bitfield,
3868            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3869
3870         if (xmit_type & XMIT_GSO) {
3871
3872                 DP(NETIF_MSG_TX_QUEUED,
3873                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3874                    skb->len, hlen, skb_headlen(skb),
3875                    skb_shinfo(skb)->gso_size);
3876
3877                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3878
3879                 if (unlikely(skb_headlen(skb) > hlen)) {
3880                         nbd++;
3881                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3882                                                  &tx_start_bd, hlen,
3883                                                  bd_prod);
3884                 }
3885                 if (!CHIP_IS_E1x(bp))
3886                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3887                                              xmit_type);
3888                 else
3889                         bnx2x_set_pbd_gso(skb, pbd_e1x, first_bd, xmit_type);
3890         }
3891
3892         /* Set the PBD's parsing_data field if not zero
3893          * (for the chips newer than 57711).
3894          */
3895         if (pbd_e2_parsing_data)
3896                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3897
3898         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3899
3900         /* Handle fragmented skb */
3901         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3902                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3903
3904                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3905                                            skb_frag_size(frag), DMA_TO_DEVICE);
3906                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3907                         unsigned int pkts_compl = 0, bytes_compl = 0;
3908
3909                         DP(NETIF_MSG_TX_QUEUED,
3910                            "Unable to map page - dropping packet...\n");
3911
3912                         /* we need unmap all buffers already mapped
3913                          * for this SKB;
3914                          * first_bd->nbd need to be properly updated
3915                          * before call to bnx2x_free_tx_pkt
3916                          */
3917                         first_bd->nbd = cpu_to_le16(nbd);
3918                         bnx2x_free_tx_pkt(bp, txdata,
3919                                           TX_BD(txdata->tx_pkt_prod),
3920                                           &pkts_compl, &bytes_compl);
3921                         return NETDEV_TX_OK;
3922                 }
3923
3924                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3925                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3926                 if (total_pkt_bd == NULL)
3927                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3928
3929                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3930                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3931                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3932                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3933                 nbd++;
3934
3935                 DP(NETIF_MSG_TX_QUEUED,
3936                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3937                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3938                    le16_to_cpu(tx_data_bd->nbytes));
3939         }
3940
3941         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3942
3943         /* update with actual num BDs */
3944         first_bd->nbd = cpu_to_le16(nbd);
3945
3946         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3947
3948         /* now send a tx doorbell, counting the next BD
3949          * if the packet contains or ends with it
3950          */
3951         if (TX_BD_POFF(bd_prod) < nbd)
3952                 nbd++;
3953
3954         /* total_pkt_bytes should be set on the first data BD if
3955          * it's not an LSO packet and there is more than one
3956          * data BD. In this case pkt_size is limited by an MTU value.
3957          * However we prefer to set it for an LSO packet (while we don't
3958          * have to) in order to save some CPU cycles in a none-LSO
3959          * case, when we much more care about them.
3960          */
3961         if (total_pkt_bd != NULL)
3962                 total_pkt_bd->total_pkt_bytes = pkt_size;
3963
3964         if (pbd_e1x)
3965                 DP(NETIF_MSG_TX_QUEUED,
3966                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3967                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3968                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3969                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3970                     le16_to_cpu(pbd_e1x->total_hlen_w));
3971         if (pbd_e2)
3972                 DP(NETIF_MSG_TX_QUEUED,
3973                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3974                    pbd_e2,
3975                    pbd_e2->data.mac_addr.dst_hi,
3976                    pbd_e2->data.mac_addr.dst_mid,
3977                    pbd_e2->data.mac_addr.dst_lo,
3978                    pbd_e2->data.mac_addr.src_hi,
3979                    pbd_e2->data.mac_addr.src_mid,
3980                    pbd_e2->data.mac_addr.src_lo,
3981                    pbd_e2->parsing_data);
3982         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3983
3984         netdev_tx_sent_queue(txq, skb->len);
3985
3986         skb_tx_timestamp(skb);
3987
3988         txdata->tx_pkt_prod++;
3989         /*
3990          * Make sure that the BD data is updated before updating the producer
3991          * since FW might read the BD right after the producer is updated.
3992          * This is only applicable for weak-ordered memory model archs such
3993          * as IA-64. The following barrier is also mandatory since FW will
3994          * assumes packets must have BDs.
3995          */
3996         wmb();
3997
3998         txdata->tx_db.data.prod += nbd;
3999         barrier();
4000
4001         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
4002
4003         mmiowb();
4004
4005         txdata->tx_bd_prod += nbd;
4006
4007         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
4008                 netif_tx_stop_queue(txq);
4009
4010                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
4011                  * ordering of set_bit() in netif_tx_stop_queue() and read of
4012                  * fp->bd_tx_cons */
4013                 smp_mb();
4014
4015                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
4016                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
4017                         netif_tx_wake_queue(txq);
4018         }
4019         txdata->tx_pkt++;
4020
4021         return NETDEV_TX_OK;
4022 }
4023
4024 /**
4025  * bnx2x_setup_tc - routine to configure net_device for multi tc
4026  *
4027  * @netdev: net device to configure
4028  * @tc: number of traffic classes to enable
4029  *
4030  * callback connected to the ndo_setup_tc function pointer
4031  */
4032 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
4033 {
4034         int cos, prio, count, offset;
4035         struct bnx2x *bp = netdev_priv(dev);
4036
4037         /* setup tc must be called under rtnl lock */
4038         ASSERT_RTNL();
4039
4040         /* no traffic classes requested. Aborting */
4041         if (!num_tc) {
4042                 netdev_reset_tc(dev);
4043                 return 0;
4044         }
4045
4046         /* requested to support too many traffic classes */
4047         if (num_tc > bp->max_cos) {
4048                 BNX2X_ERR("support for too many traffic classes requested: %d. Max supported is %d\n",
4049                           num_tc, bp->max_cos);
4050                 return -EINVAL;
4051         }
4052
4053         /* declare amount of supported traffic classes */
4054         if (netdev_set_num_tc(dev, num_tc)) {
4055                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
4056                 return -EINVAL;
4057         }
4058
4059         /* configure priority to traffic class mapping */
4060         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
4061                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
4062                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4063                    "mapping priority %d to tc %d\n",
4064                    prio, bp->prio_to_cos[prio]);
4065         }
4066
4067         /* Use this configuration to differentiate tc0 from other COSes
4068            This can be used for ets or pfc, and save the effort of setting
4069            up a multio class queue disc or negotiating DCBX with a switch
4070         netdev_set_prio_tc_map(dev, 0, 0);
4071         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
4072         for (prio = 1; prio < 16; prio++) {
4073                 netdev_set_prio_tc_map(dev, prio, 1);
4074                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
4075         } */
4076
4077         /* configure traffic class to transmission queue mapping */
4078         for (cos = 0; cos < bp->max_cos; cos++) {
4079                 count = BNX2X_NUM_ETH_QUEUES(bp);
4080                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
4081                 netdev_set_tc_queue(dev, cos, count, offset);
4082                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
4083                    "mapping tc %d to offset %d count %d\n",
4084                    cos, offset, count);
4085         }
4086
4087         return 0;
4088 }
4089
4090 /* called with rtnl_lock */
4091 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
4092 {
4093         struct sockaddr *addr = p;
4094         struct bnx2x *bp = netdev_priv(dev);
4095         int rc = 0;
4096
4097         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
4098                 BNX2X_ERR("Requested MAC address is not valid\n");
4099                 return -EINVAL;
4100         }
4101
4102         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
4103             !is_zero_ether_addr(addr->sa_data)) {
4104                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
4105                 return -EINVAL;
4106         }
4107
4108         if (netif_running(dev))  {
4109                 rc = bnx2x_set_eth_mac(bp, false);
4110                 if (rc)
4111                         return rc;
4112         }
4113
4114         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4115
4116         if (netif_running(dev))
4117                 rc = bnx2x_set_eth_mac(bp, true);
4118
4119         return rc;
4120 }
4121
4122 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
4123 {
4124         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
4125         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
4126         u8 cos;
4127
4128         /* Common */
4129
4130         if (IS_FCOE_IDX(fp_index)) {
4131                 memset(sb, 0, sizeof(union host_hc_status_block));
4132                 fp->status_blk_mapping = 0;
4133         } else {
4134                 /* status blocks */
4135                 if (!CHIP_IS_E1x(bp))
4136                         BNX2X_PCI_FREE(sb->e2_sb,
4137                                        bnx2x_fp(bp, fp_index,
4138                                                 status_blk_mapping),
4139                                        sizeof(struct host_hc_status_block_e2));
4140                 else
4141                         BNX2X_PCI_FREE(sb->e1x_sb,
4142                                        bnx2x_fp(bp, fp_index,
4143                                                 status_blk_mapping),
4144                                        sizeof(struct host_hc_status_block_e1x));
4145         }
4146
4147         /* Rx */
4148         if (!skip_rx_queue(bp, fp_index)) {
4149                 bnx2x_free_rx_bds(fp);
4150
4151                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4152                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
4153                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
4154                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
4155                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
4156
4157                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
4158                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
4159                                sizeof(struct eth_fast_path_rx_cqe) *
4160                                NUM_RCQ_BD);
4161
4162                 /* SGE ring */
4163                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
4164                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
4165                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
4166                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4167         }
4168
4169         /* Tx */
4170         if (!skip_tx_queue(bp, fp_index)) {
4171                 /* fastpath tx rings: tx_buf tx_desc */
4172                 for_each_cos_in_tx_queue(fp, cos) {
4173                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4174
4175                         DP(NETIF_MSG_IFDOWN,
4176                            "freeing tx memory of fp %d cos %d cid %d\n",
4177                            fp_index, cos, txdata->cid);
4178
4179                         BNX2X_FREE(txdata->tx_buf_ring);
4180                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
4181                                 txdata->tx_desc_mapping,
4182                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4183                 }
4184         }
4185         /* end of fastpath */
4186 }
4187
4188 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
4189 {
4190         int i;
4191         for_each_cnic_queue(bp, i)
4192                 bnx2x_free_fp_mem_at(bp, i);
4193 }
4194
4195 void bnx2x_free_fp_mem(struct bnx2x *bp)
4196 {
4197         int i;
4198         for_each_eth_queue(bp, i)
4199                 bnx2x_free_fp_mem_at(bp, i);
4200 }
4201
4202 static void set_sb_shortcuts(struct bnx2x *bp, int index)
4203 {
4204         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
4205         if (!CHIP_IS_E1x(bp)) {
4206                 bnx2x_fp(bp, index, sb_index_values) =
4207                         (__le16 *)status_blk.e2_sb->sb.index_values;
4208                 bnx2x_fp(bp, index, sb_running_index) =
4209                         (__le16 *)status_blk.e2_sb->sb.running_index;
4210         } else {
4211                 bnx2x_fp(bp, index, sb_index_values) =
4212                         (__le16 *)status_blk.e1x_sb->sb.index_values;
4213                 bnx2x_fp(bp, index, sb_running_index) =
4214                         (__le16 *)status_blk.e1x_sb->sb.running_index;
4215         }
4216 }
4217
4218 /* Returns the number of actually allocated BDs */
4219 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
4220                               int rx_ring_size)
4221 {
4222         struct bnx2x *bp = fp->bp;
4223         u16 ring_prod, cqe_ring_prod;
4224         int i, failure_cnt = 0;
4225
4226         fp->rx_comp_cons = 0;
4227         cqe_ring_prod = ring_prod = 0;
4228
4229         /* This routine is called only during fo init so
4230          * fp->eth_q_stats.rx_skb_alloc_failed = 0
4231          */
4232         for (i = 0; i < rx_ring_size; i++) {
4233                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod, GFP_KERNEL) < 0) {
4234                         failure_cnt++;
4235                         continue;
4236                 }
4237                 ring_prod = NEXT_RX_IDX(ring_prod);
4238                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
4239                 WARN_ON(ring_prod <= (i - failure_cnt));
4240         }
4241
4242         if (failure_cnt)
4243                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
4244                           i - failure_cnt, fp->index);
4245
4246         fp->rx_bd_prod = ring_prod;
4247         /* Limit the CQE producer by the CQE ring size */
4248         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
4249                                cqe_ring_prod);
4250         fp->rx_pkt = fp->rx_calls = 0;
4251
4252         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
4253
4254         return i - failure_cnt;
4255 }
4256
4257 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
4258 {
4259         int i;
4260
4261         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
4262                 struct eth_rx_cqe_next_page *nextpg;
4263
4264                 nextpg = (struct eth_rx_cqe_next_page *)
4265                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
4266                 nextpg->addr_hi =
4267                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
4268                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4269                 nextpg->addr_lo =
4270                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
4271                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
4272         }
4273 }
4274
4275 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
4276 {
4277         union host_hc_status_block *sb;
4278         struct bnx2x_fastpath *fp = &bp->fp[index];
4279         int ring_size = 0;
4280         u8 cos;
4281         int rx_ring_size = 0;
4282
4283         if (!bp->rx_ring_size &&
4284             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
4285                 rx_ring_size = MIN_RX_SIZE_NONTPA;
4286                 bp->rx_ring_size = rx_ring_size;
4287         } else if (!bp->rx_ring_size) {
4288                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
4289
4290                 if (CHIP_IS_E3(bp)) {
4291                         u32 cfg = SHMEM_RD(bp,
4292                                            dev_info.port_hw_config[BP_PORT(bp)].
4293                                            default_cfg);
4294
4295                         /* Decrease ring size for 1G functions */
4296                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
4297                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
4298                                 rx_ring_size /= 10;
4299                 }
4300
4301                 /* allocate at least number of buffers required by FW */
4302                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
4303                                      MIN_RX_SIZE_TPA, rx_ring_size);
4304
4305                 bp->rx_ring_size = rx_ring_size;
4306         } else /* if rx_ring_size specified - use it */
4307                 rx_ring_size = bp->rx_ring_size;
4308
4309         DP(BNX2X_MSG_SP, "calculated rx_ring_size %d\n", rx_ring_size);
4310
4311         /* Common */
4312         sb = &bnx2x_fp(bp, index, status_blk);
4313
4314         if (!IS_FCOE_IDX(index)) {
4315                 /* status blocks */
4316                 if (!CHIP_IS_E1x(bp))
4317                         BNX2X_PCI_ALLOC(sb->e2_sb,
4318                                 &bnx2x_fp(bp, index, status_blk_mapping),
4319                                 sizeof(struct host_hc_status_block_e2));
4320                 else
4321                         BNX2X_PCI_ALLOC(sb->e1x_sb,
4322                                 &bnx2x_fp(bp, index, status_blk_mapping),
4323                             sizeof(struct host_hc_status_block_e1x));
4324         }
4325
4326         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
4327          * set shortcuts for it.
4328          */
4329         if (!IS_FCOE_IDX(index))
4330                 set_sb_shortcuts(bp, index);
4331
4332         /* Tx */
4333         if (!skip_tx_queue(bp, index)) {
4334                 /* fastpath tx rings: tx_buf tx_desc */
4335                 for_each_cos_in_tx_queue(fp, cos) {
4336                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
4337
4338                         DP(NETIF_MSG_IFUP,
4339                            "allocating tx memory of fp %d cos %d\n",
4340                            index, cos);
4341
4342                         BNX2X_ALLOC(txdata->tx_buf_ring,
4343                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
4344                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
4345                                 &txdata->tx_desc_mapping,
4346                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
4347                 }
4348         }
4349
4350         /* Rx */
4351         if (!skip_rx_queue(bp, index)) {
4352                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
4353                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
4354                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
4355                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
4356                                 &bnx2x_fp(bp, index, rx_desc_mapping),
4357                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
4358
4359                 /* Seed all CQEs by 1s */
4360                 BNX2X_PCI_FALLOC(bnx2x_fp(bp, index, rx_comp_ring),
4361                                  &bnx2x_fp(bp, index, rx_comp_mapping),
4362                                  sizeof(struct eth_fast_path_rx_cqe) *
4363                                  NUM_RCQ_BD);
4364
4365                 /* SGE ring */
4366                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
4367                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
4368                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
4369                                 &bnx2x_fp(bp, index, rx_sge_mapping),
4370                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
4371                 /* RX BD ring */
4372                 bnx2x_set_next_page_rx_bd(fp);
4373
4374                 /* CQ ring */
4375                 bnx2x_set_next_page_rx_cq(fp);
4376
4377                 /* BDs */
4378                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
4379                 if (ring_size < rx_ring_size)
4380                         goto alloc_mem_err;
4381         }
4382
4383         return 0;
4384
4385 /* handles low memory cases */
4386 alloc_mem_err:
4387         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
4388                                                 index, ring_size);
4389         /* FW will drop all packets if queue is not big enough,
4390          * In these cases we disable the queue
4391          * Min size is different for OOO, TPA and non-TPA queues
4392          */
4393         if (ring_size < (fp->disable_tpa ?
4394                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
4395                         /* release memory allocated for this queue */
4396                         bnx2x_free_fp_mem_at(bp, index);
4397                         return -ENOMEM;
4398         }
4399         return 0;
4400 }
4401
4402 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
4403 {
4404         if (!NO_FCOE(bp))
4405                 /* FCoE */
4406                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
4407                         /* we will fail load process instead of mark
4408                          * NO_FCOE_FLAG
4409                          */
4410                         return -ENOMEM;
4411
4412         return 0;
4413 }
4414
4415 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
4416 {
4417         int i;
4418
4419         /* 1. Allocate FP for leading - fatal if error
4420          * 2. Allocate RSS - fix number of queues if error
4421          */
4422
4423         /* leading */
4424         if (bnx2x_alloc_fp_mem_at(bp, 0))
4425                 return -ENOMEM;
4426
4427         /* RSS */
4428         for_each_nondefault_eth_queue(bp, i)
4429                 if (bnx2x_alloc_fp_mem_at(bp, i))
4430                         break;
4431
4432         /* handle memory failures */
4433         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
4434                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
4435
4436                 WARN_ON(delta < 0);
4437                 bnx2x_shrink_eth_fp(bp, delta);
4438                 if (CNIC_SUPPORT(bp))
4439                         /* move non eth FPs next to last eth FP
4440                          * must be done in that order
4441                          * FCOE_IDX < FWD_IDX < OOO_IDX
4442                          */
4443
4444                         /* move FCoE fp even NO_FCOE_FLAG is on */
4445                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
4446                 bp->num_ethernet_queues -= delta;
4447                 bp->num_queues = bp->num_ethernet_queues +
4448                                  bp->num_cnic_queues;
4449                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
4450                           bp->num_queues + delta, bp->num_queues);
4451         }
4452
4453         return 0;
4454 }
4455
4456 void bnx2x_free_mem_bp(struct bnx2x *bp)
4457 {
4458         int i;
4459
4460         for (i = 0; i < bp->fp_array_size; i++)
4461                 kfree(bp->fp[i].tpa_info);
4462         kfree(bp->fp);
4463         kfree(bp->sp_objs);
4464         kfree(bp->fp_stats);
4465         kfree(bp->bnx2x_txq);
4466         kfree(bp->msix_table);
4467         kfree(bp->ilt);
4468 }
4469
4470 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
4471 {
4472         struct bnx2x_fastpath *fp;
4473         struct msix_entry *tbl;
4474         struct bnx2x_ilt *ilt;
4475         int msix_table_size = 0;
4476         int fp_array_size, txq_array_size;
4477         int i;
4478
4479         /*
4480          * The biggest MSI-X table we might need is as a maximum number of fast
4481          * path IGU SBs plus default SB (for PF only).
4482          */
4483         msix_table_size = bp->igu_sb_cnt;
4484         if (IS_PF(bp))
4485                 msix_table_size++;
4486         BNX2X_DEV_INFO("msix_table_size %d\n", msix_table_size);
4487
4488         /* fp array: RSS plus CNIC related L2 queues */
4489         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
4490         bp->fp_array_size = fp_array_size;
4491         BNX2X_DEV_INFO("fp_array_size %d\n", bp->fp_array_size);
4492
4493         fp = kcalloc(bp->fp_array_size, sizeof(*fp), GFP_KERNEL);
4494         if (!fp)
4495                 goto alloc_err;
4496         for (i = 0; i < bp->fp_array_size; i++) {
4497                 fp[i].tpa_info =
4498                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
4499                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
4500                 if (!(fp[i].tpa_info))
4501                         goto alloc_err;
4502         }
4503
4504         bp->fp = fp;
4505
4506         /* allocate sp objs */
4507         bp->sp_objs = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_sp_objs),
4508                               GFP_KERNEL);
4509         if (!bp->sp_objs)
4510                 goto alloc_err;
4511
4512         /* allocate fp_stats */
4513         bp->fp_stats = kcalloc(bp->fp_array_size, sizeof(struct bnx2x_fp_stats),
4514                                GFP_KERNEL);
4515         if (!bp->fp_stats)
4516                 goto alloc_err;
4517
4518         /* Allocate memory for the transmission queues array */
4519         txq_array_size =
4520                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
4521         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
4522
4523         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
4524                                 GFP_KERNEL);
4525         if (!bp->bnx2x_txq)
4526                 goto alloc_err;
4527
4528         /* msix table */
4529         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
4530         if (!tbl)
4531                 goto alloc_err;
4532         bp->msix_table = tbl;
4533
4534         /* ilt */
4535         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
4536         if (!ilt)
4537                 goto alloc_err;
4538         bp->ilt = ilt;
4539
4540         return 0;
4541 alloc_err:
4542         bnx2x_free_mem_bp(bp);
4543         return -ENOMEM;
4544 }
4545
4546 int bnx2x_reload_if_running(struct net_device *dev)
4547 {
4548         struct bnx2x *bp = netdev_priv(dev);
4549
4550         if (unlikely(!netif_running(dev)))
4551                 return 0;
4552
4553         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
4554         return bnx2x_nic_load(bp, LOAD_NORMAL);
4555 }
4556
4557 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
4558 {
4559         u32 sel_phy_idx = 0;
4560         if (bp->link_params.num_phys <= 1)
4561                 return INT_PHY;
4562
4563         if (bp->link_vars.link_up) {
4564                 sel_phy_idx = EXT_PHY1;
4565                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
4566                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
4567                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
4568                         sel_phy_idx = EXT_PHY2;
4569         } else {
4570
4571                 switch (bnx2x_phy_selection(&bp->link_params)) {
4572                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
4573                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
4574                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
4575                        sel_phy_idx = EXT_PHY1;
4576                        break;
4577                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4578                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4579                        sel_phy_idx = EXT_PHY2;
4580                        break;
4581                 }
4582         }
4583
4584         return sel_phy_idx;
4585 }
4586 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4587 {
4588         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4589         /*
4590          * The selected activated PHY is always after swapping (in case PHY
4591          * swapping is enabled). So when swapping is enabled, we need to reverse
4592          * the configuration
4593          */
4594
4595         if (bp->link_params.multi_phy_config &
4596             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4597                 if (sel_phy_idx == EXT_PHY1)
4598                         sel_phy_idx = EXT_PHY2;
4599                 else if (sel_phy_idx == EXT_PHY2)
4600                         sel_phy_idx = EXT_PHY1;
4601         }
4602         return LINK_CONFIG_IDX(sel_phy_idx);
4603 }
4604
4605 #ifdef NETDEV_FCOE_WWNN
4606 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4607 {
4608         struct bnx2x *bp = netdev_priv(dev);
4609         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4610
4611         switch (type) {
4612         case NETDEV_FCOE_WWNN:
4613                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4614                                 cp->fcoe_wwn_node_name_lo);
4615                 break;
4616         case NETDEV_FCOE_WWPN:
4617                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4618                                 cp->fcoe_wwn_port_name_lo);
4619                 break;
4620         default:
4621                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4622                 return -EINVAL;
4623         }
4624
4625         return 0;
4626 }
4627 #endif
4628
4629 /* called with rtnl_lock */
4630 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4631 {
4632         struct bnx2x *bp = netdev_priv(dev);
4633
4634         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4635                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4636                 return -EAGAIN;
4637         }
4638
4639         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4640             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4641                 BNX2X_ERR("Can't support requested MTU size\n");
4642                 return -EINVAL;
4643         }
4644
4645         /* This does not race with packet allocation
4646          * because the actual alloc size is
4647          * only updated as part of load
4648          */
4649         dev->mtu = new_mtu;
4650
4651         return bnx2x_reload_if_running(dev);
4652 }
4653
4654 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4655                                      netdev_features_t features)
4656 {
4657         struct bnx2x *bp = netdev_priv(dev);
4658
4659         /* TPA requires Rx CSUM offloading */
4660         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4661                 features &= ~NETIF_F_LRO;
4662                 features &= ~NETIF_F_GRO;
4663         }
4664
4665         return features;
4666 }
4667
4668 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4669 {
4670         struct bnx2x *bp = netdev_priv(dev);
4671         u32 flags = bp->flags;
4672         u32 changes;
4673         bool bnx2x_reload = false;
4674
4675         if (features & NETIF_F_LRO)
4676                 flags |= TPA_ENABLE_FLAG;
4677         else
4678                 flags &= ~TPA_ENABLE_FLAG;
4679
4680         if (features & NETIF_F_GRO)
4681                 flags |= GRO_ENABLE_FLAG;
4682         else
4683                 flags &= ~GRO_ENABLE_FLAG;
4684
4685         if (features & NETIF_F_LOOPBACK) {
4686                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4687                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4688                         bnx2x_reload = true;
4689                 }
4690         } else {
4691                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4692                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4693                         bnx2x_reload = true;
4694                 }
4695         }
4696
4697         changes = flags ^ bp->flags;
4698
4699         /* if GRO is changed while LRO is enabled, don't force a reload */
4700         if ((changes & GRO_ENABLE_FLAG) && (flags & TPA_ENABLE_FLAG))
4701                 changes &= ~GRO_ENABLE_FLAG;
4702
4703         if (changes)
4704                 bnx2x_reload = true;
4705
4706         bp->flags = flags;
4707
4708         if (bnx2x_reload) {
4709                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4710                         return bnx2x_reload_if_running(dev);
4711                 /* else: bnx2x_nic_load() will be called at end of recovery */
4712         }
4713
4714         return 0;
4715 }
4716
4717 void bnx2x_tx_timeout(struct net_device *dev)
4718 {
4719         struct bnx2x *bp = netdev_priv(dev);
4720
4721 #ifdef BNX2X_STOP_ON_ERROR
4722         if (!bp->panic)
4723                 bnx2x_panic();
4724 #endif
4725
4726         smp_mb__before_clear_bit();
4727         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4728         smp_mb__after_clear_bit();
4729
4730         /* This allows the netif to be shutdown gracefully before resetting */
4731         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4732 }
4733
4734 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4735 {
4736         struct net_device *dev = pci_get_drvdata(pdev);
4737         struct bnx2x *bp;
4738
4739         if (!dev) {
4740                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4741                 return -ENODEV;
4742         }
4743         bp = netdev_priv(dev);
4744
4745         rtnl_lock();
4746
4747         pci_save_state(pdev);
4748
4749         if (!netif_running(dev)) {
4750                 rtnl_unlock();
4751                 return 0;
4752         }
4753
4754         netif_device_detach(dev);
4755
4756         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4757
4758         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4759
4760         rtnl_unlock();
4761
4762         return 0;
4763 }
4764
4765 int bnx2x_resume(struct pci_dev *pdev)
4766 {
4767         struct net_device *dev = pci_get_drvdata(pdev);
4768         struct bnx2x *bp;
4769         int rc;
4770
4771         if (!dev) {
4772                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4773                 return -ENODEV;
4774         }
4775         bp = netdev_priv(dev);
4776
4777         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4778                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4779                 return -EAGAIN;
4780         }
4781
4782         rtnl_lock();
4783
4784         pci_restore_state(pdev);
4785
4786         if (!netif_running(dev)) {
4787                 rtnl_unlock();
4788                 return 0;
4789         }
4790
4791         bnx2x_set_power_state(bp, PCI_D0);
4792         netif_device_attach(dev);
4793
4794         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4795
4796         rtnl_unlock();
4797
4798         return rc;
4799 }
4800
4801 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4802                               u32 cid)
4803 {
4804         if (!cxt) {
4805                 BNX2X_ERR("bad context pointer %p\n", cxt);
4806                 return;
4807         }
4808
4809         /* ustorm cxt validation */
4810         cxt->ustorm_ag_context.cdu_usage =
4811                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4812                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4813         /* xcontext validation */
4814         cxt->xstorm_ag_context.cdu_reserved =
4815                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4816                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4817 }
4818
4819 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4820                                     u8 fw_sb_id, u8 sb_index,
4821                                     u8 ticks)
4822 {
4823         u32 addr = BAR_CSTRORM_INTMEM +
4824                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4825         REG_WR8(bp, addr, ticks);
4826         DP(NETIF_MSG_IFUP,
4827            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4828            port, fw_sb_id, sb_index, ticks);
4829 }
4830
4831 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4832                                     u16 fw_sb_id, u8 sb_index,
4833                                     u8 disable)
4834 {
4835         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4836         u32 addr = BAR_CSTRORM_INTMEM +
4837                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4838         u8 flags = REG_RD8(bp, addr);
4839         /* clear and set */
4840         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4841         flags |= enable_flag;
4842         REG_WR8(bp, addr, flags);
4843         DP(NETIF_MSG_IFUP,
4844            "port %x fw_sb_id %d sb_index %d disable %d\n",
4845            port, fw_sb_id, sb_index, disable);
4846 }
4847
4848 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4849                                     u8 sb_index, u8 disable, u16 usec)
4850 {
4851         int port = BP_PORT(bp);
4852         u8 ticks = usec / BNX2X_BTR;
4853
4854         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4855
4856         disable = disable ? 1 : (usec ? 0 : 1);
4857         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4858 }