drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/firmware.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32
  33
  34 /**
  35  * bnx2x_move_fp - move content of the fastpath structure.
  36  *
  37  * @bp:         driver handle
  38  * @from:       source FP index
  39  * @to:         destination FP index
  40  *
  41  * Makes sure the contents of the bp->fp[to].napi is kept
  42  * intact. This is done by first copying the napi struct from
  43  * the target to the source, and then mem copying the entire
  44  * source onto the target
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50
  51         /* Copy the NAPI object as it has been already initialized */
  52         from_fp->napi = to_fp->napi;
  53
  54         /* Move bnx2x_fastpath contents */
  55         memcpy(to_fp, from_fp, sizeof(*to_fp));
  56         to_fp->index = to;
  57 }
  58
  59 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  60
  61 /* free skb in the packet ring at pos idx
  62  * return idx of last bd freed
  63  */
  64 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  65                              u16 idx, unsigned int *pkts_compl,
  66                              unsigned int *bytes_compl)
  67 {
  68         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  69         struct eth_tx_start_bd *tx_start_bd;
  70         struct eth_tx_bd *tx_data_bd;
  71         struct sk_buff *skb = tx_buf->skb;
  72         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  73         int nbd;
  74
  75         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  76         prefetch(&skb->end);
  77
  78         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
  79            txdata->txq_index, idx, tx_buf, skb);
  80
  81         /* unmap first bd */
  82         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
  83         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  84                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  85
  86
  87         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  88 #ifdef BNX2X_STOP_ON_ERROR
  89         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  90                 BNX2X_ERR("BAD nbd!\n");
  91                 bnx2x_panic();
  92         }
  93 #endif
  94         new_cons = nbd + tx_buf->first_bd;
  95
  96         /* Get the next bd */
  97         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  98
  99         /* Skip a parse bd... */
 100         --nbd;
 101         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 102
 103         /* ...and the TSO split header bd since they have no mapping */
 104         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 105                 --nbd;
 106                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 107         }
 108
 109         /* now free frags */
 110         while (nbd > 0) {
 111
 112                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 113                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 114                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 115                 if (--nbd)
 116                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 117         }
 118
 119         /* release skb */
 120         WARN_ON(!skb);
 121         if (likely(skb)) {
 122                 (*pkts_compl)++;
 123                 (*bytes_compl) += skb->len;
 124         }
 125
 126         dev_kfree_skb_any(skb);
 127         tx_buf->first_bd = 0;
 128         tx_buf->skb = NULL;
 129
 130         return new_cons;
 131 }
 132
 133 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 134 {
 135         struct netdev_queue *txq;
 136         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 137         unsigned int pkts_compl = 0, bytes_compl = 0;
 138
 139 #ifdef BNX2X_STOP_ON_ERROR
 140         if (unlikely(bp->panic))
 141                 return -1;
 142 #endif
 143
 144         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 145         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 146         sw_cons = txdata->tx_pkt_cons;
 147
 148         while (sw_cons != hw_cons) {
 149                 u16 pkt_cons;
 150
 151                 pkt_cons = TX_BD(sw_cons);
 152
 153                 DP(NETIF_MSG_TX_DONE,
 154                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 155                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 156
 157                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 158                     &pkts_compl, &bytes_compl);
 159
 160                 sw_cons++;
 161         }
 162
 163         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 164
 165         txdata->tx_pkt_cons = sw_cons;
 166         txdata->tx_bd_cons = bd_cons;
 167
 168         /* Need to make the tx_bd_cons update visible to start_xmit()
 169          * before checking for netif_tx_queue_stopped().  Without the
 170          * memory barrier, there is a small possibility that
 171          * start_xmit() will miss it and cause the queue to be stopped
 172          * forever.
 173          * On the other hand we need an rmb() here to ensure the proper
 174          * ordering of bit testing in the following
 175          * netif_tx_queue_stopped(txq) call.
 176          */
 177         smp_mb();
 178
 179         if (unlikely(netif_tx_queue_stopped(txq))) {
 180                 /* Taking tx_lock() is needed to prevent reenabling the queue
 181                  * while it's empty. This could have happen if rx_action() gets
 182                  * suspended in bnx2x_tx_int() after the condition before
 183                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 184                  *
 185                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 186                  * sends some packets consuming the whole queue again->
 187                  * stops the queue
 188                  */
 189
 190                 __netif_tx_lock(txq, smp_processor_id());
 191
 192                 if ((netif_tx_queue_stopped(txq)) &&
 193                     (bp->state == BNX2X_STATE_OPEN) &&
 194                     (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
 195                         netif_tx_wake_queue(txq);
 196
 197                 __netif_tx_unlock(txq);
 198         }
 199         return 0;
 200 }
 201
 202 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 203                                              u16 idx)
 204 {
 205         u16 last_max = fp->last_max_sge;
 206
 207         if (SUB_S16(idx, last_max) > 0)
 208                 fp->last_max_sge = idx;
 209 }
 210
 211 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 212                                          u16 sge_len,
 213                                          struct eth_end_agg_rx_cqe *cqe)
 214 {
 215         struct bnx2x *bp = fp->bp;
 216         u16 last_max, last_elem, first_elem;
 217         u16 delta = 0;
 218         u16 i;
 219
 220         if (!sge_len)
 221                 return;
 222
 223         /* First mark all used pages */
 224         for (i = 0; i < sge_len; i++)
 225                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 226                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 227
 228         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 229            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 230
 231         /* Here we assume that the last SGE index is the biggest */
 232         prefetch((void *)(fp->sge_mask));
 233         bnx2x_update_last_max_sge(fp,
 234                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 235
 236         last_max = RX_SGE(fp->last_max_sge);
 237         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 238         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 239
 240         /* If ring is not full */
 241         if (last_elem + 1 != first_elem)
 242                 last_elem++;
 243
 244         /* Now update the prod */
 245         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 246                 if (likely(fp->sge_mask[i]))
 247                         break;
 248
 249                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 250                 delta += BIT_VEC64_ELEM_SZ;
 251         }
 252
 253         if (delta > 0) {
 254                 fp->rx_sge_prod += delta;
 255                 /* clear page-end entries */
 256                 bnx2x_clear_sge_mask_next_elems(fp);
 257         }
 258
 259         DP(NETIF_MSG_RX_STATUS,
 260            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 261            fp->last_max_sge, fp->rx_sge_prod);
 262 }
 263
 264 /* Set Toeplitz hash value in the skb using the value from the
 265  * CQE (calculated by HW).
 266  */
 267 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 268                             const struct eth_fast_path_rx_cqe *cqe)
 269 {
 270         /* Set Toeplitz hash from CQE */
 271         if ((bp->dev->features & NETIF_F_RXHASH) &&
 272             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 273                 return le32_to_cpu(cqe->rss_hash_result);
 274         return 0;
 275 }
 276
 277 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 278                             u16 cons, u16 prod,
 279                             struct eth_fast_path_rx_cqe *cqe)
 280 {
 281         struct bnx2x *bp = fp->bp;
 282         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 283         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 284         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 285         dma_addr_t mapping;
 286         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 287         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 288
 289         /* print error if current state != stop */
 290         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 291                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 292
 293         /* Try to map an empty data buffer from the aggregation info  */
 294         mapping = dma_map_single(&bp->pdev->dev,
 295                                  first_buf->data + NET_SKB_PAD,
 296                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 297         /*
 298          *  ...if it fails - move the skb from the consumer to the producer
 299          *  and set the current aggregation state as ERROR to drop it
 300          *  when TPA_STOP arrives.
 301          */
 302
 303         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 304                 /* Move the BD from the consumer to the producer */
 305                 bnx2x_reuse_rx_data(fp, cons, prod);
 306                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 307                 return;
 308         }
 309
 310         /* move empty data from pool to prod */
 311         prod_rx_buf->data = first_buf->data;
 312         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 313         /* point prod_bd to new data */
 314         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 315         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 316
 317         /* move partial skb from cons to pool (don't unmap yet) */
 318         *first_buf = *cons_rx_buf;
 319
 320         /* mark bin state as START */
 321         tpa_info->parsing_flags =
 322                 le16_to_cpu(cqe->pars_flags.flags);
 323         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 324         tpa_info->tpa_state = BNX2X_TPA_START;
 325         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 326         tpa_info->placement_offset = cqe->placement_offset;
 327         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe);
 328         if (fp->mode == TPA_MODE_GRO) {
 329                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 330                 tpa_info->full_page =
 331                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 332                 /*
 333                  * FW 7.2.16 BUG workaround:
 334                  * if SGE size is (exactly) multiple gro_size
 335                  * fw will place one less frag on SGE.
 336                  * the calculation is done only for potentially
 337                  * dangerous MTUs.
 338                  */
 339                 if (unlikely(bp->gro_check))
 340                         if (!(SGE_PAGE_SIZE * PAGES_PER_SGE % gro_size))
 341                                 tpa_info->full_page -= gro_size;
 342                 tpa_info->gro_size = gro_size;
 343         }
 344
 345 #ifdef BNX2X_STOP_ON_ERROR
 346         fp->tpa_queue_used |= (1 << queue);
 347 #ifdef _ASM_GENERIC_INT_L64_H
 348         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 349 #else
 350         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 351 #endif
 352            fp->tpa_queue_used);
 353 #endif
 354 }
 355
 356 /* Timestamp option length allowed for TPA aggregation:
 357  *
 358  *              nop nop kind length echo val
 359  */
 360 #define TPA_TSTAMP_OPT_LEN      12
 361 /**
 362  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 363  *
 364  * @bp:                 driver handle
 365  * @parsing_flags:      parsing flags from the START CQE
 366  * @len_on_bd:          total length of the first packet for the
 367  *                      aggregation.
 368  *
 369  * Approximate value of the MSS for this aggregation calculated using
 370  * the first packet of it.
 371  */
 372 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 373                                     u16 len_on_bd)
 374 {
 375         /*
 376          * TPA arrgregation won't have either IP options or TCP options
 377          * other than timestamp or IPv6 extension headers.
 378          */
 379         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 380
 381         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 382             PRS_FLAG_OVERETH_IPV6)
 383                 hdrs_len += sizeof(struct ipv6hdr);
 384         else /* IPv4 */
 385                 hdrs_len += sizeof(struct iphdr);
 386
 387
 388         /* Check if there was a TCP timestamp, if there is it's will
 389          * always be 12 bytes length: nop nop kind length echo val.
 390          *
 391          * Otherwise FW would close the aggregation.
 392          */
 393         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 394                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 395
 396         return len_on_bd - hdrs_len;
 397 }
 398
 399 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 400                                struct bnx2x_agg_info *tpa_info,
 401                                u16 pages,
 402                                struct sk_buff *skb,
 403                                struct eth_end_agg_rx_cqe *cqe,
 404                                u16 cqe_idx)
 405 {
 406         struct sw_rx_page *rx_pg, old_rx_pg;
 407         u32 i, frag_len, frag_size;
 408         int err, j, frag_id = 0;
 409         u16 len_on_bd = tpa_info->len_on_bd;
 410         u16 full_page = 0, gro_size = 0;
 411
 412         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 413
 414         if (fp->mode == TPA_MODE_GRO) {
 415                 gro_size = tpa_info->gro_size;
 416                 full_page = tpa_info->full_page;
 417         }
 418
 419         /* This is needed in order to enable forwarding support */
 420         if (frag_size) {
 421                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 422                                         tpa_info->parsing_flags, len_on_bd);
 423
 424                 /* set for GRO */
 425                 if (fp->mode == TPA_MODE_GRO)
 426                         skb_shinfo(skb)->gso_type =
 427                             (GET_FLAG(tpa_info->parsing_flags,
 428                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 429                                                 PRS_FLAG_OVERETH_IPV6) ?
 430                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 431         }
 432
 433
 434 #ifdef BNX2X_STOP_ON_ERROR
 435         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 436                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 437                           pages, cqe_idx);
 438                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 439                 bnx2x_panic();
 440                 return -EINVAL;
 441         }
 442 #endif
 443
 444         /* Run through the SGL and compose the fragmented skb */
 445         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 446                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 447
 448                 /* FW gives the indices of the SGE as if the ring is an array
 449                    (meaning that "next" element will consume 2 indices) */
 450                 if (fp->mode == TPA_MODE_GRO)
 451                         frag_len = min_t(u32, frag_size, (u32)full_page);
 452                 else /* LRO */
 453                         frag_len = min_t(u32, frag_size,
 454                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 455
 456                 rx_pg = &fp->rx_page_ring[sge_idx];
 457                 old_rx_pg = *rx_pg;
 458
 459                 /* If we fail to allocate a substitute page, we simply stop
 460                    where we are and drop the whole packet */
 461                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 462                 if (unlikely(err)) {
 463                         fp->eth_q_stats.rx_skb_alloc_failed++;
 464                         return err;
 465                 }
 466
 467                 /* Unmap the page as we r going to pass it to the stack */
 468                 dma_unmap_page(&bp->pdev->dev,
 469                                dma_unmap_addr(&old_rx_pg, mapping),
 470                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 471                 /* Add one frag and update the appropriate fields in the skb */
 472                 if (fp->mode == TPA_MODE_LRO)
 473                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 474                 else { /* GRO */
 475                         int rem;
 476                         int offset = 0;
 477                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 478                                 int len = rem > gro_size ? gro_size : rem;
 479                                 skb_fill_page_desc(skb, frag_id++,
 480                                                    old_rx_pg.page, offset, len);
 481                                 if (offset)
 482                                         get_page(old_rx_pg.page);
 483                                 offset += len;
 484                         }
 485                 }
 486
 487                 skb->data_len += frag_len;
 488                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 489                 skb->len += frag_len;
 490
 491                 frag_size -= frag_len;
 492         }
 493
 494         return 0;
 495 }
 496
 497 static inline void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 498                                   struct bnx2x_agg_info *tpa_info,
 499                                   u16 pages,
 500                                   struct eth_end_agg_rx_cqe *cqe,
 501                                   u16 cqe_idx)
 502 {
 503         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 504         u8 pad = tpa_info->placement_offset;
 505         u16 len = tpa_info->len_on_bd;
 506         struct sk_buff *skb = NULL;
 507         u8 *new_data, *data = rx_buf->data;
 508         u8 old_tpa_state = tpa_info->tpa_state;
 509
 510         tpa_info->tpa_state = BNX2X_TPA_STOP;
 511
 512         /* If we there was an error during the handling of the TPA_START -
 513          * drop this aggregation.
 514          */
 515         if (old_tpa_state == BNX2X_TPA_ERROR)
 516                 goto drop;
 517
 518         /* Try to allocate the new data */
 519         new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 520
 521         /* Unmap skb in the pool anyway, as we are going to change
 522            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 523            fails. */
 524         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 525                          fp->rx_buf_size, DMA_FROM_DEVICE);
 526         if (likely(new_data))
 527                 skb = build_skb(data);
 528
 529         if (likely(skb)) {
 530 #ifdef BNX2X_STOP_ON_ERROR
 531                 if (pad + len > fp->rx_buf_size) {
 532                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 533                                   pad, len, fp->rx_buf_size);
 534                         bnx2x_panic();
 535                         return;
 536                 }
 537 #endif
 538
 539                 skb_reserve(skb, pad + NET_SKB_PAD);
 540                 skb_put(skb, len);
 541                 skb->rxhash = tpa_info->rxhash;
 542
 543                 skb->protocol = eth_type_trans(skb, bp->dev);
 544                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 545
 546                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 547                                          skb, cqe, cqe_idx)) {
 548                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 549                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 550                         napi_gro_receive(&fp->napi, skb);
 551                 } else {
 552                         DP(NETIF_MSG_RX_STATUS,
 553                            "Failed to allocate new pages - dropping packet!\n");
 554                         dev_kfree_skb_any(skb);
 555                 }
 556
 557
 558                 /* put new data in bin */
 559                 rx_buf->data = new_data;
 560
 561                 return;
 562         }
 563         kfree(new_data);
 564 drop:
 565         /* drop the packet and keep the buffer in the bin */
 566         DP(NETIF_MSG_RX_STATUS,
 567            "Failed to allocate or map a new skb - dropping packet!\n");
 568         fp->eth_q_stats.rx_skb_alloc_failed++;
 569 }
 570
 571
 572 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 573 {
 574         struct bnx2x *bp = fp->bp;
 575         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 576         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 577         int rx_pkt = 0;
 578
 579 #ifdef BNX2X_STOP_ON_ERROR
 580         if (unlikely(bp->panic))
 581                 return 0;
 582 #endif
 583
 584         /* CQ "next element" is of the size of the regular element,
 585            that's why it's ok here */
 586         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 587         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 588                 hw_comp_cons++;
 589
 590         bd_cons = fp->rx_bd_cons;
 591         bd_prod = fp->rx_bd_prod;
 592         bd_prod_fw = bd_prod;
 593         sw_comp_cons = fp->rx_comp_cons;
 594         sw_comp_prod = fp->rx_comp_prod;
 595
 596         /* Memory barrier necessary as speculative reads of the rx
 597          * buffer can be ahead of the index in the status block
 598          */
 599         rmb();
 600
 601         DP(NETIF_MSG_RX_STATUS,
 602            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 603            fp->index, hw_comp_cons, sw_comp_cons);
 604
 605         while (sw_comp_cons != hw_comp_cons) {
 606                 struct sw_rx_bd *rx_buf = NULL;
 607                 struct sk_buff *skb;
 608                 union eth_rx_cqe *cqe;
 609                 struct eth_fast_path_rx_cqe *cqe_fp;
 610                 u8 cqe_fp_flags;
 611                 enum eth_rx_cqe_type cqe_fp_type;
 612                 u16 len, pad, queue;
 613                 u8 *data;
 614
 615 #ifdef BNX2X_STOP_ON_ERROR
 616                 if (unlikely(bp->panic))
 617                         return 0;
 618 #endif
 619
 620                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 621                 bd_prod = RX_BD(bd_prod);
 622                 bd_cons = RX_BD(bd_cons);
 623
 624                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 625                 cqe_fp = &cqe->fast_path_cqe;
 626                 cqe_fp_flags = cqe_fp->type_error_flags;
 627                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 628
 629                 DP(NETIF_MSG_RX_STATUS,
 630                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 631                    CQE_TYPE(cqe_fp_flags),
 632                    cqe_fp_flags, cqe_fp->status_flags,
 633                    le32_to_cpu(cqe_fp->rss_hash_result),
 634                    le16_to_cpu(cqe_fp->vlan_tag),
 635                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 636
 637                 /* is this a slowpath msg? */
 638                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 639                         bnx2x_sp_event(fp, cqe);
 640                         goto next_cqe;
 641                 }
 642
 643                 rx_buf = &fp->rx_buf_ring[bd_cons];
 644                 data = rx_buf->data;
 645
 646                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 647                         struct bnx2x_agg_info *tpa_info;
 648                         u16 frag_size, pages;
 649 #ifdef BNX2X_STOP_ON_ERROR
 650                         /* sanity check */
 651                         if (fp->disable_tpa &&
 652                             (CQE_TYPE_START(cqe_fp_type) ||
 653                              CQE_TYPE_STOP(cqe_fp_type)))
 654                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 655                                           CQE_TYPE(cqe_fp_type));
 656 #endif
 657
 658                         if (CQE_TYPE_START(cqe_fp_type)) {
 659                                 u16 queue = cqe_fp->queue_index;
 660                                 DP(NETIF_MSG_RX_STATUS,
 661                                    "calling tpa_start on queue %d\n",
 662                                    queue);
 663
 664                                 bnx2x_tpa_start(fp, queue,
 665                                                 bd_cons, bd_prod,
 666                                                 cqe_fp);
 667
 668                                 goto next_rx;
 669
 670                         }
 671                         queue = cqe->end_agg_cqe.queue_index;
 672                         tpa_info = &fp->tpa_info[queue];
 673                         DP(NETIF_MSG_RX_STATUS,
 674                            "calling tpa_stop on queue %d\n",
 675                            queue);
 676
 677                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 678                                     tpa_info->len_on_bd;
 679
 680                         if (fp->mode == TPA_MODE_GRO)
 681                                 pages = (frag_size + tpa_info->full_page - 1) /
 682                                          tpa_info->full_page;
 683                         else
 684                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 685                                         SGE_PAGE_SHIFT;
 686
 687                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 688                                        &cqe->end_agg_cqe, comp_ring_cons);
 689 #ifdef BNX2X_STOP_ON_ERROR
 690                         if (bp->panic)
 691                                 return 0;
 692 #endif
 693
 694                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 695                         goto next_cqe;
 696                 }
 697                 /* non TPA */
 698                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 699                 pad = cqe_fp->placement_offset;
 700                 dma_sync_single_for_cpu(&bp->pdev->dev,
 701                                         dma_unmap_addr(rx_buf, mapping),
 702                                         pad + RX_COPY_THRESH,
 703                                         DMA_FROM_DEVICE);
 704                 pad += NET_SKB_PAD;
 705                 prefetch(data + pad); /* speedup eth_type_trans() */
 706                 /* is this an error packet? */
 707                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 708                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 709                            "ERROR  flags %x  rx packet %u\n",
 710                            cqe_fp_flags, sw_comp_cons);
 711                         fp->eth_q_stats.rx_err_discard_pkt++;
 712                         goto reuse_rx;
 713                 }
 714
 715                 /* Since we don't have a jumbo ring
 716                  * copy small packets if mtu > 1500
 717                  */
 718                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 719                     (len <= RX_COPY_THRESH)) {
 720                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 721                         if (skb == NULL) {
 722                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 723                                    "ERROR  packet dropped because of alloc failure\n");
 724                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 725                                 goto reuse_rx;
 726                         }
 727                         memcpy(skb->data, data + pad, len);
 728                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 729                 } else {
 730                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 731                                 dma_unmap_single(&bp->pdev->dev,
 732                                                  dma_unmap_addr(rx_buf, mapping),
 733                                                  fp->rx_buf_size,
 734                                                  DMA_FROM_DEVICE);
 735                                 skb = build_skb(data);
 736                                 if (unlikely(!skb)) {
 737                                         kfree(data);
 738                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 739                                         goto next_rx;
 740                                 }
 741                                 skb_reserve(skb, pad);
 742                         } else {
 743                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 744                                    "ERROR  packet dropped because of alloc failure\n");
 745                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 746 reuse_rx:
 747                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 748                                 goto next_rx;
 749                         }
 750                 }
 751
 752                 skb_put(skb, len);
 753                 skb->protocol = eth_type_trans(skb, bp->dev);
 754
 755                 /* Set Toeplitz hash for a none-LRO skb */
 756                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp);
 757
 758                 skb_checksum_none_assert(skb);
 759
 760                 if (bp->dev->features & NETIF_F_RXCSUM) {
 761
 762                         if (likely(BNX2X_RX_CSUM_OK(cqe)))
 763                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 764                         else
 765                                 fp->eth_q_stats.hw_csum_err++;
 766                 }
 767
 768                 skb_record_rx_queue(skb, fp->rx_queue);
 769
 770                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 771                     PARSING_FLAGS_VLAN)
 772                         __vlan_hwaccel_put_tag(skb,
 773                                                le16_to_cpu(cqe_fp->vlan_tag));
 774                 napi_gro_receive(&fp->napi, skb);
 775
 776
 777 next_rx:
 778                 rx_buf->data = NULL;
 779
 780                 bd_cons = NEXT_RX_IDX(bd_cons);
 781                 bd_prod = NEXT_RX_IDX(bd_prod);
 782                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 783                 rx_pkt++;
 784 next_cqe:
 785                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 786                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 787
 788                 if (rx_pkt == budget)
 789                         break;
 790         } /* while */
 791
 792         fp->rx_bd_cons = bd_cons;
 793         fp->rx_bd_prod = bd_prod_fw;
 794         fp->rx_comp_cons = sw_comp_cons;
 795         fp->rx_comp_prod = sw_comp_prod;
 796
 797         /* Update producers */
 798         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 799                              fp->rx_sge_prod);
 800
 801         fp->rx_pkt += rx_pkt;
 802         fp->rx_calls++;
 803
 804         return rx_pkt;
 805 }
 806
 807 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 808 {
 809         struct bnx2x_fastpath *fp = fp_cookie;
 810         struct bnx2x *bp = fp->bp;
 811         u8 cos;
 812
 813         DP(NETIF_MSG_INTR,
 814            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 815            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 816         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 817
 818 #ifdef BNX2X_STOP_ON_ERROR
 819         if (unlikely(bp->panic))
 820                 return IRQ_HANDLED;
 821 #endif
 822
 823         /* Handle Rx and Tx according to MSI-X vector */
 824         prefetch(fp->rx_cons_sb);
 825
 826         for_each_cos_in_tx_queue(fp, cos)
 827                 prefetch(fp->txdata[cos].tx_cons_sb);
 828
 829         prefetch(&fp->sb_running_index[SM_RX_ID]);
 830         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 831
 832         return IRQ_HANDLED;
 833 }
 834
 835 /* HW Lock for shared dual port PHYs */
 836 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 837 {
 838         mutex_lock(&bp->port.phy_mutex);
 839
 840         if (bp->port.need_hw_lock)
 841                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 842 }
 843
 844 void bnx2x_release_phy_lock(struct bnx2x *bp)
 845 {
 846         if (bp->port.need_hw_lock)
 847                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 848
 849         mutex_unlock(&bp->port.phy_mutex);
 850 }
 851
 852 /* calculates MF speed according to current linespeed and MF configuration */
 853 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 854 {
 855         u16 line_speed = bp->link_vars.line_speed;
 856         if (IS_MF(bp)) {
 857                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 858                                                    bp->mf_config[BP_VN(bp)]);
 859
 860                 /* Calculate the current MAX line speed limit for the MF
 861                  * devices
 862                  */
 863                 if (IS_MF_SI(bp))
 864                         line_speed = (line_speed * maxCfg) / 100;
 865                 else { /* SD mode */
 866                         u16 vn_max_rate = maxCfg * 100;
 867
 868                         if (vn_max_rate < line_speed)
 869                                 line_speed = vn_max_rate;
 870                 }
 871         }
 872
 873         return line_speed;
 874 }
 875
 876 /**
 877  * bnx2x_fill_report_data - fill link report data to report
 878  *
 879  * @bp:         driver handle
 880  * @data:       link state to update
 881  *
 882  * It uses a none-atomic bit operations because is called under the mutex.
 883  */
 884 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
 885                                           struct bnx2x_link_report_data *data)
 886 {
 887         u16 line_speed = bnx2x_get_mf_speed(bp);
 888
 889         memset(data, 0, sizeof(*data));
 890
 891         /* Fill the report data: efective line speed */
 892         data->line_speed = line_speed;
 893
 894         /* Link is down */
 895         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
 896                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 897                           &data->link_report_flags);
 898
 899         /* Full DUPLEX */
 900         if (bp->link_vars.duplex == DUPLEX_FULL)
 901                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
 902
 903         /* Rx Flow Control is ON */
 904         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
 905                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
 906
 907         /* Tx Flow Control is ON */
 908         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
 909                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
 910 }
 911
 912 /**
 913  * bnx2x_link_report - report link status to OS.
 914  *
 915  * @bp:         driver handle
 916  *
 917  * Calls the __bnx2x_link_report() under the same locking scheme
 918  * as a link/PHY state managing code to ensure a consistent link
 919  * reporting.
 920  */
 921
 922 void bnx2x_link_report(struct bnx2x *bp)
 923 {
 924         bnx2x_acquire_phy_lock(bp);
 925         __bnx2x_link_report(bp);
 926         bnx2x_release_phy_lock(bp);
 927 }
 928
 929 /**
 930  * __bnx2x_link_report - report link status to OS.
 931  *
 932  * @bp:         driver handle
 933  *
 934  * None atomic inmlementation.
 935  * Should be called under the phy_lock.
 936  */
 937 void __bnx2x_link_report(struct bnx2x *bp)
 938 {
 939         struct bnx2x_link_report_data cur_data;
 940
 941         /* reread mf_cfg */
 942         if (!CHIP_IS_E1(bp))
 943                 bnx2x_read_mf_cfg(bp);
 944
 945         /* Read the current link report info */
 946         bnx2x_fill_report_data(bp, &cur_data);
 947
 948         /* Don't report link down or exactly the same link status twice */
 949         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
 950             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 951                       &bp->last_reported_link.link_report_flags) &&
 952              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 953                       &cur_data.link_report_flags)))
 954                 return;
 955
 956         bp->link_cnt++;
 957
 958         /* We are going to report a new link parameters now -
 959          * remember the current data for the next time.
 960          */
 961         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
 962
 963         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 964                      &cur_data.link_report_flags)) {
 965                 netif_carrier_off(bp->dev);
 966                 netdev_err(bp->dev, "NIC Link is Down\n");
 967                 return;
 968         } else {
 969                 const char *duplex;
 970                 const char *flow;
 971
 972                 netif_carrier_on(bp->dev);
 973
 974                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
 975                                        &cur_data.link_report_flags))
 976                         duplex = "full";
 977                 else
 978                         duplex = "half";
 979
 980                 /* Handle the FC at the end so that only these flags would be
 981                  * possibly set. This way we may easily check if there is no FC
 982                  * enabled.
 983                  */
 984                 if (cur_data.link_report_flags) {
 985                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
 986                                      &cur_data.link_report_flags)) {
 987                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
 988                                      &cur_data.link_report_flags))
 989                                         flow = "ON - receive & transmit";
 990                                 else
 991                                         flow = "ON - receive";
 992                         } else {
 993                                 flow = "ON - transmit";
 994                         }
 995                 } else {
 996                         flow = "none";
 997                 }
 998                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
 999                             cur_data.line_speed, duplex, flow);
1000         }
1001 }
1002
1003 void bnx2x_init_rx_rings(struct bnx2x *bp)
1004 {
1005         int func = BP_FUNC(bp);
1006         u16 ring_prod;
1007         int i, j;
1008
1009         /* Allocate TPA resources */
1010         for_each_rx_queue(bp, j) {
1011                 struct bnx2x_fastpath *fp = &bp->fp[j];
1012
1013                 DP(NETIF_MSG_IFUP,
1014                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1015
1016                 if (!fp->disable_tpa) {
1017                         /* Fill the per-aggregtion pool */
1018                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1019                                 struct bnx2x_agg_info *tpa_info =
1020                                         &fp->tpa_info[i];
1021                                 struct sw_rx_bd *first_buf =
1022                                         &tpa_info->first_buf;
1023
1024                                 first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
1025                                                           GFP_ATOMIC);
1026                                 if (!first_buf->data) {
1027                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1028                                                   j);
1029                                         bnx2x_free_tpa_pool(bp, fp, i);
1030                                         fp->disable_tpa = 1;
1031                                         break;
1032                                 }
1033                                 dma_unmap_addr_set(first_buf, mapping, 0);
1034                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1035                         }
1036
1037                         /* "next page" elements initialization */
1038                         bnx2x_set_next_page_sgl(fp);
1039
1040                         /* set SGEs bit mask */
1041                         bnx2x_init_sge_ring_bit_mask(fp);
1042
1043                         /* Allocate SGEs and initialize the ring elements */
1044                         for (i = 0, ring_prod = 0;
1045                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1046
1047                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1048                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1049                                                   i);
1050                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1051                                                   j);
1052                                         /* Cleanup already allocated elements */
1053                                         bnx2x_free_rx_sge_range(bp, fp,
1054                                                                 ring_prod);
1055                                         bnx2x_free_tpa_pool(bp, fp,
1056                                                             MAX_AGG_QS(bp));
1057                                         fp->disable_tpa = 1;
1058                                         ring_prod = 0;
1059                                         break;
1060                                 }
1061                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1062                         }
1063
1064                         fp->rx_sge_prod = ring_prod;
1065                 }
1066         }
1067
1068         for_each_rx_queue(bp, j) {
1069                 struct bnx2x_fastpath *fp = &bp->fp[j];
1070
1071                 fp->rx_bd_cons = 0;
1072
1073                 /* Activate BD ring */
1074                 /* Warning!
1075                  * this will generate an interrupt (to the TSTORM)
1076                  * must only be done after chip is initialized
1077                  */
1078                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1079                                      fp->rx_sge_prod);
1080
1081                 if (j != 0)
1082                         continue;
1083
1084                 if (CHIP_IS_E1(bp)) {
1085                         REG_WR(bp, BAR_USTRORM_INTMEM +
1086                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1087                                U64_LO(fp->rx_comp_mapping));
1088                         REG_WR(bp, BAR_USTRORM_INTMEM +
1089                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1090                                U64_HI(fp->rx_comp_mapping));
1091                 }
1092         }
1093 }
1094
1095 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1096 {
1097         int i;
1098         u8 cos;
1099
1100         for_each_tx_queue(bp, i) {
1101                 struct bnx2x_fastpath *fp = &bp->fp[i];
1102                 for_each_cos_in_tx_queue(fp, cos) {
1103                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1104                         unsigned pkts_compl = 0, bytes_compl = 0;
1105
1106                         u16 sw_prod = txdata->tx_pkt_prod;
1107                         u16 sw_cons = txdata->tx_pkt_cons;
1108
1109                         while (sw_cons != sw_prod) {
1110                                 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1111                                     &pkts_compl, &bytes_compl);
1112                                 sw_cons++;
1113                         }
1114                         netdev_tx_reset_queue(
1115                             netdev_get_tx_queue(bp->dev, txdata->txq_index));
1116                 }
1117         }
1118 }
1119
1120 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1121 {
1122         struct bnx2x *bp = fp->bp;
1123         int i;
1124
1125         /* ring wasn't allocated */
1126         if (fp->rx_buf_ring == NULL)
1127                 return;
1128
1129         for (i = 0; i < NUM_RX_BD; i++) {
1130                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1131                 u8 *data = rx_buf->data;
1132
1133                 if (data == NULL)
1134                         continue;
1135                 dma_unmap_single(&bp->pdev->dev,
1136                                  dma_unmap_addr(rx_buf, mapping),
1137                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1138
1139                 rx_buf->data = NULL;
1140                 kfree(data);
1141         }
1142 }
1143
1144 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1145 {
1146         int j;
1147
1148         for_each_rx_queue(bp, j) {
1149                 struct bnx2x_fastpath *fp = &bp->fp[j];
1150
1151                 bnx2x_free_rx_bds(fp);
1152
1153                 if (!fp->disable_tpa)
1154                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1155         }
1156 }
1157
1158 void bnx2x_free_skbs(struct bnx2x *bp)
1159 {
1160         bnx2x_free_tx_skbs(bp);
1161         bnx2x_free_rx_skbs(bp);
1162 }
1163
1164 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1165 {
1166         /* load old values */
1167         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1168
1169         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1170                 /* leave all but MAX value */
1171                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1172
1173                 /* set new MAX value */
1174                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1175                                 & FUNC_MF_CFG_MAX_BW_MASK;
1176
1177                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1178         }
1179 }
1180
1181 /**
1182  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1183  *
1184  * @bp:         driver handle
1185  * @nvecs:      number of vectors to be released
1186  */
1187 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1188 {
1189         int i, offset = 0;
1190
1191         if (nvecs == offset)
1192                 return;
1193         free_irq(bp->msix_table[offset].vector, bp->dev);
1194         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1195            bp->msix_table[offset].vector);
1196         offset++;
1197 #ifdef BCM_CNIC
1198         if (nvecs == offset)
1199                 return;
1200         offset++;
1201 #endif
1202
1203         for_each_eth_queue(bp, i) {
1204                 if (nvecs == offset)
1205                         return;
1206                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1207                    i, bp->msix_table[offset].vector);
1208
1209                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1210         }
1211 }
1212
1213 void bnx2x_free_irq(struct bnx2x *bp)
1214 {
1215         if (bp->flags & USING_MSIX_FLAG)
1216                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1217                                      CNIC_PRESENT + 1);
1218         else if (bp->flags & USING_MSI_FLAG)
1219                 free_irq(bp->pdev->irq, bp->dev);
1220         else
1221                 free_irq(bp->pdev->irq, bp->dev);
1222 }
1223
1224 int bnx2x_enable_msix(struct bnx2x *bp)
1225 {
1226         int msix_vec = 0, i, rc, req_cnt;
1227
1228         bp->msix_table[msix_vec].entry = msix_vec;
1229         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1230            bp->msix_table[0].entry);
1231         msix_vec++;
1232
1233 #ifdef BCM_CNIC
1234         bp->msix_table[msix_vec].entry = msix_vec;
1235         BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1236            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1237         msix_vec++;
1238 #endif
1239         /* We need separate vectors for ETH queues only (not FCoE) */
1240         for_each_eth_queue(bp, i) {
1241                 bp->msix_table[msix_vec].entry = msix_vec;
1242                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1243                                msix_vec, msix_vec, i);
1244                 msix_vec++;
1245         }
1246
1247         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1248
1249         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1250
1251         /*
1252          * reconfigure number of tx/rx queues according to available
1253          * MSI-X vectors
1254          */
1255         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1256                 /* how less vectors we will have? */
1257                 int diff = req_cnt - rc;
1258
1259                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1260
1261                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1262
1263                 if (rc) {
1264                         BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1265                         return rc;
1266                 }
1267                 /*
1268                  * decrease number of queues by number of unallocated entries
1269                  */
1270                 bp->num_queues -= diff;
1271
1272                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1273                                   bp->num_queues);
1274         } else if (rc) {
1275                 /* fall to INTx if not enough memory */
1276                 if (rc == -ENOMEM)
1277                         bp->flags |= DISABLE_MSI_FLAG;
1278                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1279                 return rc;
1280         }
1281
1282         bp->flags |= USING_MSIX_FLAG;
1283
1284         return 0;
1285 }
1286
1287 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1288 {
1289         int i, rc, offset = 0;
1290
1291         rc = request_irq(bp->msix_table[offset++].vector,
1292                          bnx2x_msix_sp_int, 0,
1293                          bp->dev->name, bp->dev);
1294         if (rc) {
1295                 BNX2X_ERR("request sp irq failed\n");
1296                 return -EBUSY;
1297         }
1298
1299 #ifdef BCM_CNIC
1300         offset++;
1301 #endif
1302         for_each_eth_queue(bp, i) {
1303                 struct bnx2x_fastpath *fp = &bp->fp[i];
1304                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1305                          bp->dev->name, i);
1306
1307                 rc = request_irq(bp->msix_table[offset].vector,
1308                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1309                 if (rc) {
1310                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1311                               bp->msix_table[offset].vector, rc);
1312                         bnx2x_free_msix_irqs(bp, offset);
1313                         return -EBUSY;
1314                 }
1315
1316                 offset++;
1317         }
1318
1319         i = BNX2X_NUM_ETH_QUEUES(bp);
1320         offset = 1 + CNIC_PRESENT;
1321         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1322                bp->msix_table[0].vector,
1323                0, bp->msix_table[offset].vector,
1324                i - 1, bp->msix_table[offset + i - 1].vector);
1325
1326         return 0;
1327 }
1328
1329 int bnx2x_enable_msi(struct bnx2x *bp)
1330 {
1331         int rc;
1332
1333         rc = pci_enable_msi(bp->pdev);
1334         if (rc) {
1335                 BNX2X_DEV_INFO("MSI is not attainable\n");
1336                 return -1;
1337         }
1338         bp->flags |= USING_MSI_FLAG;
1339
1340         return 0;
1341 }
1342
1343 static int bnx2x_req_irq(struct bnx2x *bp)
1344 {
1345         unsigned long flags;
1346         int rc;
1347
1348         if (bp->flags & USING_MSI_FLAG)
1349                 flags = 0;
1350         else
1351                 flags = IRQF_SHARED;
1352
1353         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1354                          bp->dev->name, bp->dev);
1355         return rc;
1356 }
1357
1358 static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1359 {
1360         int rc = 0;
1361         if (bp->flags & USING_MSIX_FLAG) {
1362                 rc = bnx2x_req_msix_irqs(bp);
1363                 if (rc)
1364                         return rc;
1365         } else {
1366                 bnx2x_ack_int(bp);
1367                 rc = bnx2x_req_irq(bp);
1368                 if (rc) {
1369                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1370                         return rc;
1371                 }
1372                 if (bp->flags & USING_MSI_FLAG) {
1373                         bp->dev->irq = bp->pdev->irq;
1374                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
1375                                bp->pdev->irq);
1376                 }
1377         }
1378
1379         return 0;
1380 }
1381
1382 static inline void bnx2x_napi_enable(struct bnx2x *bp)
1383 {
1384         int i;
1385
1386         for_each_rx_queue(bp, i)
1387                 napi_enable(&bnx2x_fp(bp, i, napi));
1388 }
1389
1390 static inline void bnx2x_napi_disable(struct bnx2x *bp)
1391 {
1392         int i;
1393
1394         for_each_rx_queue(bp, i)
1395                 napi_disable(&bnx2x_fp(bp, i, napi));
1396 }
1397
1398 void bnx2x_netif_start(struct bnx2x *bp)
1399 {
1400         if (netif_running(bp->dev)) {
1401                 bnx2x_napi_enable(bp);
1402                 bnx2x_int_enable(bp);
1403                 if (bp->state == BNX2X_STATE_OPEN)
1404                         netif_tx_wake_all_queues(bp->dev);
1405         }
1406 }
1407
1408 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1409 {
1410         bnx2x_int_disable_sync(bp, disable_hw);
1411         bnx2x_napi_disable(bp);
1412 }
1413
1414 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1415 {
1416         struct bnx2x *bp = netdev_priv(dev);
1417
1418 #ifdef BCM_CNIC
1419         if (!NO_FCOE(bp)) {
1420                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1421                 u16 ether_type = ntohs(hdr->h_proto);
1422
1423                 /* Skip VLAN tag if present */
1424                 if (ether_type == ETH_P_8021Q) {
1425                         struct vlan_ethhdr *vhdr =
1426                                 (struct vlan_ethhdr *)skb->data;
1427
1428                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1429                 }
1430
1431                 /* If ethertype is FCoE or FIP - use FCoE ring */
1432                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1433                         return bnx2x_fcoe_tx(bp, txq_index);
1434         }
1435 #endif
1436         /* select a non-FCoE queue */
1437         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1438 }
1439
1440 void bnx2x_set_num_queues(struct bnx2x *bp)
1441 {
1442         switch (bp->multi_mode) {
1443         case ETH_RSS_MODE_DISABLED:
1444                 bp->num_queues = 1;
1445                 break;
1446         case ETH_RSS_MODE_REGULAR:
1447                 bp->num_queues = bnx2x_calc_num_queues(bp);
1448                 break;
1449
1450         default:
1451                 bp->num_queues = 1;
1452                 break;
1453         }
1454
1455 #ifdef BCM_CNIC
1456         /* override in STORAGE SD mode */
1457         if (IS_MF_STORAGE_SD(bp))
1458                 bp->num_queues = 1;
1459 #endif
1460         /* Add special queues */
1461         bp->num_queues += NON_ETH_CONTEXT_USE;
1462 }
1463
1464 /**
1465  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1466  *
1467  * @bp:         Driver handle
1468  *
1469  * We currently support for at most 16 Tx queues for each CoS thus we will
1470  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1471  * bp->max_cos.
1472  *
1473  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1474  * index after all ETH L2 indices.
1475  *
1476  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1477  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1478  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1479  *
1480  * The proper configuration of skb->queue_mapping is handled by
1481  * bnx2x_select_queue() and __skb_tx_hash().
1482  *
1483  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1484  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1485  */
1486 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1487 {
1488         int rc, tx, rx;
1489
1490         tx = MAX_TXQS_PER_COS * bp->max_cos;
1491         rx = BNX2X_NUM_ETH_QUEUES(bp);
1492
1493 /* account for fcoe queue */
1494 #ifdef BCM_CNIC
1495         if (!NO_FCOE(bp)) {
1496                 rx += FCOE_PRESENT;
1497                 tx += FCOE_PRESENT;
1498         }
1499 #endif
1500
1501         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1502         if (rc) {
1503                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1504                 return rc;
1505         }
1506         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1507         if (rc) {
1508                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1509                 return rc;
1510         }
1511
1512         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1513                           tx, rx);
1514
1515         return rc;
1516 }
1517
1518 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1519 {
1520         int i;
1521
1522         for_each_queue(bp, i) {
1523                 struct bnx2x_fastpath *fp = &bp->fp[i];
1524                 u32 mtu;
1525
1526                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1527                 if (IS_FCOE_IDX(i))
1528                         /*
1529                          * Although there are no IP frames expected to arrive to
1530                          * this ring we still want to add an
1531                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1532                          * overrun attack.
1533                          */
1534                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1535                 else
1536                         mtu = bp->dev->mtu;
1537                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1538                                   IP_HEADER_ALIGNMENT_PADDING +
1539                                   ETH_OVREHEAD +
1540                                   mtu +
1541                                   BNX2X_FW_RX_ALIGN_END;
1542                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1543         }
1544 }
1545
1546 static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1547 {
1548         int i;
1549         u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1550         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1551
1552         /*
1553          * Prepare the inital contents fo the indirection table if RSS is
1554          * enabled
1555          */
1556         if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1557                 for (i = 0; i < sizeof(ind_table); i++)
1558                         ind_table[i] =
1559                                 bp->fp->cl_id +
1560                                 ethtool_rxfh_indir_default(i, num_eth_queues);
1561         }
1562
1563         /*
1564          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1565          * per-port, so if explicit configuration is needed , do it only
1566          * for a PMF.
1567          *
1568          * For 57712 and newer on the other hand it's a per-function
1569          * configuration.
1570          */
1571         return bnx2x_config_rss_pf(bp, ind_table,
1572                                    bp->port.pmf || !CHIP_IS_E1x(bp));
1573 }
1574
1575 int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
1576 {
1577         struct bnx2x_config_rss_params params = {NULL};
1578         int i;
1579
1580         /* Although RSS is meaningless when there is a single HW queue we
1581          * still need it enabled in order to have HW Rx hash generated.
1582          *
1583          * if (!is_eth_multi(bp))
1584          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1585          */
1586
1587         params.rss_obj = &bp->rss_conf_obj;
1588
1589         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1590
1591         /* RSS mode */
1592         switch (bp->multi_mode) {
1593         case ETH_RSS_MODE_DISABLED:
1594                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
1595                 break;
1596         case ETH_RSS_MODE_REGULAR:
1597                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1598                 break;
1599         case ETH_RSS_MODE_VLAN_PRI:
1600                 __set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
1601                 break;
1602         case ETH_RSS_MODE_E1HOV_PRI:
1603                 __set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
1604                 break;
1605         case ETH_RSS_MODE_IP_DSCP:
1606                 __set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
1607                 break;
1608         default:
1609                 BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
1610                 return -EINVAL;
1611         }
1612
1613         /* If RSS is enabled */
1614         if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1615                 /* RSS configuration */
1616                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1617                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1618                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1619                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1620
1621                 /* Hash bits */
1622                 params.rss_result_mask = MULTI_MASK;
1623
1624                 memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1625
1626                 if (config_hash) {
1627                         /* RSS keys */
1628                         for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1629                                 params.rss_key[i] = random32();
1630
1631                         __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1632                 }
1633         }
1634
1635         return bnx2x_config_rss(bp, &params);
1636 }
1637
1638 static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1639 {
1640         struct bnx2x_func_state_params func_params = {NULL};
1641
1642         /* Prepare parameters for function state transitions */
1643         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1644
1645         func_params.f_obj = &bp->func_obj;
1646         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1647
1648         func_params.params.hw_init.load_phase = load_code;
1649
1650         return bnx2x_func_state_change(bp, &func_params);
1651 }
1652
1653 /*
1654  * Cleans the object that have internal lists without sending
1655  * ramrods. Should be run when interrutps are disabled.
1656  */
1657 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1658 {
1659         int rc;
1660         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1661         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1662         struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1663
1664         /***************** Cleanup MACs' object first *************************/
1665
1666         /* Wait for completion of requested */
1667         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1668         /* Perform a dry cleanup */
1669         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1670
1671         /* Clean ETH primary MAC */
1672         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1673         rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1674                                  &ramrod_flags);
1675         if (rc != 0)
1676                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1677
1678         /* Cleanup UC list */
1679         vlan_mac_flags = 0;
1680         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1681         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1682                                  &ramrod_flags);
1683         if (rc != 0)
1684                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1685
1686         /***************** Now clean mcast object *****************************/
1687         rparam.mcast_obj = &bp->mcast_obj;
1688         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1689
1690         /* Add a DEL command... */
1691         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1692         if (rc < 0)
1693                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1694                           rc);
1695
1696         /* ...and wait until all pending commands are cleared */
1697         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1698         while (rc != 0) {
1699                 if (rc < 0) {
1700                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1701                                   rc);
1702                         return;
1703                 }
1704
1705                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1706         }
1707 }
1708
1709 #ifndef BNX2X_STOP_ON_ERROR
1710 #define LOAD_ERROR_EXIT(bp, label) \
1711         do { \
1712                 (bp)->state = BNX2X_STATE_ERROR; \
1713                 goto label; \
1714         } while (0)
1715 #else
1716 #define LOAD_ERROR_EXIT(bp, label) \
1717         do { \
1718                 (bp)->state = BNX2X_STATE_ERROR; \
1719                 (bp)->panic = 1; \
1720                 return -EBUSY; \
1721         } while (0)
1722 #endif
1723
1724 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1725 {
1726         /* build FW version dword */
1727         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1728                     (BCM_5710_FW_MINOR_VERSION << 8) +
1729                     (BCM_5710_FW_REVISION_VERSION << 16) +
1730                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1731
1732         /* read loaded FW from chip */
1733         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1734
1735         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1736
1737         if (loaded_fw != my_fw) {
1738                 if (is_err)
1739                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1740                                   loaded_fw, my_fw);
1741                 return false;
1742         }
1743
1744         return true;
1745 }
1746
1747 /* must be called with rtnl_lock */
1748 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1749 {
1750         int port = BP_PORT(bp);
1751         u32 load_code;
1752         int i, rc;
1753
1754 #ifdef BNX2X_STOP_ON_ERROR
1755         if (unlikely(bp->panic)) {
1756                 BNX2X_ERR("Can't load NIC when there is panic\n");
1757                 return -EPERM;
1758         }
1759 #endif
1760
1761         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1762
1763         /* Set the initial link reported state to link down */
1764         bnx2x_acquire_phy_lock(bp);
1765         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1766         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1767                 &bp->last_reported_link.link_report_flags);
1768         bnx2x_release_phy_lock(bp);
1769
1770         /* must be called before memory allocation and HW init */
1771         bnx2x_ilt_set_info(bp);
1772
1773         /*
1774          * Zero fastpath structures preserving invariants like napi, which are
1775          * allocated only once, fp index, max_cos, bp pointer.
1776          * Also set fp->disable_tpa.
1777          */
1778         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
1779         for_each_queue(bp, i)
1780                 bnx2x_bz_fp(bp, i);
1781
1782
1783         /* Set the receive queues buffer size */
1784         bnx2x_set_rx_buf_size(bp);
1785
1786         if (bnx2x_alloc_mem(bp))
1787                 return -ENOMEM;
1788
1789         /* As long as bnx2x_alloc_mem() may possibly update
1790          * bp->num_queues, bnx2x_set_real_num_queues() should always
1791          * come after it.
1792          */
1793         rc = bnx2x_set_real_num_queues(bp);
1794         if (rc) {
1795                 BNX2X_ERR("Unable to set real_num_queues\n");
1796                 LOAD_ERROR_EXIT(bp, load_error0);
1797         }
1798
1799         /* configure multi cos mappings in kernel.
1800          * this configuration may be overriden by a multi class queue discipline
1801          * or by a dcbx negotiation result.
1802          */
1803         bnx2x_setup_tc(bp->dev, bp->max_cos);
1804
1805         bnx2x_napi_enable(bp);
1806
1807         /* set pf load just before approaching the MCP */
1808         bnx2x_set_pf_load(bp);
1809
1810         /* Send LOAD_REQUEST command to MCP
1811          * Returns the type of LOAD command:
1812          * if it is the first port to be initialized
1813          * common blocks should be initialized, otherwise - not
1814          */
1815         if (!BP_NOMCP(bp)) {
1816                 /* init fw_seq */
1817                 bp->fw_seq =
1818                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
1819                          DRV_MSG_SEQ_NUMBER_MASK);
1820                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
1821
1822                 /* Get current FW pulse sequence */
1823                 bp->fw_drv_pulse_wr_seq =
1824                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
1825                          DRV_PULSE_SEQ_MASK);
1826                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
1827
1828                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1829                 if (!load_code) {
1830                         BNX2X_ERR("MCP response failure, aborting\n");
1831                         rc = -EBUSY;
1832                         LOAD_ERROR_EXIT(bp, load_error1);
1833                 }
1834                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1835                         BNX2X_ERR("Driver load refused\n");
1836                         rc = -EBUSY; /* other port in diagnostic mode */
1837                         LOAD_ERROR_EXIT(bp, load_error1);
1838                 }
1839                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
1840                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
1841                         /* abort nic load if version mismatch */
1842                         if (!bnx2x_test_firmware_version(bp, true)) {
1843                                 rc = -EBUSY;
1844                                 LOAD_ERROR_EXIT(bp, load_error2);
1845                         }
1846                 }
1847
1848         } else {
1849                 int path = BP_PATH(bp);
1850
1851                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1852                    path, load_count[path][0], load_count[path][1],
1853                    load_count[path][2]);
1854                 load_count[path][0]++;
1855                 load_count[path][1 + port]++;
1856                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1857                    path, load_count[path][0], load_count[path][1],
1858                    load_count[path][2]);
1859                 if (load_count[path][0] == 1)
1860                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1861                 else if (load_count[path][1 + port] == 1)
1862                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1863                 else
1864                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1865         }
1866
1867         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1868             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1869             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1870                 bp->port.pmf = 1;
1871                 /*
1872                  * We need the barrier to ensure the ordering between the
1873                  * writing to bp->port.pmf here and reading it from the
1874                  * bnx2x_periodic_task().
1875                  */
1876                 smp_mb();
1877                 queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
1878         } else
1879                 bp->port.pmf = 0;
1880
1881         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
1882
1883         /* Init Function state controlling object */
1884         bnx2x__init_func_obj(bp);
1885
1886         /* Initialize HW */
1887         rc = bnx2x_init_hw(bp, load_code);
1888         if (rc) {
1889                 BNX2X_ERR("HW init failed, aborting\n");
1890                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1891                 LOAD_ERROR_EXIT(bp, load_error2);
1892         }
1893
1894         /* Connect to IRQs */
1895         rc = bnx2x_setup_irqs(bp);
1896         if (rc) {
1897                 BNX2X_ERR("IRQs setup failed\n");
1898                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1899                 LOAD_ERROR_EXIT(bp, load_error2);
1900         }
1901
1902         /* Setup NIC internals and enable interrupts */
1903         bnx2x_nic_init(bp, load_code);
1904
1905         /* Init per-function objects */
1906         bnx2x_init_bp_objs(bp);
1907
1908         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1909             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1910             (bp->common.shmem2_base)) {
1911                 if (SHMEM2_HAS(bp, dcc_support))
1912                         SHMEM2_WR(bp, dcc_support,
1913                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1914                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1915         }
1916
1917         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1918         rc = bnx2x_func_start(bp);
1919         if (rc) {
1920                 BNX2X_ERR("Function start failed!\n");
1921                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1922                 LOAD_ERROR_EXIT(bp, load_error3);
1923         }
1924
1925         /* Send LOAD_DONE command to MCP */
1926         if (!BP_NOMCP(bp)) {
1927                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1928                 if (!load_code) {
1929                         BNX2X_ERR("MCP response failure, aborting\n");
1930                         rc = -EBUSY;
1931                         LOAD_ERROR_EXIT(bp, load_error3);
1932                 }
1933         }
1934
1935         rc = bnx2x_setup_leading(bp);
1936         if (rc) {
1937                 BNX2X_ERR("Setup leading failed!\n");
1938                 LOAD_ERROR_EXIT(bp, load_error3);
1939         }
1940
1941 #ifdef BCM_CNIC
1942         /* Enable Timer scan */
1943         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1944 #endif
1945
1946         for_each_nondefault_queue(bp, i) {
1947                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1948                 if (rc) {
1949                         BNX2X_ERR("Queue setup failed\n");
1950                         LOAD_ERROR_EXIT(bp, load_error4);
1951                 }
1952         }
1953
1954         rc = bnx2x_init_rss_pf(bp);
1955         if (rc) {
1956                 BNX2X_ERR("PF RSS init failed\n");
1957                 LOAD_ERROR_EXIT(bp, load_error4);
1958         }
1959
1960         /* Now when Clients are configured we are ready to work */
1961         bp->state = BNX2X_STATE_OPEN;
1962
1963         /* Configure a ucast MAC */
1964         rc = bnx2x_set_eth_mac(bp, true);
1965         if (rc) {
1966                 BNX2X_ERR("Setting Ethernet MAC failed\n");
1967                 LOAD_ERROR_EXIT(bp, load_error4);
1968         }
1969
1970         if (bp->pending_max) {
1971                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1972                 bp->pending_max = 0;
1973         }
1974
1975         if (bp->port.pmf)
1976                 bnx2x_initial_phy_init(bp, load_mode);
1977
1978         /* Start fast path */
1979
1980         /* Initialize Rx filter. */
1981         netif_addr_lock_bh(bp->dev);
1982         bnx2x_set_rx_mode(bp->dev);
1983         netif_addr_unlock_bh(bp->dev);
1984
1985         /* Start the Tx */
1986         switch (load_mode) {
1987         case LOAD_NORMAL:
1988                 /* Tx queue should be only reenabled */
1989                 netif_tx_wake_all_queues(bp->dev);
1990                 break;
1991
1992         case LOAD_OPEN:
1993                 netif_tx_start_all_queues(bp->dev);
1994                 smp_mb__after_clear_bit();
1995                 break;
1996
1997         case LOAD_DIAG:
1998                 bp->state = BNX2X_STATE_DIAG;
1999                 break;
2000
2001         default:
2002                 break;
2003         }
2004
2005         if (bp->port.pmf)
2006                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_DCB_CONFIGURED, 0);
2007         else
2008                 bnx2x__link_status_update(bp);
2009
2010         /* start the timer */
2011         mod_timer(&bp->timer, jiffies + bp->current_interval);
2012
2013 #ifdef BCM_CNIC
2014         /* re-read iscsi info */
2015         bnx2x_get_iscsi_info(bp);
2016         bnx2x_setup_cnic_irq_info(bp);
2017         if (bp->state == BNX2X_STATE_OPEN)
2018                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2019 #endif
2020
2021         /* mark driver is loaded in shmem2 */
2022         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2023                 u32 val;
2024                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2025                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2026                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2027                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2028         }
2029
2030         /* Wait for all pending SP commands to complete */
2031         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2032                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2033                 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2034                 return -EBUSY;
2035         }
2036
2037         bnx2x_dcbx_init(bp);
2038         return 0;
2039
2040 #ifndef BNX2X_STOP_ON_ERROR
2041 load_error4:
2042 #ifdef BCM_CNIC
2043         /* Disable Timer scan */
2044         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2045 #endif
2046 load_error3:
2047         bnx2x_int_disable_sync(bp, 1);
2048
2049         /* Clean queueable objects */
2050         bnx2x_squeeze_objects(bp);
2051
2052         /* Free SKBs, SGEs, TPA pool and driver internals */
2053         bnx2x_free_skbs(bp);
2054         for_each_rx_queue(bp, i)
2055                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2056
2057         /* Release IRQs */
2058         bnx2x_free_irq(bp);
2059 load_error2:
2060         if (!BP_NOMCP(bp)) {
2061                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2062                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2063         }
2064
2065         bp->port.pmf = 0;
2066 load_error1:
2067         bnx2x_napi_disable(bp);
2068         /* clear pf_load status, as it was already set */
2069         bnx2x_clear_pf_load(bp);
2070 load_error0:
2071         bnx2x_free_mem(bp);
2072
2073         return rc;
2074 #endif /* ! BNX2X_STOP_ON_ERROR */
2075 }
2076
2077 /* must be called with rtnl_lock */
2078 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
2079 {
2080         int i;
2081         bool global = false;
2082
2083         /* mark driver is unloaded in shmem2 */
2084         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2085                 u32 val;
2086                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2087                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2088                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2089         }
2090
2091         if ((bp->state == BNX2X_STATE_CLOSED) ||
2092             (bp->state == BNX2X_STATE_ERROR)) {
2093                 /* We can get here if the driver has been unloaded
2094                  * during parity error recovery and is either waiting for a
2095                  * leader to complete or for other functions to unload and
2096                  * then ifdown has been issued. In this case we want to
2097                  * unload and let other functions to complete a recovery
2098                  * process.
2099                  */
2100                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2101                 bp->is_leader = 0;
2102                 bnx2x_release_leader_lock(bp);
2103                 smp_mb();
2104
2105                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2106                 BNX2X_ERR("Can't unload in closed or error state\n");
2107                 return -EINVAL;
2108         }
2109
2110         /*
2111          * It's important to set the bp->state to the value different from
2112          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2113          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2114          */
2115         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2116         smp_mb();
2117
2118         /* Stop Tx */
2119         bnx2x_tx_disable(bp);
2120
2121 #ifdef BCM_CNIC
2122         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2123 #endif
2124
2125         bp->rx_mode = BNX2X_RX_MODE_NONE;
2126
2127         del_timer_sync(&bp->timer);
2128
2129         /* Set ALWAYS_ALIVE bit in shmem */
2130         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2131
2132         bnx2x_drv_pulse(bp);
2133
2134         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2135         bnx2x_save_statistics(bp);
2136
2137         /* Cleanup the chip if needed */
2138         if (unload_mode != UNLOAD_RECOVERY)
2139                 bnx2x_chip_cleanup(bp, unload_mode);
2140         else {
2141                 /* Send the UNLOAD_REQUEST to the MCP */
2142                 bnx2x_send_unload_req(bp, unload_mode);
2143
2144                 /*
2145                  * Prevent transactions to host from the functions on the
2146                  * engine that doesn't reset global blocks in case of global
2147                  * attention once gloabl blocks are reset and gates are opened
2148                  * (the engine which leader will perform the recovery
2149                  * last).
2150                  */
2151                 if (!CHIP_IS_E1x(bp))
2152                         bnx2x_pf_disable(bp);
2153
2154                 /* Disable HW interrupts, NAPI */
2155                 bnx2x_netif_stop(bp, 1);
2156
2157                 /* Release IRQs */
2158                 bnx2x_free_irq(bp);
2159
2160                 /* Report UNLOAD_DONE to MCP */
2161                 bnx2x_send_unload_done(bp);
2162         }
2163
2164         /*
2165          * At this stage no more interrupts will arrive so we may safly clean
2166          * the queueable objects here in case they failed to get cleaned so far.
2167          */
2168         bnx2x_squeeze_objects(bp);
2169
2170         /* There should be no more pending SP commands at this stage */
2171         bp->sp_state = 0;
2172
2173         bp->port.pmf = 0;
2174
2175         /* Free SKBs, SGEs, TPA pool and driver internals */
2176         bnx2x_free_skbs(bp);
2177         for_each_rx_queue(bp, i)
2178                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2179
2180         bnx2x_free_mem(bp);
2181
2182         bp->state = BNX2X_STATE_CLOSED;
2183
2184         /* Check if there are pending parity attentions. If there are - set
2185          * RECOVERY_IN_PROGRESS.
2186          */
2187         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2188                 bnx2x_set_reset_in_progress(bp);
2189
2190                 /* Set RESET_IS_GLOBAL if needed */
2191                 if (global)
2192                         bnx2x_set_reset_global(bp);
2193         }
2194
2195
2196         /* The last driver must disable a "close the gate" if there is no
2197          * parity attention or "process kill" pending.
2198          */
2199         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2200                 bnx2x_disable_close_the_gate(bp);
2201
2202         return 0;
2203 }
2204
2205 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2206 {
2207         u16 pmcsr;
2208
2209         /* If there is no power capability, silently succeed */
2210         if (!bp->pm_cap) {
2211                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2212                 return 0;
2213         }
2214
2215         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2216
2217         switch (state) {
2218         case PCI_D0:
2219                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2220                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2221                                        PCI_PM_CTRL_PME_STATUS));
2222
2223                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2224                         /* delay required during transition out of D3hot */
2225                         msleep(20);
2226                 break;
2227
2228         case PCI_D3hot:
2229                 /* If there are other clients above don't
2230                    shut down the power */
2231                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2232                         return 0;
2233                 /* Don't shut down the power for emulation and FPGA */
2234                 if (CHIP_REV_IS_SLOW(bp))
2235                         return 0;
2236
2237                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2238                 pmcsr |= 3;
2239
2240                 if (bp->wol)
2241                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2242
2243                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2244                                       pmcsr);
2245
2246                 /* No more memory access after this point until
2247                 * device is brought back to D0.
2248                 */
2249                 break;
2250
2251         default:
2252                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2253                 return -EINVAL;
2254         }
2255         return 0;
2256 }
2257
2258 /*
2259  * net_device service functions
2260  */
2261 int bnx2x_poll(struct napi_struct *napi, int budget)
2262 {
2263         int work_done = 0;
2264         u8 cos;
2265         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2266                                                  napi);
2267         struct bnx2x *bp = fp->bp;
2268
2269         while (1) {
2270 #ifdef BNX2X_STOP_ON_ERROR
2271                 if (unlikely(bp->panic)) {
2272                         napi_complete(napi);
2273                         return 0;
2274                 }
2275 #endif
2276
2277                 for_each_cos_in_tx_queue(fp, cos)
2278                         if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2279                                 bnx2x_tx_int(bp, &fp->txdata[cos]);
2280
2281
2282                 if (bnx2x_has_rx_work(fp)) {
2283                         work_done += bnx2x_rx_int(fp, budget - work_done);
2284
2285                         /* must not complete if we consumed full budget */
2286                         if (work_done >= budget)
2287                                 break;
2288                 }
2289
2290                 /* Fall out from the NAPI loop if needed */
2291                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2292 #ifdef BCM_CNIC
2293                         /* No need to update SB for FCoE L2 ring as long as
2294                          * it's connected to the default SB and the SB
2295                          * has been updated when NAPI was scheduled.
2296                          */
2297                         if (IS_FCOE_FP(fp)) {
2298                                 napi_complete(napi);
2299                                 break;
2300                         }
2301 #endif
2302
2303                         bnx2x_update_fpsb_idx(fp);
2304                         /* bnx2x_has_rx_work() reads the status block,
2305                          * thus we need to ensure that status block indices
2306                          * have been actually read (bnx2x_update_fpsb_idx)
2307                          * prior to this check (bnx2x_has_rx_work) so that
2308                          * we won't write the "newer" value of the status block
2309                          * to IGU (if there was a DMA right after
2310                          * bnx2x_has_rx_work and if there is no rmb, the memory
2311                          * reading (bnx2x_update_fpsb_idx) may be postponed
2312                          * to right before bnx2x_ack_sb). In this case there
2313                          * will never be another interrupt until there is
2314                          * another update of the status block, while there
2315                          * is still unhandled work.
2316                          */
2317                         rmb();
2318
2319                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2320                                 napi_complete(napi);
2321                                 /* Re-enable interrupts */
2322                                 DP(NETIF_MSG_RX_STATUS,
2323                                    "Update index to %d\n", fp->fp_hc_idx);
2324                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2325                                              le16_to_cpu(fp->fp_hc_idx),
2326                                              IGU_INT_ENABLE, 1);
2327                                 break;
2328                         }
2329                 }
2330         }
2331
2332         return work_done;
2333 }
2334
2335 /* we split the first BD into headers and data BDs
2336  * to ease the pain of our fellow microcode engineers
2337  * we use one mapping for both BDs
2338  * So far this has only been observed to happen
2339  * in Other Operating Systems(TM)
2340  */
2341 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2342                                    struct bnx2x_fp_txdata *txdata,
2343                                    struct sw_tx_bd *tx_buf,
2344                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2345                                    u16 bd_prod, int nbd)
2346 {
2347         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2348         struct eth_tx_bd *d_tx_bd;
2349         dma_addr_t mapping;
2350         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2351
2352         /* first fix first BD */
2353         h_tx_bd->nbd = cpu_to_le16(nbd);
2354         h_tx_bd->nbytes = cpu_to_le16(hlen);
2355
2356         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2357            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2358
2359         /* now get a new data BD
2360          * (after the pbd) and fill it */
2361         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2362         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2363
2364         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2365                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2366
2367         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2368         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2369         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2370
2371         /* this marks the BD as one that has no individual mapping */
2372         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2373
2374         DP(NETIF_MSG_TX_QUEUED,
2375            "TSO split data size is %d (%x:%x)\n",
2376            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2377
2378         /* update tx_bd */
2379         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2380
2381         return bd_prod;
2382 }
2383
2384 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2385 {
2386         if (fix > 0)
2387                 csum = (u16) ~csum_fold(csum_sub(csum,
2388                                 csum_partial(t_header - fix, fix, 0)));
2389
2390         else if (fix < 0)
2391                 csum = (u16) ~csum_fold(csum_add(csum,
2392                                 csum_partial(t_header, -fix, 0)));
2393
2394         return swab16(csum);
2395 }
2396
2397 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2398 {
2399         u32 rc;
2400
2401         if (skb->ip_summed != CHECKSUM_PARTIAL)
2402                 rc = XMIT_PLAIN;
2403
2404         else {
2405                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2406                         rc = XMIT_CSUM_V6;
2407                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2408                                 rc |= XMIT_CSUM_TCP;
2409
2410                 } else {
2411                         rc = XMIT_CSUM_V4;
2412                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2413                                 rc |= XMIT_CSUM_TCP;
2414                 }
2415         }
2416
2417         if (skb_is_gso_v6(skb))
2418                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2419         else if (skb_is_gso(skb))
2420                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2421
2422         return rc;
2423 }
2424
2425 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2426 /* check if packet requires linearization (packet is too fragmented)
2427    no need to check fragmentation if page size > 8K (there will be no
2428    violation to FW restrictions) */
2429 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2430                              u32 xmit_type)
2431 {
2432         int to_copy = 0;
2433         int hlen = 0;
2434         int first_bd_sz = 0;
2435
2436         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2437         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2438
2439                 if (xmit_type & XMIT_GSO) {
2440                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2441                         /* Check if LSO packet needs to be copied:
2442                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2443                         int wnd_size = MAX_FETCH_BD - 3;
2444                         /* Number of windows to check */
2445                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2446                         int wnd_idx = 0;
2447                         int frag_idx = 0;
2448                         u32 wnd_sum = 0;
2449
2450                         /* Headers length */
2451                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2452                                 tcp_hdrlen(skb);
2453
2454                         /* Amount of data (w/o headers) on linear part of SKB*/
2455                         first_bd_sz = skb_headlen(skb) - hlen;
2456
2457                         wnd_sum  = first_bd_sz;
2458
2459                         /* Calculate the first sum - it's special */
2460                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2461                                 wnd_sum +=
2462                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2463
2464                         /* If there was data on linear skb data - check it */
2465                         if (first_bd_sz > 0) {
2466                                 if (unlikely(wnd_sum < lso_mss)) {
2467                                         to_copy = 1;
2468                                         goto exit_lbl;
2469                                 }
2470
2471                                 wnd_sum -= first_bd_sz;
2472                         }
2473
2474                         /* Others are easier: run through the frag list and
2475                            check all windows */
2476                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2477                                 wnd_sum +=
2478                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2479
2480                                 if (unlikely(wnd_sum < lso_mss)) {
2481                                         to_copy = 1;
2482                                         break;
2483                                 }
2484                                 wnd_sum -=
2485                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2486                         }
2487                 } else {
2488                         /* in non-LSO too fragmented packet should always
2489                            be linearized */
2490                         to_copy = 1;
2491                 }
2492         }
2493
2494 exit_lbl:
2495         if (unlikely(to_copy))
2496                 DP(NETIF_MSG_TX_QUEUED,
2497                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2498                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2499                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2500
2501         return to_copy;
2502 }
2503 #endif
2504
2505 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2506                                         u32 xmit_type)
2507 {
2508         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2509                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2510                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2511         if ((xmit_type & XMIT_GSO_V6) &&
2512             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2513                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2514 }
2515
2516 /**
2517  * bnx2x_set_pbd_gso - update PBD in GSO case.
2518  *
2519  * @skb:        packet skb
2520  * @pbd:        parse BD
2521  * @xmit_type:  xmit flags
2522  */
2523 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2524                                      struct eth_tx_parse_bd_e1x *pbd,
2525                                      u32 xmit_type)
2526 {
2527         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2528         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2529         pbd->tcp_flags = pbd_tcp_flags(skb);
2530
2531         if (xmit_type & XMIT_GSO_V4) {
2532                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2533                 pbd->tcp_pseudo_csum =
2534                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2535                                                   ip_hdr(skb)->daddr,
2536                                                   0, IPPROTO_TCP, 0));
2537
2538         } else
2539                 pbd->tcp_pseudo_csum =
2540                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2541                                                 &ipv6_hdr(skb)->daddr,
2542                                                 0, IPPROTO_TCP, 0));
2543
2544         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2545 }
2546
2547 /**
2548  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2549  *
2550  * @bp:                 driver handle
2551  * @skb:                packet skb
2552  * @parsing_data:       data to be updated
2553  * @xmit_type:          xmit flags
2554  *
2555  * 57712 related
2556  */
2557 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2558         u32 *parsing_data, u32 xmit_type)
2559 {
2560         *parsing_data |=
2561                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2562                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2563                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2564
2565         if (xmit_type & XMIT_CSUM_TCP) {
2566                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2567                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2568                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2569
2570                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2571         } else
2572                 /* We support checksum offload for TCP and UDP only.
2573                  * No need to pass the UDP header length - it's a constant.
2574                  */
2575                 return skb_transport_header(skb) +
2576                                 sizeof(struct udphdr) - skb->data;
2577 }
2578
2579 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2580         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2581 {
2582         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2583
2584         if (xmit_type & XMIT_CSUM_V4)
2585                 tx_start_bd->bd_flags.as_bitfield |=
2586                                         ETH_TX_BD_FLAGS_IP_CSUM;
2587         else
2588                 tx_start_bd->bd_flags.as_bitfield |=
2589                                         ETH_TX_BD_FLAGS_IPV6;
2590
2591         if (!(xmit_type & XMIT_CSUM_TCP))
2592                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2593 }
2594
2595 /**
2596  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2597  *
2598  * @bp:         driver handle
2599  * @skb:        packet skb
2600  * @pbd:        parse BD to be updated
2601  * @xmit_type:  xmit flags
2602  */
2603 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2604         struct eth_tx_parse_bd_e1x *pbd,
2605         u32 xmit_type)
2606 {
2607         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2608
2609         /* for now NS flag is not used in Linux */
2610         pbd->global_data =
2611                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2612                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2613
2614         pbd->ip_hlen_w = (skb_transport_header(skb) -
2615                         skb_network_header(skb)) >> 1;
2616
2617         hlen += pbd->ip_hlen_w;
2618
2619         /* We support checksum offload for TCP and UDP only */
2620         if (xmit_type & XMIT_CSUM_TCP)
2621                 hlen += tcp_hdrlen(skb) / 2;
2622         else
2623                 hlen += sizeof(struct udphdr) / 2;
2624
2625         pbd->total_hlen_w = cpu_to_le16(hlen);
2626         hlen = hlen*2;
2627
2628         if (xmit_type & XMIT_CSUM_TCP) {
2629                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2630
2631         } else {
2632                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2633
2634                 DP(NETIF_MSG_TX_QUEUED,
2635                    "hlen %d  fix %d  csum before fix %x\n",
2636                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2637
2638                 /* HW bug: fixup the CSUM */
2639                 pbd->tcp_pseudo_csum =
2640                         bnx2x_csum_fix(skb_transport_header(skb),
2641                                        SKB_CS(skb), fix);
2642
2643                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2644                    pbd->tcp_pseudo_csum);
2645         }
2646
2647         return hlen;
2648 }
2649
2650 /* called with netif_tx_lock
2651  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2652  * netif_wake_queue()
2653  */
2654 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2655 {
2656         struct bnx2x *bp = netdev_priv(dev);
2657
2658         struct bnx2x_fastpath *fp;
2659         struct netdev_queue *txq;
2660         struct bnx2x_fp_txdata *txdata;
2661         struct sw_tx_bd *tx_buf;
2662         struct eth_tx_start_bd *tx_start_bd, *first_bd;
2663         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2664         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2665         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2666         u32 pbd_e2_parsing_data = 0;
2667         u16 pkt_prod, bd_prod;
2668         int nbd, txq_index, fp_index, txdata_index;
2669         dma_addr_t mapping;
2670         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2671         int i;
2672         u8 hlen = 0;
2673         __le16 pkt_size = 0;
2674         struct ethhdr *eth;
2675         u8 mac_type = UNICAST_ADDRESS;
2676
2677 #ifdef BNX2X_STOP_ON_ERROR
2678         if (unlikely(bp->panic))
2679                 return NETDEV_TX_BUSY;
2680 #endif
2681
2682         txq_index = skb_get_queue_mapping(skb);
2683         txq = netdev_get_tx_queue(dev, txq_index);
2684
2685         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2686
2687         /* decode the fastpath index and the cos index from the txq */
2688         fp_index = TXQ_TO_FP(txq_index);
2689         txdata_index = TXQ_TO_COS(txq_index);
2690
2691 #ifdef BCM_CNIC
2692         /*
2693          * Override the above for the FCoE queue:
2694          *   - FCoE fp entry is right after the ETH entries.
2695          *   - FCoE L2 queue uses bp->txdata[0] only.
2696          */
2697         if (unlikely(!NO_FCOE(bp) && (txq_index ==
2698                                       bnx2x_fcoe_tx(bp, txq_index)))) {
2699                 fp_index = FCOE_IDX;
2700                 txdata_index = 0;
2701         }
2702 #endif
2703
2704         /* enable this debug print to view the transmission queue being used
2705         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
2706            txq_index, fp_index, txdata_index); */
2707
2708         /* locate the fastpath and the txdata */
2709         fp = &bp->fp[fp_index];
2710         txdata = &fp->txdata[txdata_index];
2711
2712         /* enable this debug print to view the tranmission details
2713         DP(NETIF_MSG_TX_QUEUED,
2714            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
2715            txdata->cid, fp_index, txdata_index, txdata, fp); */
2716
2717         if (unlikely(bnx2x_tx_avail(bp, txdata) <
2718                      (skb_shinfo(skb)->nr_frags + 3))) {
2719                 fp->eth_q_stats.driver_xoff++;
2720                 netif_tx_stop_queue(txq);
2721                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2722                 return NETDEV_TX_BUSY;
2723         }
2724
2725         DP(NETIF_MSG_TX_QUEUED,
2726            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
2727            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2728            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2729
2730         eth = (struct ethhdr *)skb->data;
2731
2732         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2733         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2734                 if (is_broadcast_ether_addr(eth->h_dest))
2735                         mac_type = BROADCAST_ADDRESS;
2736                 else
2737                         mac_type = MULTICAST_ADDRESS;
2738         }
2739
2740 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2741         /* First, check if we need to linearize the skb (due to FW
2742            restrictions). No need to check fragmentation if page size > 8K
2743            (there will be no violation to FW restrictions) */
2744         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2745                 /* Statistics of linearization */
2746                 bp->lin_cnt++;
2747                 if (skb_linearize(skb) != 0) {
2748                         DP(NETIF_MSG_TX_QUEUED,
2749                            "SKB linearization failed - silently dropping this SKB\n");
2750                         dev_kfree_skb_any(skb);
2751                         return NETDEV_TX_OK;
2752                 }
2753         }
2754 #endif
2755         /* Map skb linear data for DMA */
2756         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2757                                  skb_headlen(skb), DMA_TO_DEVICE);
2758         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2759                 DP(NETIF_MSG_TX_QUEUED,
2760                    "SKB mapping failed - silently dropping this SKB\n");
2761                 dev_kfree_skb_any(skb);
2762                 return NETDEV_TX_OK;
2763         }
2764         /*
2765         Please read carefully. First we use one BD which we mark as start,
2766         then we have a parsing info BD (used for TSO or xsum),
2767         and only then we have the rest of the TSO BDs.
2768         (don't forget to mark the last one as last,
2769         and to unmap only AFTER you write to the BD ...)
2770         And above all, all pdb sizes are in words - NOT DWORDS!
2771         */
2772
2773         /* get current pkt produced now - advance it just before sending packet
2774          * since mapping of pages may fail and cause packet to be dropped
2775          */
2776         pkt_prod = txdata->tx_pkt_prod;
2777         bd_prod = TX_BD(txdata->tx_bd_prod);
2778
2779         /* get a tx_buf and first BD
2780          * tx_start_bd may be changed during SPLIT,
2781          * but first_bd will always stay first
2782          */
2783         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2784         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2785         first_bd = tx_start_bd;
2786
2787         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2788         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2789                  mac_type);
2790
2791         /* header nbd */
2792         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2793
2794         /* remember the first BD of the packet */
2795         tx_buf->first_bd = txdata->tx_bd_prod;
2796         tx_buf->skb = skb;
2797         tx_buf->flags = 0;
2798
2799         DP(NETIF_MSG_TX_QUEUED,
2800            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2801            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2802
2803         if (vlan_tx_tag_present(skb)) {
2804                 tx_start_bd->vlan_or_ethertype =
2805                     cpu_to_le16(vlan_tx_tag_get(skb));
2806                 tx_start_bd->bd_flags.as_bitfield |=
2807                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2808         } else
2809                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2810
2811         /* turn on parsing and get a BD */
2812         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2813
2814         if (xmit_type & XMIT_CSUM)
2815                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2816
2817         if (!CHIP_IS_E1x(bp)) {
2818                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
2819                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2820                 /* Set PBD in checksum offload case */
2821                 if (xmit_type & XMIT_CSUM)
2822                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2823                                                      &pbd_e2_parsing_data,
2824                                                      xmit_type);
2825                 if (IS_MF_SI(bp)) {
2826                         /*
2827                          * fill in the MAC addresses in the PBD - for local
2828                          * switching
2829                          */
2830                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2831                                               &pbd_e2->src_mac_addr_mid,
2832                                               &pbd_e2->src_mac_addr_lo,
2833                                               eth->h_source);
2834                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2835                                               &pbd_e2->dst_mac_addr_mid,
2836                                               &pbd_e2->dst_mac_addr_lo,
2837                                               eth->h_dest);
2838                 }
2839         } else {
2840                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
2841                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2842                 /* Set PBD in checksum offload case */
2843                 if (xmit_type & XMIT_CSUM)
2844                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2845
2846         }
2847
2848         /* Setup the data pointer of the first BD of the packet */
2849         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2850         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2851         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2852         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2853         pkt_size = tx_start_bd->nbytes;
2854
2855         DP(NETIF_MSG_TX_QUEUED,
2856            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
2857            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2858            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2859            tx_start_bd->bd_flags.as_bitfield,
2860            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2861
2862         if (xmit_type & XMIT_GSO) {
2863
2864                 DP(NETIF_MSG_TX_QUEUED,
2865                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2866                    skb->len, hlen, skb_headlen(skb),
2867                    skb_shinfo(skb)->gso_size);
2868
2869                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2870
2871                 if (unlikely(skb_headlen(skb) > hlen))
2872                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2873                                                  &tx_start_bd, hlen,
2874                                                  bd_prod, ++nbd);
2875                 if (!CHIP_IS_E1x(bp))
2876                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2877                                              xmit_type);
2878                 else
2879                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2880         }
2881
2882         /* Set the PBD's parsing_data field if not zero
2883          * (for the chips newer than 57711).
2884          */
2885         if (pbd_e2_parsing_data)
2886                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2887
2888         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2889
2890         /* Handle fragmented skb */
2891         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2892                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2893
2894                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
2895                                            skb_frag_size(frag), DMA_TO_DEVICE);
2896                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2897                         unsigned int pkts_compl = 0, bytes_compl = 0;
2898
2899                         DP(NETIF_MSG_TX_QUEUED,
2900                            "Unable to map page - dropping packet...\n");
2901
2902                         /* we need unmap all buffers already mapped
2903                          * for this SKB;
2904                          * first_bd->nbd need to be properly updated
2905                          * before call to bnx2x_free_tx_pkt
2906                          */
2907                         first_bd->nbd = cpu_to_le16(nbd);
2908                         bnx2x_free_tx_pkt(bp, txdata,
2909                                           TX_BD(txdata->tx_pkt_prod),
2910                                           &pkts_compl, &bytes_compl);
2911                         return NETDEV_TX_OK;
2912                 }
2913
2914                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2915                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2916                 if (total_pkt_bd == NULL)
2917                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2918
2919                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2920                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2921                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
2922                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
2923                 nbd++;
2924
2925                 DP(NETIF_MSG_TX_QUEUED,
2926                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2927                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2928                    le16_to_cpu(tx_data_bd->nbytes));
2929         }
2930
2931         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2932
2933         /* update with actual num BDs */
2934         first_bd->nbd = cpu_to_le16(nbd);
2935
2936         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2937
2938         /* now send a tx doorbell, counting the next BD
2939          * if the packet contains or ends with it
2940          */
2941         if (TX_BD_POFF(bd_prod) < nbd)
2942                 nbd++;
2943
2944         /* total_pkt_bytes should be set on the first data BD if
2945          * it's not an LSO packet and there is more than one
2946          * data BD. In this case pkt_size is limited by an MTU value.
2947          * However we prefer to set it for an LSO packet (while we don't
2948          * have to) in order to save some CPU cycles in a none-LSO
2949          * case, when we much more care about them.
2950          */
2951         if (total_pkt_bd != NULL)
2952                 total_pkt_bd->total_pkt_bytes = pkt_size;
2953
2954         if (pbd_e1x)
2955                 DP(NETIF_MSG_TX_QUEUED,
2956                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2957                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2958                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2959                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2960                     le16_to_cpu(pbd_e1x->total_hlen_w));
2961         if (pbd_e2)
2962                 DP(NETIF_MSG_TX_QUEUED,
2963                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2964                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2965                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2966                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2967                    pbd_e2->parsing_data);
2968         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2969
2970         netdev_tx_sent_queue(txq, skb->len);
2971
2972         txdata->tx_pkt_prod++;
2973         /*
2974          * Make sure that the BD data is updated before updating the producer
2975          * since FW might read the BD right after the producer is updated.
2976          * This is only applicable for weak-ordered memory model archs such
2977          * as IA-64. The following barrier is also mandatory since FW will
2978          * assumes packets must have BDs.
2979          */
2980         wmb();
2981
2982         txdata->tx_db.data.prod += nbd;
2983         barrier();
2984
2985         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2986
2987         mmiowb();
2988
2989         txdata->tx_bd_prod += nbd;
2990
2991         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2992                 netif_tx_stop_queue(txq);
2993
2994                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2995                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2996                  * fp->bd_tx_cons */
2997                 smp_mb();
2998
2999                 fp->eth_q_stats.driver_xoff++;
3000                 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
3001                         netif_tx_wake_queue(txq);
3002         }
3003         txdata->tx_pkt++;
3004
3005         return NETDEV_TX_OK;
3006 }
3007
3008 /**
3009  * bnx2x_setup_tc - routine to configure net_device for multi tc
3010  *
3011  * @netdev: net device to configure
3012  * @tc: number of traffic classes to enable
3013  *
3014  * callback connected to the ndo_setup_tc function pointer
3015  */
3016 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3017 {
3018         int cos, prio, count, offset;
3019         struct bnx2x *bp = netdev_priv(dev);
3020
3021         /* setup tc must be called under rtnl lock */
3022         ASSERT_RTNL();
3023
3024         /* no traffic classes requested. aborting */
3025         if (!num_tc) {
3026                 netdev_reset_tc(dev);
3027                 return 0;
3028         }
3029
3030         /* requested to support too many traffic classes */
3031         if (num_tc > bp->max_cos) {
3032                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3033                           num_tc, bp->max_cos);
3034                 return -EINVAL;
3035         }
3036
3037         /* declare amount of supported traffic classes */
3038         if (netdev_set_num_tc(dev, num_tc)) {
3039                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3040                 return -EINVAL;
3041         }
3042
3043         /* configure priority to traffic class mapping */
3044         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3045                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3046                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3047                    "mapping priority %d to tc %d\n",
3048                    prio, bp->prio_to_cos[prio]);
3049         }
3050
3051
3052         /* Use this configuration to diffrentiate tc0 from other COSes
3053            This can be used for ets or pfc, and save the effort of setting
3054            up a multio class queue disc or negotiating DCBX with a switch
3055         netdev_set_prio_tc_map(dev, 0, 0);
3056         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3057         for (prio = 1; prio < 16; prio++) {
3058                 netdev_set_prio_tc_map(dev, prio, 1);
3059                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3060         } */
3061
3062         /* configure traffic class to transmission queue mapping */
3063         for (cos = 0; cos < bp->max_cos; cos++) {
3064                 count = BNX2X_NUM_ETH_QUEUES(bp);
3065                 offset = cos * MAX_TXQS_PER_COS;
3066                 netdev_set_tc_queue(dev, cos, count, offset);
3067                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3068                    "mapping tc %d to offset %d count %d\n",
3069                    cos, offset, count);
3070         }
3071
3072         return 0;
3073 }
3074
3075 /* called with rtnl_lock */
3076 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3077 {
3078         struct sockaddr *addr = p;
3079         struct bnx2x *bp = netdev_priv(dev);
3080         int rc = 0;
3081
3082         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3083                 BNX2X_ERR("Requested MAC address is not valid\n");
3084                 return -EINVAL;
3085         }
3086
3087 #ifdef BCM_CNIC
3088         if (IS_MF_STORAGE_SD(bp) && !is_zero_ether_addr(addr->sa_data)) {
3089                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3090                 return -EINVAL;
3091         }
3092 #endif
3093
3094         if (netif_running(dev))  {
3095                 rc = bnx2x_set_eth_mac(bp, false);
3096                 if (rc)
3097                         return rc;
3098         }
3099
3100         dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3101         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3102
3103         if (netif_running(dev))
3104                 rc = bnx2x_set_eth_mac(bp, true);
3105
3106         return rc;
3107 }
3108
3109 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3110 {
3111         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3112         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3113         u8 cos;
3114
3115         /* Common */
3116 #ifdef BCM_CNIC
3117         if (IS_FCOE_IDX(fp_index)) {
3118                 memset(sb, 0, sizeof(union host_hc_status_block));
3119                 fp->status_blk_mapping = 0;
3120
3121         } else {
3122 #endif
3123                 /* status blocks */
3124                 if (!CHIP_IS_E1x(bp))
3125                         BNX2X_PCI_FREE(sb->e2_sb,
3126                                        bnx2x_fp(bp, fp_index,
3127                                                 status_blk_mapping),
3128                                        sizeof(struct host_hc_status_block_e2));
3129                 else
3130                         BNX2X_PCI_FREE(sb->e1x_sb,
3131                                        bnx2x_fp(bp, fp_index,
3132                                                 status_blk_mapping),
3133                                        sizeof(struct host_hc_status_block_e1x));
3134 #ifdef BCM_CNIC
3135         }
3136 #endif
3137         /* Rx */
3138         if (!skip_rx_queue(bp, fp_index)) {
3139                 bnx2x_free_rx_bds(fp);
3140
3141                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3142                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3143                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3144                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3145                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3146
3147                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3148                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3149                                sizeof(struct eth_fast_path_rx_cqe) *
3150                                NUM_RCQ_BD);
3151
3152                 /* SGE ring */
3153                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3154                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3155                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3156                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3157         }
3158
3159         /* Tx */
3160         if (!skip_tx_queue(bp, fp_index)) {
3161                 /* fastpath tx rings: tx_buf tx_desc */
3162                 for_each_cos_in_tx_queue(fp, cos) {
3163                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3164
3165                         DP(NETIF_MSG_IFDOWN,
3166                            "freeing tx memory of fp %d cos %d cid %d\n",
3167                            fp_index, cos, txdata->cid);
3168
3169                         BNX2X_FREE(txdata->tx_buf_ring);
3170                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3171                                 txdata->tx_desc_mapping,
3172                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3173                 }
3174         }
3175         /* end of fastpath */
3176 }
3177
3178 void bnx2x_free_fp_mem(struct bnx2x *bp)
3179 {
3180         int i;
3181         for_each_queue(bp, i)
3182                 bnx2x_free_fp_mem_at(bp, i);
3183 }
3184
3185 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3186 {
3187         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3188         if (!CHIP_IS_E1x(bp)) {
3189                 bnx2x_fp(bp, index, sb_index_values) =
3190                         (__le16 *)status_blk.e2_sb->sb.index_values;
3191                 bnx2x_fp(bp, index, sb_running_index) =
3192                         (__le16 *)status_blk.e2_sb->sb.running_index;
3193         } else {
3194                 bnx2x_fp(bp, index, sb_index_values) =
3195                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3196                 bnx2x_fp(bp, index, sb_running_index) =
3197                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3198         }
3199 }
3200
3201 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3202 {
3203         union host_hc_status_block *sb;
3204         struct bnx2x_fastpath *fp = &bp->fp[index];
3205         int ring_size = 0;
3206         u8 cos;
3207         int rx_ring_size = 0;
3208
3209 #ifdef BCM_CNIC
3210         if (!bp->rx_ring_size && IS_MF_STORAGE_SD(bp)) {
3211                 rx_ring_size = MIN_RX_SIZE_NONTPA;
3212                 bp->rx_ring_size = rx_ring_size;
3213         } else
3214 #endif
3215         if (!bp->rx_ring_size) {
3216                 u32 cfg = SHMEM_RD(bp,
3217                              dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3218
3219                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3220
3221                 /* Dercease ring size for 1G functions */
3222                 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3223                     PORT_HW_CFG_NET_SERDES_IF_SGMII)
3224                         rx_ring_size /= 10;
3225
3226                 /* allocate at least number of buffers required by FW */
3227                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3228                                      MIN_RX_SIZE_TPA, rx_ring_size);
3229
3230                 bp->rx_ring_size = rx_ring_size;
3231         } else /* if rx_ring_size specified - use it */
3232                 rx_ring_size = bp->rx_ring_size;
3233
3234         /* Common */
3235         sb = &bnx2x_fp(bp, index, status_blk);
3236 #ifdef BCM_CNIC
3237         if (!IS_FCOE_IDX(index)) {
3238 #endif
3239                 /* status blocks */
3240                 if (!CHIP_IS_E1x(bp))
3241                         BNX2X_PCI_ALLOC(sb->e2_sb,
3242                                 &bnx2x_fp(bp, index, status_blk_mapping),
3243                                 sizeof(struct host_hc_status_block_e2));
3244                 else
3245                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3246                                 &bnx2x_fp(bp, index, status_blk_mapping),
3247                             sizeof(struct host_hc_status_block_e1x));
3248 #ifdef BCM_CNIC
3249         }
3250 #endif
3251
3252         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3253          * set shortcuts for it.
3254          */
3255         if (!IS_FCOE_IDX(index))
3256                 set_sb_shortcuts(bp, index);
3257
3258         /* Tx */
3259         if (!skip_tx_queue(bp, index)) {
3260                 /* fastpath tx rings: tx_buf tx_desc */
3261                 for_each_cos_in_tx_queue(fp, cos) {
3262                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3263
3264                         DP(NETIF_MSG_IFUP,
3265                            "allocating tx memory of fp %d cos %d\n",
3266                            index, cos);
3267
3268                         BNX2X_ALLOC(txdata->tx_buf_ring,
3269                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3270                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3271                                 &txdata->tx_desc_mapping,
3272                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3273                 }
3274         }
3275
3276         /* Rx */
3277         if (!skip_rx_queue(bp, index)) {
3278                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3279                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3280                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3281                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3282                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3283                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3284
3285                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3286                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3287                                 sizeof(struct eth_fast_path_rx_cqe) *
3288                                 NUM_RCQ_BD);
3289
3290                 /* SGE ring */
3291                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3292                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3293                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3294                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3295                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3296                 /* RX BD ring */
3297                 bnx2x_set_next_page_rx_bd(fp);
3298
3299                 /* CQ ring */
3300                 bnx2x_set_next_page_rx_cq(fp);
3301
3302                 /* BDs */
3303                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3304                 if (ring_size < rx_ring_size)
3305                         goto alloc_mem_err;
3306         }
3307
3308         return 0;
3309
3310 /* handles low memory cases */
3311 alloc_mem_err:
3312         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3313                                                 index, ring_size);
3314         /* FW will drop all packets if queue is not big enough,
3315          * In these cases we disable the queue
3316          * Min size is different for OOO, TPA and non-TPA queues
3317          */
3318         if (ring_size < (fp->disable_tpa ?
3319                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3320                         /* release memory allocated for this queue */
3321                         bnx2x_free_fp_mem_at(bp, index);
3322                         return -ENOMEM;
3323         }
3324         return 0;
3325 }
3326
3327 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3328 {
3329         int i;
3330
3331         /**
3332          * 1. Allocate FP for leading - fatal if error
3333          * 2. {CNIC} Allocate FCoE FP - fatal if error
3334          * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3335          * 4. Allocate RSS - fix number of queues if error
3336          */
3337
3338         /* leading */
3339         if (bnx2x_alloc_fp_mem_at(bp, 0))
3340                 return -ENOMEM;
3341
3342 #ifdef BCM_CNIC
3343         if (!NO_FCOE(bp))
3344                 /* FCoE */
3345                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3346                         /* we will fail load process instead of mark
3347                          * NO_FCOE_FLAG
3348                          */
3349                         return -ENOMEM;
3350 #endif
3351
3352         /* RSS */
3353         for_each_nondefault_eth_queue(bp, i)
3354                 if (bnx2x_alloc_fp_mem_at(bp, i))
3355                         break;
3356
3357         /* handle memory failures */
3358         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3359                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3360
3361                 WARN_ON(delta < 0);
3362 #ifdef BCM_CNIC
3363                 /**
3364                  * move non eth FPs next to last eth FP
3365                  * must be done in that order
3366                  * FCOE_IDX < FWD_IDX < OOO_IDX
3367                  */
3368
3369                 /* move FCoE fp even NO_FCOE_FLAG is on */
3370                 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3371 #endif
3372                 bp->num_queues -= delta;
3373                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3374                           bp->num_queues + delta, bp->num_queues);
3375         }
3376
3377         return 0;
3378 }
3379
3380 void bnx2x_free_mem_bp(struct bnx2x *bp)
3381 {
3382         kfree(bp->fp);
3383         kfree(bp->msix_table);
3384         kfree(bp->ilt);
3385 }
3386
3387 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3388 {
3389         struct bnx2x_fastpath *fp;
3390         struct msix_entry *tbl;
3391         struct bnx2x_ilt *ilt;
3392         int msix_table_size = 0;
3393
3394         /*
3395          * The biggest MSI-X table we might need is as a maximum number of fast
3396          * path IGU SBs plus default SB (for PF).
3397          */
3398         msix_table_size = bp->igu_sb_cnt + 1;
3399
3400         /* fp array: RSS plus CNIC related L2 queues */
3401         fp = kcalloc(BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE,
3402                      sizeof(*fp), GFP_KERNEL);
3403         if (!fp)
3404                 goto alloc_err;
3405         bp->fp = fp;
3406
3407         /* msix table */
3408         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3409         if (!tbl)
3410                 goto alloc_err;
3411         bp->msix_table = tbl;
3412
3413         /* ilt */
3414         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3415         if (!ilt)
3416                 goto alloc_err;
3417         bp->ilt = ilt;
3418
3419         return 0;
3420 alloc_err:
3421         bnx2x_free_mem_bp(bp);
3422         return -ENOMEM;
3423
3424 }
3425
3426 int bnx2x_reload_if_running(struct net_device *dev)
3427 {
3428         struct bnx2x *bp = netdev_priv(dev);
3429
3430         if (unlikely(!netif_running(dev)))
3431                 return 0;
3432
3433         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3434         return bnx2x_nic_load(bp, LOAD_NORMAL);
3435 }
3436
3437 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3438 {
3439         u32 sel_phy_idx = 0;
3440         if (bp->link_params.num_phys <= 1)
3441                 return INT_PHY;
3442
3443         if (bp->link_vars.link_up) {
3444                 sel_phy_idx = EXT_PHY1;
3445                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3446                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3447                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3448                         sel_phy_idx = EXT_PHY2;
3449         } else {
3450
3451                 switch (bnx2x_phy_selection(&bp->link_params)) {
3452                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3453                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3454                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3455                        sel_phy_idx = EXT_PHY1;
3456                        break;
3457                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3458                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3459                        sel_phy_idx = EXT_PHY2;
3460                        break;
3461                 }
3462         }
3463
3464         return sel_phy_idx;
3465
3466 }
3467 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3468 {
3469         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3470         /*
3471          * The selected actived PHY is always after swapping (in case PHY
3472          * swapping is enabled). So when swapping is enabled, we need to reverse
3473          * the configuration
3474          */
3475
3476         if (bp->link_params.multi_phy_config &
3477             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3478                 if (sel_phy_idx == EXT_PHY1)
3479                         sel_phy_idx = EXT_PHY2;
3480                 else if (sel_phy_idx == EXT_PHY2)
3481                         sel_phy_idx = EXT_PHY1;
3482         }
3483         return LINK_CONFIG_IDX(sel_phy_idx);
3484 }
3485
3486 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3487 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3488 {
3489         struct bnx2x *bp = netdev_priv(dev);
3490         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3491
3492         switch (type) {
3493         case NETDEV_FCOE_WWNN:
3494                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3495                                 cp->fcoe_wwn_node_name_lo);
3496                 break;
3497         case NETDEV_FCOE_WWPN:
3498                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3499                                 cp->fcoe_wwn_port_name_lo);
3500                 break;
3501         default:
3502                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
3503                 return -EINVAL;
3504         }
3505
3506         return 0;
3507 }
3508 #endif
3509
3510 /* called with rtnl_lock */
3511 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3512 {
3513         struct bnx2x *bp = netdev_priv(dev);
3514
3515         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3516                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
3517                 return -EAGAIN;
3518         }
3519
3520         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3521             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
3522                 BNX2X_ERR("Can't support requested MTU size\n");
3523                 return -EINVAL;
3524         }
3525
3526         /* This does not race with packet allocation
3527          * because the actual alloc size is
3528          * only updated as part of load
3529          */
3530         dev->mtu = new_mtu;
3531
3532         bp->gro_check = bnx2x_need_gro_check(new_mtu);
3533
3534         return bnx2x_reload_if_running(dev);
3535 }
3536
3537 netdev_features_t bnx2x_fix_features(struct net_device *dev,
3538                                      netdev_features_t features)
3539 {
3540         struct bnx2x *bp = netdev_priv(dev);
3541
3542         /* TPA requires Rx CSUM offloading */
3543         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
3544                 features &= ~NETIF_F_LRO;
3545                 features &= ~NETIF_F_GRO;
3546         }
3547
3548         return features;
3549 }
3550
3551 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3552 {
3553         struct bnx2x *bp = netdev_priv(dev);
3554         u32 flags = bp->flags;
3555         bool bnx2x_reload = false;
3556
3557         if (features & NETIF_F_LRO)
3558                 flags |= TPA_ENABLE_FLAG;
3559         else
3560                 flags &= ~TPA_ENABLE_FLAG;
3561
3562         if (features & NETIF_F_GRO)
3563                 flags |= GRO_ENABLE_FLAG;
3564         else
3565                 flags &= ~GRO_ENABLE_FLAG;
3566
3567         if (features & NETIF_F_LOOPBACK) {
3568                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3569                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
3570                         bnx2x_reload = true;
3571                 }
3572         } else {
3573                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3574                         bp->link_params.loopback_mode = LOOPBACK_NONE;
3575                         bnx2x_reload = true;
3576                 }
3577         }
3578
3579         if (flags ^ bp->flags) {
3580                 bp->flags = flags;
3581                 bnx2x_reload = true;
3582         }
3583
3584         if (bnx2x_reload) {
3585                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3586                         return bnx2x_reload_if_running(dev);
3587                 /* else: bnx2x_nic_load() will be called at end of recovery */
3588         }
3589
3590         return 0;
3591 }
3592
3593 void bnx2x_tx_timeout(struct net_device *dev)
3594 {
3595         struct bnx2x *bp = netdev_priv(dev);
3596
3597 #ifdef BNX2X_STOP_ON_ERROR
3598         if (!bp->panic)
3599                 bnx2x_panic();
3600 #endif
3601
3602         smp_mb__before_clear_bit();
3603         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3604         smp_mb__after_clear_bit();
3605
3606         /* This allows the netif to be shutdown gracefully before resetting */
3607         schedule_delayed_work(&bp->sp_rtnl_task, 0);
3608 }
3609
3610 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3611 {
3612         struct net_device *dev = pci_get_drvdata(pdev);
3613         struct bnx2x *bp;
3614
3615         if (!dev) {
3616                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3617                 return -ENODEV;
3618         }
3619         bp = netdev_priv(dev);
3620
3621         rtnl_lock();
3622
3623         pci_save_state(pdev);
3624
3625         if (!netif_running(dev)) {
3626                 rtnl_unlock();
3627                 return 0;
3628         }
3629
3630         netif_device_detach(dev);
3631
3632         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3633
3634         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3635
3636         rtnl_unlock();
3637
3638         return 0;
3639 }
3640
3641 int bnx2x_resume(struct pci_dev *pdev)
3642 {
3643         struct net_device *dev = pci_get_drvdata(pdev);
3644         struct bnx2x *bp;
3645         int rc;
3646
3647         if (!dev) {
3648                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3649                 return -ENODEV;
3650         }
3651         bp = netdev_priv(dev);
3652
3653         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3654                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
3655                 return -EAGAIN;
3656         }
3657
3658         rtnl_lock();
3659
3660         pci_restore_state(pdev);
3661
3662         if (!netif_running(dev)) {
3663                 rtnl_unlock();
3664                 return 0;
3665         }
3666
3667         bnx2x_set_power_state(bp, PCI_D0);
3668         netif_device_attach(dev);
3669
3670         rc = bnx2x_nic_load(bp, LOAD_OPEN);
3671
3672         rtnl_unlock();
3673
3674         return rc;
3675 }
3676
3677
3678 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3679                               u32 cid)
3680 {
3681         /* ustorm cxt validation */
3682         cxt->ustorm_ag_context.cdu_usage =
3683                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3684                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3685         /* xcontext validation */
3686         cxt->xstorm_ag_context.cdu_reserved =
3687                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3688                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3689 }
3690
3691 static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3692                                              u8 fw_sb_id, u8 sb_index,
3693                                              u8 ticks)
3694 {
3695
3696         u32 addr = BAR_CSTRORM_INTMEM +
3697                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3698         REG_WR8(bp, addr, ticks);
3699         DP(NETIF_MSG_IFUP,
3700            "port %x fw_sb_id %d sb_index %d ticks %d\n",
3701            port, fw_sb_id, sb_index, ticks);
3702 }
3703
3704 static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3705                                              u16 fw_sb_id, u8 sb_index,
3706                                              u8 disable)
3707 {
3708         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3709         u32 addr = BAR_CSTRORM_INTMEM +
3710                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3711         u16 flags = REG_RD16(bp, addr);
3712         /* clear and set */
3713         flags &= ~HC_INDEX_DATA_HC_ENABLED;
3714         flags |= enable_flag;
3715         REG_WR16(bp, addr, flags);
3716         DP(NETIF_MSG_IFUP,
3717            "port %x fw_sb_id %d sb_index %d disable %d\n",
3718            port, fw_sb_id, sb_index, disable);
3719 }
3720
3721 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3722                                     u8 sb_index, u8 disable, u16 usec)
3723 {
3724         int port = BP_PORT(bp);
3725         u8 ticks = usec / BNX2X_BTR;
3726
3727         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3728
3729         disable = disable ? 1 : (usec ? 0 : 1);
3730         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3731 }