drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.6.0-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69 };
  70
  71 /* ixgbevf_pci_tbl - PCI Device ID Table
  72  *
  73  * Wildcard entries (PCI_ANY_ID) should come last
  74  * Last entry must be all 0s
  75  *
  76  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  77  *   Class, Class Mask, private data (not used) }
  78  */
  79 static struct pci_device_id ixgbevf_pci_tbl[] = {
  80         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
  81         board_82599_vf},
  82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
  83         board_X540_vf},
  84
  85         /* required last entry */
  86         {0, }
  87 };
  88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  89
  90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
  92 MODULE_LICENSE("GPL");
  93 MODULE_VERSION(DRV_VERSION);
  94
  95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  96 static int debug = -1;
  97 module_param(debug, int, 0);
  98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  99
 100 /* forward decls */
 101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
 102
 103 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
 104                                            struct ixgbevf_ring *rx_ring,
 105                                            u32 val)
 106 {
 107         /*
 108          * Force memory writes to complete before letting h/w
 109          * know there are new descriptors to fetch.  (Only
 110          * applicable for weak-ordered memory model archs,
 111          * such as IA-64).
 112          */
 113         wmb();
 114         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
 115 }
 116
 117 /**
 118  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 119  * @adapter: pointer to adapter struct
 120  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 121  * @queue: queue to map the corresponding interrupt to
 122  * @msix_vector: the vector to map to the corresponding queue
 123  *
 124  */
 125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 126                              u8 queue, u8 msix_vector)
 127 {
 128         u32 ivar, index;
 129         struct ixgbe_hw *hw = &adapter->hw;
 130         if (direction == -1) {
 131                 /* other causes */
 132                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 133                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 134                 ivar &= ~0xFF;
 135                 ivar |= msix_vector;
 136                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 137         } else {
 138                 /* tx or rx causes */
 139                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 140                 index = ((16 * (queue & 1)) + (8 * direction));
 141                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 142                 ivar &= ~(0xFF << index);
 143                 ivar |= (msix_vector << index);
 144                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 145         }
 146 }
 147
 148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 149                                                struct ixgbevf_tx_buffer
 150                                                *tx_buffer_info)
 151 {
 152         if (tx_buffer_info->dma) {
 153                 if (tx_buffer_info->mapped_as_page)
 154                         dma_unmap_page(tx_ring->dev,
 155                                        tx_buffer_info->dma,
 156                                        tx_buffer_info->length,
 157                                        DMA_TO_DEVICE);
 158                 else
 159                         dma_unmap_single(tx_ring->dev,
 160                                          tx_buffer_info->dma,
 161                                          tx_buffer_info->length,
 162                                          DMA_TO_DEVICE);
 163                 tx_buffer_info->dma = 0;
 164         }
 165         if (tx_buffer_info->skb) {
 166                 dev_kfree_skb_any(tx_buffer_info->skb);
 167                 tx_buffer_info->skb = NULL;
 168         }
 169         tx_buffer_info->time_stamp = 0;
 170         /* tx_buffer_info must be completely set up in the transmit path */
 171 }
 172
 173 #define IXGBE_MAX_TXD_PWR       14
 174 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 175
 176 /* Tx Descriptors needed, worst case */
 177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 179
 180 static void ixgbevf_tx_timeout(struct net_device *netdev);
 181
 182 /**
 183  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 184  * @q_vector: board private structure
 185  * @tx_ring: tx ring to clean
 186  **/
 187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 188                                  struct ixgbevf_ring *tx_ring)
 189 {
 190         struct ixgbevf_adapter *adapter = q_vector->adapter;
 191         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 192         struct ixgbevf_tx_buffer *tx_buffer_info;
 193         unsigned int i, eop, count = 0;
 194         unsigned int total_bytes = 0, total_packets = 0;
 195
 196         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 197                 return true;
 198
 199         i = tx_ring->next_to_clean;
 200         eop = tx_ring->tx_buffer_info[i].next_to_watch;
 201         eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
 202
 203         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
 204                (count < tx_ring->count)) {
 205                 bool cleaned = false;
 206                 rmb(); /* read buffer_info after eop_desc */
 207                 /* eop could change between read and DD-check */
 208                 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
 209                         goto cont_loop;
 210                 for ( ; !cleaned; count++) {
 211                         struct sk_buff *skb;
 212                         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 213                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 214                         cleaned = (i == eop);
 215                         skb = tx_buffer_info->skb;
 216
 217                         if (cleaned && skb) {
 218                                 unsigned int segs, bytecount;
 219
 220                                 /* gso_segs is currently only valid for tcp */
 221                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
 222                                 /* multiply data chunks by size of headers */
 223                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
 224                                             skb->len;
 225                                 total_packets += segs;
 226                                 total_bytes += bytecount;
 227                         }
 228
 229                         ixgbevf_unmap_and_free_tx_resource(tx_ring,
 230                                                            tx_buffer_info);
 231
 232                         tx_desc->wb.status = 0;
 233
 234                         i++;
 235                         if (i == tx_ring->count)
 236                                 i = 0;
 237                 }
 238
 239 cont_loop:
 240                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
 241                 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
 242         }
 243
 244         tx_ring->next_to_clean = i;
 245
 246 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 247         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
 248                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 249                 /* Make sure that anybody stopping the queue after this
 250                  * sees the new next_to_clean.
 251                  */
 252                 smp_mb();
 253                 if (__netif_subqueue_stopped(tx_ring->netdev,
 254                                              tx_ring->queue_index) &&
 255                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 256                         netif_wake_subqueue(tx_ring->netdev,
 257                                             tx_ring->queue_index);
 258                         ++adapter->restart_queue;
 259                 }
 260         }
 261
 262         u64_stats_update_begin(&tx_ring->syncp);
 263         tx_ring->total_bytes += total_bytes;
 264         tx_ring->total_packets += total_packets;
 265         u64_stats_update_end(&tx_ring->syncp);
 266         q_vector->tx.total_bytes += total_bytes;
 267         q_vector->tx.total_packets += total_packets;
 268
 269         return count < tx_ring->count;
 270 }
 271
 272 /**
 273  * ixgbevf_receive_skb - Send a completed packet up the stack
 274  * @q_vector: structure containing interrupt and ring information
 275  * @skb: packet to send up
 276  * @status: hardware indication of status of receive
 277  * @rx_desc: rx descriptor
 278  **/
 279 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
 280                                 struct sk_buff *skb, u8 status,
 281                                 union ixgbe_adv_rx_desc *rx_desc)
 282 {
 283         struct ixgbevf_adapter *adapter = q_vector->adapter;
 284         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
 285         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
 286
 287         if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
 288                 __vlan_hwaccel_put_tag(skb, tag);
 289
 290         napi_gro_receive(&q_vector->napi, skb);
 291 }
 292
 293 /**
 294  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 295  * @adapter: address of board private structure
 296  * @status_err: hardware indication of status of receive
 297  * @skb: skb currently being received and modified
 298  **/
 299 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
 300                                        struct ixgbevf_ring *ring,
 301                                        u32 status_err, struct sk_buff *skb)
 302 {
 303         skb_checksum_none_assert(skb);
 304
 305         /* Rx csum disabled */
 306         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 307                 return;
 308
 309         /* if IP and error */
 310         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
 311             (status_err & IXGBE_RXDADV_ERR_IPE)) {
 312                 adapter->hw_csum_rx_error++;
 313                 return;
 314         }
 315
 316         if (!(status_err & IXGBE_RXD_STAT_L4CS))
 317                 return;
 318
 319         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
 320                 adapter->hw_csum_rx_error++;
 321                 return;
 322         }
 323
 324         /* It must be a TCP or UDP packet with a valid checksum */
 325         skb->ip_summed = CHECKSUM_UNNECESSARY;
 326         adapter->hw_csum_rx_good++;
 327 }
 328
 329 /**
 330  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 331  * @adapter: address of board private structure
 332  **/
 333 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
 334                                      struct ixgbevf_ring *rx_ring,
 335                                      int cleaned_count)
 336 {
 337         struct pci_dev *pdev = adapter->pdev;
 338         union ixgbe_adv_rx_desc *rx_desc;
 339         struct ixgbevf_rx_buffer *bi;
 340         struct sk_buff *skb;
 341         unsigned int i = rx_ring->next_to_use;
 342
 343         bi = &rx_ring->rx_buffer_info[i];
 344
 345         while (cleaned_count--) {
 346                 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 347                 skb = bi->skb;
 348                 if (!skb) {
 349                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 350                                                         rx_ring->rx_buf_len);
 351                         if (!skb) {
 352                                 adapter->alloc_rx_buff_failed++;
 353                                 goto no_buffers;
 354                         }
 355                         bi->skb = skb;
 356                 }
 357                 if (!bi->dma) {
 358                         bi->dma = dma_map_single(&pdev->dev, skb->data,
 359                                                  rx_ring->rx_buf_len,
 360                                                  DMA_FROM_DEVICE);
 361                 }
 362                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
 363
 364                 i++;
 365                 if (i == rx_ring->count)
 366                         i = 0;
 367                 bi = &rx_ring->rx_buffer_info[i];
 368         }
 369
 370 no_buffers:
 371         if (rx_ring->next_to_use != i) {
 372                 rx_ring->next_to_use = i;
 373
 374                 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
 375         }
 376 }
 377
 378 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 379                                              u32 qmask)
 380 {
 381         struct ixgbe_hw *hw = &adapter->hw;
 382
 383         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 384 }
 385
 386 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 387                                  struct ixgbevf_ring *rx_ring,
 388                                  int budget)
 389 {
 390         struct ixgbevf_adapter *adapter = q_vector->adapter;
 391         struct pci_dev *pdev = adapter->pdev;
 392         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
 393         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
 394         struct sk_buff *skb;
 395         unsigned int i;
 396         u32 len, staterr;
 397         int cleaned_count = 0;
 398         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 399
 400         i = rx_ring->next_to_clean;
 401         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 402         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 403         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 404
 405         while (staterr & IXGBE_RXD_STAT_DD) {
 406                 if (!budget)
 407                         break;
 408                 budget--;
 409
 410                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 411                 len = le16_to_cpu(rx_desc->wb.upper.length);
 412                 skb = rx_buffer_info->skb;
 413                 prefetch(skb->data - NET_IP_ALIGN);
 414                 rx_buffer_info->skb = NULL;
 415
 416                 if (rx_buffer_info->dma) {
 417                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
 418                                          rx_ring->rx_buf_len,
 419                                          DMA_FROM_DEVICE);
 420                         rx_buffer_info->dma = 0;
 421                         skb_put(skb, len);
 422                 }
 423
 424                 i++;
 425                 if (i == rx_ring->count)
 426                         i = 0;
 427
 428                 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
 429                 prefetch(next_rxd);
 430                 cleaned_count++;
 431
 432                 next_buffer = &rx_ring->rx_buffer_info[i];
 433
 434                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
 435                         skb->next = next_buffer->skb;
 436                         IXGBE_CB(skb->next)->prev = skb;
 437                         adapter->non_eop_descs++;
 438                         goto next_desc;
 439                 }
 440
 441                 /* we should not be chaining buffers, if we did drop the skb */
 442                 if (IXGBE_CB(skb)->prev) {
 443                         do {
 444                                 struct sk_buff *this = skb;
 445                                 skb = IXGBE_CB(skb)->prev;
 446                                 dev_kfree_skb(this);
 447                         } while (skb);
 448                         goto next_desc;
 449                 }
 450
 451                 /* ERR_MASK will only have valid bits if EOP set */
 452                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
 453                         dev_kfree_skb_irq(skb);
 454                         goto next_desc;
 455                 }
 456
 457                 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb);
 458
 459                 /* probably a little skewed due to removing CRC */
 460                 total_rx_bytes += skb->len;
 461                 total_rx_packets++;
 462
 463                 /*
 464                  * Work around issue of some types of VM to VM loop back
 465                  * packets not getting split correctly
 466                  */
 467                 if (staterr & IXGBE_RXD_STAT_LB) {
 468                         u32 header_fixup_len = skb_headlen(skb);
 469                         if (header_fixup_len < 14)
 470                                 skb_push(skb, header_fixup_len);
 471                 }
 472                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 473
 474                 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc);
 475
 476 next_desc:
 477                 rx_desc->wb.upper.status_error = 0;
 478
 479                 /* return some buffers to hardware, one at a time is too slow */
 480                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 481                         ixgbevf_alloc_rx_buffers(adapter, rx_ring,
 482                                                  cleaned_count);
 483                         cleaned_count = 0;
 484                 }
 485
 486                 /* use prefetched values */
 487                 rx_desc = next_rxd;
 488                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
 489
 490                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 491         }
 492
 493         rx_ring->next_to_clean = i;
 494         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
 495
 496         if (cleaned_count)
 497                 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
 498
 499         u64_stats_update_begin(&rx_ring->syncp);
 500         rx_ring->total_packets += total_rx_packets;
 501         rx_ring->total_bytes += total_rx_bytes;
 502         u64_stats_update_end(&rx_ring->syncp);
 503         q_vector->rx.total_packets += total_rx_packets;
 504         q_vector->rx.total_bytes += total_rx_bytes;
 505
 506         return !!budget;
 507 }
 508
 509 /**
 510  * ixgbevf_poll - NAPI polling calback
 511  * @napi: napi struct with our devices info in it
 512  * @budget: amount of work driver is allowed to do this pass, in packets
 513  *
 514  * This function will clean more than one or more rings associated with a
 515  * q_vector.
 516  **/
 517 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 518 {
 519         struct ixgbevf_q_vector *q_vector =
 520                 container_of(napi, struct ixgbevf_q_vector, napi);
 521         struct ixgbevf_adapter *adapter = q_vector->adapter;
 522         struct ixgbevf_ring *ring;
 523         int per_ring_budget;
 524         bool clean_complete = true;
 525
 526         ixgbevf_for_each_ring(ring, q_vector->tx)
 527                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 528
 529         /* attempt to distribute budget to each queue fairly, but don't allow
 530          * the budget to go below 1 because we'll exit polling */
 531         if (q_vector->rx.count > 1)
 532                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 533         else
 534                 per_ring_budget = budget;
 535
 536         ixgbevf_for_each_ring(ring, q_vector->rx)
 537                 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
 538                                                        per_ring_budget);
 539
 540         /* If all work not completed, return budget and keep polling */
 541         if (!clean_complete)
 542                 return budget;
 543         /* all work done, exit the polling mode */
 544         napi_complete(napi);
 545         if (adapter->rx_itr_setting & 1)
 546                 ixgbevf_set_itr(q_vector);
 547         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 548                 ixgbevf_irq_enable_queues(adapter,
 549                                           1 << q_vector->v_idx);
 550
 551         return 0;
 552 }
 553
 554 /**
 555  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 556  * @q_vector: structure containing interrupt and ring information
 557  */
 558 static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 559 {
 560         struct ixgbevf_adapter *adapter = q_vector->adapter;
 561         struct ixgbe_hw *hw = &adapter->hw;
 562         int v_idx = q_vector->v_idx;
 563         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 564
 565         /*
 566          * set the WDIS bit to not clear the timer bits and cause an
 567          * immediate assertion of the interrupt
 568          */
 569         itr_reg |= IXGBE_EITR_CNT_WDIS;
 570
 571         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 572 }
 573
 574 /**
 575  * ixgbevf_configure_msix - Configure MSI-X hardware
 576  * @adapter: board private structure
 577  *
 578  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 579  * interrupts.
 580  **/
 581 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
 582 {
 583         struct ixgbevf_q_vector *q_vector;
 584         int q_vectors, v_idx;
 585
 586         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 587         adapter->eims_enable_mask = 0;
 588
 589         /*
 590          * Populate the IVAR table and set the ITR values to the
 591          * corresponding register.
 592          */
 593         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
 594                 struct ixgbevf_ring *ring;
 595                 q_vector = adapter->q_vector[v_idx];
 596
 597                 ixgbevf_for_each_ring(ring, q_vector->rx)
 598                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
 599
 600                 ixgbevf_for_each_ring(ring, q_vector->tx)
 601                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
 602
 603                 if (q_vector->tx.ring && !q_vector->rx.ring) {
 604                         /* tx only vector */
 605                         if (adapter->tx_itr_setting == 1)
 606                                 q_vector->itr = IXGBE_10K_ITR;
 607                         else
 608                                 q_vector->itr = adapter->tx_itr_setting;
 609                 } else {
 610                         /* rx or rx/tx vector */
 611                         if (adapter->rx_itr_setting == 1)
 612                                 q_vector->itr = IXGBE_20K_ITR;
 613                         else
 614                                 q_vector->itr = adapter->rx_itr_setting;
 615                 }
 616
 617                 /* add q_vector eims value to global eims_enable_mask */
 618                 adapter->eims_enable_mask |= 1 << v_idx;
 619
 620                 ixgbevf_write_eitr(q_vector);
 621         }
 622
 623         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
 624         /* setup eims_other and add value to global eims_enable_mask */
 625         adapter->eims_other = 1 << v_idx;
 626         adapter->eims_enable_mask |= adapter->eims_other;
 627 }
 628
 629 enum latency_range {
 630         lowest_latency = 0,
 631         low_latency = 1,
 632         bulk_latency = 2,
 633         latency_invalid = 255
 634 };
 635
 636 /**
 637  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
 638  * @q_vector: structure containing interrupt and ring information
 639  * @ring_container: structure containing ring performance data
 640  *
 641  *      Stores a new ITR value based on packets and byte
 642  *      counts during the last interrupt.  The advantage of per interrupt
 643  *      computation is faster updates and more accurate ITR for the current
 644  *      traffic pattern.  Constants in this function were computed
 645  *      based on theoretical maximum wire speed and thresholds were set based
 646  *      on testing data as well as attempting to minimize response time
 647  *      while increasing bulk throughput.
 648  **/
 649 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
 650                                struct ixgbevf_ring_container *ring_container)
 651 {
 652         int bytes = ring_container->total_bytes;
 653         int packets = ring_container->total_packets;
 654         u32 timepassed_us;
 655         u64 bytes_perint;
 656         u8 itr_setting = ring_container->itr;
 657
 658         if (packets == 0)
 659                 return;
 660
 661         /* simple throttlerate management
 662          *    0-20MB/s lowest (100000 ints/s)
 663          *   20-100MB/s low   (20000 ints/s)
 664          *  100-1249MB/s bulk (8000 ints/s)
 665          */
 666         /* what was last interrupt timeslice? */
 667         timepassed_us = q_vector->itr >> 2;
 668         bytes_perint = bytes / timepassed_us; /* bytes/usec */
 669
 670         switch (itr_setting) {
 671         case lowest_latency:
 672                 if (bytes_perint > 10)
 673                         itr_setting = low_latency;
 674                 break;
 675         case low_latency:
 676                 if (bytes_perint > 20)
 677                         itr_setting = bulk_latency;
 678                 else if (bytes_perint <= 10)
 679                         itr_setting = lowest_latency;
 680                 break;
 681         case bulk_latency:
 682                 if (bytes_perint <= 20)
 683                         itr_setting = low_latency;
 684                 break;
 685         }
 686
 687         /* clear work counters since we have the values we need */
 688         ring_container->total_bytes = 0;
 689         ring_container->total_packets = 0;
 690
 691         /* write updated itr to ring container */
 692         ring_container->itr = itr_setting;
 693 }
 694
 695 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
 696 {
 697         u32 new_itr = q_vector->itr;
 698         u8 current_itr;
 699
 700         ixgbevf_update_itr(q_vector, &q_vector->tx);
 701         ixgbevf_update_itr(q_vector, &q_vector->rx);
 702
 703         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
 704
 705         switch (current_itr) {
 706         /* counts and packets in update_itr are dependent on these numbers */
 707         case lowest_latency:
 708                 new_itr = IXGBE_100K_ITR;
 709                 break;
 710         case low_latency:
 711                 new_itr = IXGBE_20K_ITR;
 712                 break;
 713         case bulk_latency:
 714         default:
 715                 new_itr = IXGBE_8K_ITR;
 716                 break;
 717         }
 718
 719         if (new_itr != q_vector->itr) {
 720                 /* do an exponential smoothing */
 721                 new_itr = (10 * new_itr * q_vector->itr) /
 722                           ((9 * new_itr) + q_vector->itr);
 723
 724                 /* save the algorithm value here */
 725                 q_vector->itr = new_itr;
 726
 727                 ixgbevf_write_eitr(q_vector);
 728         }
 729 }
 730
 731 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
 732 {
 733         struct ixgbevf_adapter *adapter = data;
 734         struct ixgbe_hw *hw = &adapter->hw;
 735
 736         hw->mac.get_link_status = 1;
 737
 738         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 739                 mod_timer(&adapter->watchdog_timer, jiffies);
 740
 741         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
 742
 743         return IRQ_HANDLED;
 744 }
 745
 746
 747 /**
 748  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
 749  * @irq: unused
 750  * @data: pointer to our q_vector struct for this interrupt vector
 751  **/
 752 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
 753 {
 754         struct ixgbevf_q_vector *q_vector = data;
 755
 756         /* EIAM disabled interrupts (on this vector) for us */
 757         if (q_vector->rx.ring || q_vector->tx.ring)
 758                 napi_schedule(&q_vector->napi);
 759
 760         return IRQ_HANDLED;
 761 }
 762
 763 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
 764                                      int r_idx)
 765 {
 766         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 767
 768         a->rx_ring[r_idx].next = q_vector->rx.ring;
 769         q_vector->rx.ring = &a->rx_ring[r_idx];
 770         q_vector->rx.count++;
 771 }
 772
 773 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
 774                                      int t_idx)
 775 {
 776         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 777
 778         a->tx_ring[t_idx].next = q_vector->tx.ring;
 779         q_vector->tx.ring = &a->tx_ring[t_idx];
 780         q_vector->tx.count++;
 781 }
 782
 783 /**
 784  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
 785  * @adapter: board private structure to initialize
 786  *
 787  * This function maps descriptor rings to the queue-specific vectors
 788  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 789  * one vector per ring/queue, but on a constrained vector budget, we
 790  * group the rings as "efficiently" as possible.  You would add new
 791  * mapping configurations in here.
 792  **/
 793 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
 794 {
 795         int q_vectors;
 796         int v_start = 0;
 797         int rxr_idx = 0, txr_idx = 0;
 798         int rxr_remaining = adapter->num_rx_queues;
 799         int txr_remaining = adapter->num_tx_queues;
 800         int i, j;
 801         int rqpv, tqpv;
 802         int err = 0;
 803
 804         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 805
 806         /*
 807          * The ideal configuration...
 808          * We have enough vectors to map one per queue.
 809          */
 810         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
 811                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 812                         map_vector_to_rxq(adapter, v_start, rxr_idx);
 813
 814                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 815                         map_vector_to_txq(adapter, v_start, txr_idx);
 816                 goto out;
 817         }
 818
 819         /*
 820          * If we don't have enough vectors for a 1-to-1
 821          * mapping, we'll have to group them so there are
 822          * multiple queues per vector.
 823          */
 824         /* Re-adjusting *qpv takes care of the remainder. */
 825         for (i = v_start; i < q_vectors; i++) {
 826                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 827                 for (j = 0; j < rqpv; j++) {
 828                         map_vector_to_rxq(adapter, i, rxr_idx);
 829                         rxr_idx++;
 830                         rxr_remaining--;
 831                 }
 832         }
 833         for (i = v_start; i < q_vectors; i++) {
 834                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 835                 for (j = 0; j < tqpv; j++) {
 836                         map_vector_to_txq(adapter, i, txr_idx);
 837                         txr_idx++;
 838                         txr_remaining--;
 839                 }
 840         }
 841
 842 out:
 843         return err;
 844 }
 845
 846 /**
 847  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
 848  * @adapter: board private structure
 849  *
 850  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
 851  * interrupts from the kernel.
 852  **/
 853 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
 854 {
 855         struct net_device *netdev = adapter->netdev;
 856         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 857         int vector, err;
 858         int ri = 0, ti = 0;
 859
 860         for (vector = 0; vector < q_vectors; vector++) {
 861                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
 862                 struct msix_entry *entry = &adapter->msix_entries[vector];
 863
 864                 if (q_vector->tx.ring && q_vector->rx.ring) {
 865                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 866                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
 867                         ti++;
 868                 } else if (q_vector->rx.ring) {
 869                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 870                                  "%s-%s-%d", netdev->name, "rx", ri++);
 871                 } else if (q_vector->tx.ring) {
 872                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 873                                  "%s-%s-%d", netdev->name, "tx", ti++);
 874                 } else {
 875                         /* skip this unused q_vector */
 876                         continue;
 877                 }
 878                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
 879                                   q_vector->name, q_vector);
 880                 if (err) {
 881                         hw_dbg(&adapter->hw,
 882                                "request_irq failed for MSIX interrupt "
 883                                "Error: %d\n", err);
 884                         goto free_queue_irqs;
 885                 }
 886         }
 887
 888         err = request_irq(adapter->msix_entries[vector].vector,
 889                           &ixgbevf_msix_other, 0, netdev->name, adapter);
 890         if (err) {
 891                 hw_dbg(&adapter->hw,
 892                        "request_irq for msix_other failed: %d\n", err);
 893                 goto free_queue_irqs;
 894         }
 895
 896         return 0;
 897
 898 free_queue_irqs:
 899         while (vector) {
 900                 vector--;
 901                 free_irq(adapter->msix_entries[vector].vector,
 902                          adapter->q_vector[vector]);
 903         }
 904         pci_disable_msix(adapter->pdev);
 905         kfree(adapter->msix_entries);
 906         adapter->msix_entries = NULL;
 907         return err;
 908 }
 909
 910 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
 911 {
 912         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 913
 914         for (i = 0; i < q_vectors; i++) {
 915                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
 916                 q_vector->rx.ring = NULL;
 917                 q_vector->tx.ring = NULL;
 918                 q_vector->rx.count = 0;
 919                 q_vector->tx.count = 0;
 920         }
 921 }
 922
 923 /**
 924  * ixgbevf_request_irq - initialize interrupts
 925  * @adapter: board private structure
 926  *
 927  * Attempts to configure interrupts using the best available
 928  * capabilities of the hardware and kernel.
 929  **/
 930 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
 931 {
 932         int err = 0;
 933
 934         err = ixgbevf_request_msix_irqs(adapter);
 935
 936         if (err)
 937                 hw_dbg(&adapter->hw,
 938                        "request_irq failed, Error %d\n", err);
 939
 940         return err;
 941 }
 942
 943 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
 944 {
 945         int i, q_vectors;
 946
 947         q_vectors = adapter->num_msix_vectors;
 948         i = q_vectors - 1;
 949
 950         free_irq(adapter->msix_entries[i].vector, adapter);
 951         i--;
 952
 953         for (; i >= 0; i--) {
 954                 /* free only the irqs that were actually requested */
 955                 if (!adapter->q_vector[i]->rx.ring &&
 956                     !adapter->q_vector[i]->tx.ring)
 957                         continue;
 958
 959                 free_irq(adapter->msix_entries[i].vector,
 960                          adapter->q_vector[i]);
 961         }
 962
 963         ixgbevf_reset_q_vectors(adapter);
 964 }
 965
 966 /**
 967  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
 968  * @adapter: board private structure
 969  **/
 970 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
 971 {
 972         struct ixgbe_hw *hw = &adapter->hw;
 973         int i;
 974
 975         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
 976         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
 977         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
 978
 979         IXGBE_WRITE_FLUSH(hw);
 980
 981         for (i = 0; i < adapter->num_msix_vectors; i++)
 982                 synchronize_irq(adapter->msix_entries[i].vector);
 983 }
 984
 985 /**
 986  * ixgbevf_irq_enable - Enable default interrupt generation settings
 987  * @adapter: board private structure
 988  **/
 989 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
 990 {
 991         struct ixgbe_hw *hw = &adapter->hw;
 992
 993         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
 994         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
 995         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
 996 }
 997
 998 /**
 999  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1000  * @adapter: board private structure
1001  *
1002  * Configure the Tx unit of the MAC after a reset.
1003  **/
1004 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1005 {
1006         u64 tdba;
1007         struct ixgbe_hw *hw = &adapter->hw;
1008         u32 i, j, tdlen, txctrl;
1009
1010         /* Setup the HW Tx Head and Tail descriptor pointers */
1011         for (i = 0; i < adapter->num_tx_queues; i++) {
1012                 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1013                 j = ring->reg_idx;
1014                 tdba = ring->dma;
1015                 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1016                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1017                                 (tdba & DMA_BIT_MASK(32)));
1018                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1019                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1020                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1021                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1022                 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1023                 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1024                 /* Disable Tx Head Writeback RO bit, since this hoses
1025                  * bookkeeping if things aren't delivered in order.
1026                  */
1027                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1028                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1029                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1030         }
1031 }
1032
1033 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1034
1035 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1036 {
1037         struct ixgbevf_ring *rx_ring;
1038         struct ixgbe_hw *hw = &adapter->hw;
1039         u32 srrctl;
1040
1041         rx_ring = &adapter->rx_ring[index];
1042
1043         srrctl = IXGBE_SRRCTL_DROP_EN;
1044
1045         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1046
1047         srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
1048                   IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1049
1050         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1051 }
1052
1053 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter)
1054 {
1055         struct ixgbe_hw *hw = &adapter->hw;
1056         struct net_device *netdev = adapter->netdev;
1057         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1058         int i;
1059         u16 rx_buf_len;
1060
1061         /* notify the PF of our intent to use this size of frame */
1062         ixgbevf_rlpml_set_vf(hw, max_frame);
1063
1064         /* PF will allow an extra 4 bytes past for vlan tagged frames */
1065         max_frame += VLAN_HLEN;
1066
1067         /*
1068          * Make best use of allocation by using all but 1K of a
1069          * power of 2 allocation that will be used for skb->head.
1070          */
1071         if ((hw->mac.type == ixgbe_mac_X540_vf) &&
1072             (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE))
1073                 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1074         else if (max_frame <= IXGBEVF_RXBUFFER_3K)
1075                 rx_buf_len = IXGBEVF_RXBUFFER_3K;
1076         else if (max_frame <= IXGBEVF_RXBUFFER_7K)
1077                 rx_buf_len = IXGBEVF_RXBUFFER_7K;
1078         else if (max_frame <= IXGBEVF_RXBUFFER_15K)
1079                 rx_buf_len = IXGBEVF_RXBUFFER_15K;
1080         else
1081                 rx_buf_len = IXGBEVF_MAX_RXBUFFER;
1082
1083         for (i = 0; i < adapter->num_rx_queues; i++)
1084                 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1085 }
1086
1087 /**
1088  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1089  * @adapter: board private structure
1090  *
1091  * Configure the Rx unit of the MAC after a reset.
1092  **/
1093 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1094 {
1095         u64 rdba;
1096         struct ixgbe_hw *hw = &adapter->hw;
1097         int i, j;
1098         u32 rdlen;
1099
1100         /* PSRTYPE must be initialized in 82599 */
1101         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1102
1103         /* set_rx_buffer_len must be called before ring initialization */
1104         ixgbevf_set_rx_buffer_len(adapter);
1105
1106         rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1107         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1108          * the Base and Length of the Rx Descriptor Ring */
1109         for (i = 0; i < adapter->num_rx_queues; i++) {
1110                 rdba = adapter->rx_ring[i].dma;
1111                 j = adapter->rx_ring[i].reg_idx;
1112                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1113                                 (rdba & DMA_BIT_MASK(32)));
1114                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1115                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1116                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1117                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1118                 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1119                 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1120
1121                 ixgbevf_configure_srrctl(adapter, j);
1122         }
1123 }
1124
1125 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1126 {
1127         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1128         struct ixgbe_hw *hw = &adapter->hw;
1129         int err;
1130
1131         if (!hw->mac.ops.set_vfta)
1132                 return -EOPNOTSUPP;
1133
1134         spin_lock(&adapter->mbx_lock);
1135
1136         /* add VID to filter table */
1137         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1138
1139         spin_unlock(&adapter->mbx_lock);
1140
1141         /* translate error return types so error makes sense */
1142         if (err == IXGBE_ERR_MBX)
1143                 return -EIO;
1144
1145         if (err == IXGBE_ERR_INVALID_ARGUMENT)
1146                 return -EACCES;
1147
1148         set_bit(vid, adapter->active_vlans);
1149
1150         return err;
1151 }
1152
1153 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1154 {
1155         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1156         struct ixgbe_hw *hw = &adapter->hw;
1157         int err = -EOPNOTSUPP;
1158
1159         spin_lock(&adapter->mbx_lock);
1160
1161         /* remove VID from filter table */
1162         if (hw->mac.ops.set_vfta)
1163                 err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1164
1165         spin_unlock(&adapter->mbx_lock);
1166
1167         clear_bit(vid, adapter->active_vlans);
1168
1169         return err;
1170 }
1171
1172 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1173 {
1174         u16 vid;
1175
1176         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1177                 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
1178 }
1179
1180 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1181 {
1182         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1183         struct ixgbe_hw *hw = &adapter->hw;
1184         int count = 0;
1185
1186         if ((netdev_uc_count(netdev)) > 10) {
1187                 pr_err("Too many unicast filters - No Space\n");
1188                 return -ENOSPC;
1189         }
1190
1191         if (!netdev_uc_empty(netdev)) {
1192                 struct netdev_hw_addr *ha;
1193                 netdev_for_each_uc_addr(ha, netdev) {
1194                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1195                         udelay(200);
1196                 }
1197         } else {
1198                 /*
1199                  * If the list is empty then send message to PF driver to
1200                  * clear all macvlans on this VF.
1201                  */
1202                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1203         }
1204
1205         return count;
1206 }
1207
1208 /**
1209  * ixgbevf_set_rx_mode - Multicast set
1210  * @netdev: network interface device structure
1211  *
1212  * The set_rx_method entry point is called whenever the multicast address
1213  * list or the network interface flags are updated.  This routine is
1214  * responsible for configuring the hardware for proper multicast mode.
1215  **/
1216 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1217 {
1218         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1219         struct ixgbe_hw *hw = &adapter->hw;
1220
1221         spin_lock(&adapter->mbx_lock);
1222
1223         /* reprogram multicast list */
1224         if (hw->mac.ops.update_mc_addr_list)
1225                 hw->mac.ops.update_mc_addr_list(hw, netdev);
1226
1227         ixgbevf_write_uc_addr_list(netdev);
1228
1229         spin_unlock(&adapter->mbx_lock);
1230 }
1231
1232 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1233 {
1234         int q_idx;
1235         struct ixgbevf_q_vector *q_vector;
1236         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1237
1238         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1239                 q_vector = adapter->q_vector[q_idx];
1240                 napi_enable(&q_vector->napi);
1241         }
1242 }
1243
1244 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1245 {
1246         int q_idx;
1247         struct ixgbevf_q_vector *q_vector;
1248         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1249
1250         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1251                 q_vector = adapter->q_vector[q_idx];
1252                 napi_disable(&q_vector->napi);
1253         }
1254 }
1255
1256 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1257 {
1258         struct net_device *netdev = adapter->netdev;
1259         int i;
1260
1261         ixgbevf_set_rx_mode(netdev);
1262
1263         ixgbevf_restore_vlan(adapter);
1264
1265         ixgbevf_configure_tx(adapter);
1266         ixgbevf_configure_rx(adapter);
1267         for (i = 0; i < adapter->num_rx_queues; i++) {
1268                 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1269                 ixgbevf_alloc_rx_buffers(adapter, ring,
1270                                          IXGBE_DESC_UNUSED(ring));
1271         }
1272 }
1273
1274 #define IXGBE_MAX_RX_DESC_POLL 10
1275 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1276                                                 int rxr)
1277 {
1278         struct ixgbe_hw *hw = &adapter->hw;
1279         int j = adapter->rx_ring[rxr].reg_idx;
1280         int k;
1281
1282         for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1283                 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1284                         break;
1285                 else
1286                         msleep(1);
1287         }
1288         if (k >= IXGBE_MAX_RX_DESC_POLL) {
1289                 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1290                        "not set within the polling period\n", rxr);
1291         }
1292
1293         ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1294                                 (adapter->rx_ring[rxr].count - 1));
1295 }
1296
1297 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1298 {
1299         /* Only save pre-reset stats if there are some */
1300         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1301                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1302                         adapter->stats.base_vfgprc;
1303                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1304                         adapter->stats.base_vfgptc;
1305                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1306                         adapter->stats.base_vfgorc;
1307                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1308                         adapter->stats.base_vfgotc;
1309                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1310                         adapter->stats.base_vfmprc;
1311         }
1312 }
1313
1314 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1315 {
1316         struct ixgbe_hw *hw = &adapter->hw;
1317
1318         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1319         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1320         adapter->stats.last_vfgorc |=
1321                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1322         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1323         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1324         adapter->stats.last_vfgotc |=
1325                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1326         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1327
1328         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1329         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1330         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1331         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1332         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1333 }
1334
1335 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1336 {
1337         struct ixgbe_hw *hw = &adapter->hw;
1338         int api[] = { ixgbe_mbox_api_10,
1339                       ixgbe_mbox_api_unknown };
1340         int err = 0, idx = 0;
1341
1342         spin_lock(&adapter->mbx_lock);
1343
1344         while (api[idx] != ixgbe_mbox_api_unknown) {
1345                 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1346                 if (!err)
1347                         break;
1348                 idx++;
1349         }
1350
1351         spin_unlock(&adapter->mbx_lock);
1352 }
1353
1354 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1355 {
1356         struct net_device *netdev = adapter->netdev;
1357         struct ixgbe_hw *hw = &adapter->hw;
1358         int i, j = 0;
1359         int num_rx_rings = adapter->num_rx_queues;
1360         u32 txdctl, rxdctl;
1361
1362         for (i = 0; i < adapter->num_tx_queues; i++) {
1363                 j = adapter->tx_ring[i].reg_idx;
1364                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1365                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1366                 txdctl |= (8 << 16);
1367                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1368         }
1369
1370         for (i = 0; i < adapter->num_tx_queues; i++) {
1371                 j = adapter->tx_ring[i].reg_idx;
1372                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1373                 txdctl |= IXGBE_TXDCTL_ENABLE;
1374                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1375         }
1376
1377         for (i = 0; i < num_rx_rings; i++) {
1378                 j = adapter->rx_ring[i].reg_idx;
1379                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1380                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1381                 if (hw->mac.type == ixgbe_mac_X540_vf) {
1382                         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1383                         rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1384                                    IXGBE_RXDCTL_RLPML_EN);
1385                 }
1386                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1387                 ixgbevf_rx_desc_queue_enable(adapter, i);
1388         }
1389
1390         ixgbevf_configure_msix(adapter);
1391
1392         spin_lock(&adapter->mbx_lock);
1393
1394         if (hw->mac.ops.set_rar) {
1395                 if (is_valid_ether_addr(hw->mac.addr))
1396                         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1397                 else
1398                         hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1399         }
1400
1401         spin_unlock(&adapter->mbx_lock);
1402
1403         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1404         ixgbevf_napi_enable_all(adapter);
1405
1406         /* enable transmits */
1407         netif_tx_start_all_queues(netdev);
1408
1409         ixgbevf_save_reset_stats(adapter);
1410         ixgbevf_init_last_counter_stats(adapter);
1411
1412         hw->mac.get_link_status = 1;
1413         mod_timer(&adapter->watchdog_timer, jiffies);
1414 }
1415
1416 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1417 {
1418         struct ixgbe_hw *hw = &adapter->hw;
1419
1420         ixgbevf_negotiate_api(adapter);
1421
1422         ixgbevf_configure(adapter);
1423
1424         ixgbevf_up_complete(adapter);
1425
1426         /* clear any pending interrupts, may auto mask */
1427         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1428
1429         ixgbevf_irq_enable(adapter);
1430 }
1431
1432 /**
1433  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1434  * @adapter: board private structure
1435  * @rx_ring: ring to free buffers from
1436  **/
1437 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1438                                   struct ixgbevf_ring *rx_ring)
1439 {
1440         struct pci_dev *pdev = adapter->pdev;
1441         unsigned long size;
1442         unsigned int i;
1443
1444         if (!rx_ring->rx_buffer_info)
1445                 return;
1446
1447         /* Free all the Rx ring sk_buffs */
1448         for (i = 0; i < rx_ring->count; i++) {
1449                 struct ixgbevf_rx_buffer *rx_buffer_info;
1450
1451                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1452                 if (rx_buffer_info->dma) {
1453                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1454                                          rx_ring->rx_buf_len,
1455                                          DMA_FROM_DEVICE);
1456                         rx_buffer_info->dma = 0;
1457                 }
1458                 if (rx_buffer_info->skb) {
1459                         struct sk_buff *skb = rx_buffer_info->skb;
1460                         rx_buffer_info->skb = NULL;
1461                         do {
1462                                 struct sk_buff *this = skb;
1463                                 skb = IXGBE_CB(skb)->prev;
1464                                 dev_kfree_skb(this);
1465                         } while (skb);
1466                 }
1467         }
1468
1469         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1470         memset(rx_ring->rx_buffer_info, 0, size);
1471
1472         /* Zero out the descriptor ring */
1473         memset(rx_ring->desc, 0, rx_ring->size);
1474
1475         rx_ring->next_to_clean = 0;
1476         rx_ring->next_to_use = 0;
1477
1478         if (rx_ring->head)
1479                 writel(0, adapter->hw.hw_addr + rx_ring->head);
1480         if (rx_ring->tail)
1481                 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1482 }
1483
1484 /**
1485  * ixgbevf_clean_tx_ring - Free Tx Buffers
1486  * @adapter: board private structure
1487  * @tx_ring: ring to be cleaned
1488  **/
1489 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1490                                   struct ixgbevf_ring *tx_ring)
1491 {
1492         struct ixgbevf_tx_buffer *tx_buffer_info;
1493         unsigned long size;
1494         unsigned int i;
1495
1496         if (!tx_ring->tx_buffer_info)
1497                 return;
1498
1499         /* Free all the Tx ring sk_buffs */
1500
1501         for (i = 0; i < tx_ring->count; i++) {
1502                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1503                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1504         }
1505
1506         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1507         memset(tx_ring->tx_buffer_info, 0, size);
1508
1509         memset(tx_ring->desc, 0, tx_ring->size);
1510
1511         tx_ring->next_to_use = 0;
1512         tx_ring->next_to_clean = 0;
1513
1514         if (tx_ring->head)
1515                 writel(0, adapter->hw.hw_addr + tx_ring->head);
1516         if (tx_ring->tail)
1517                 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1518 }
1519
1520 /**
1521  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1522  * @adapter: board private structure
1523  **/
1524 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1525 {
1526         int i;
1527
1528         for (i = 0; i < adapter->num_rx_queues; i++)
1529                 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1530 }
1531
1532 /**
1533  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1534  * @adapter: board private structure
1535  **/
1536 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1537 {
1538         int i;
1539
1540         for (i = 0; i < adapter->num_tx_queues; i++)
1541                 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1542 }
1543
1544 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1545 {
1546         struct net_device *netdev = adapter->netdev;
1547         struct ixgbe_hw *hw = &adapter->hw;
1548         u32 txdctl;
1549         int i, j;
1550
1551         /* signal that we are down to the interrupt handler */
1552         set_bit(__IXGBEVF_DOWN, &adapter->state);
1553         /* disable receives */
1554
1555         netif_tx_disable(netdev);
1556
1557         msleep(10);
1558
1559         netif_tx_stop_all_queues(netdev);
1560
1561         ixgbevf_irq_disable(adapter);
1562
1563         ixgbevf_napi_disable_all(adapter);
1564
1565         del_timer_sync(&adapter->watchdog_timer);
1566         /* can't call flush scheduled work here because it can deadlock
1567          * if linkwatch_event tries to acquire the rtnl_lock which we are
1568          * holding */
1569         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1570                 msleep(1);
1571
1572         /* disable transmits in the hardware now that interrupts are off */
1573         for (i = 0; i < adapter->num_tx_queues; i++) {
1574                 j = adapter->tx_ring[i].reg_idx;
1575                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1576                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1577                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1578         }
1579
1580         netif_carrier_off(netdev);
1581
1582         if (!pci_channel_offline(adapter->pdev))
1583                 ixgbevf_reset(adapter);
1584
1585         ixgbevf_clean_all_tx_rings(adapter);
1586         ixgbevf_clean_all_rx_rings(adapter);
1587 }
1588
1589 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1590 {
1591         WARN_ON(in_interrupt());
1592
1593         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1594                 msleep(1);
1595
1596         /*
1597          * Check if PF is up before re-init.  If not then skip until
1598          * later when the PF is up and ready to service requests from
1599          * the VF via mailbox.  If the VF is up and running then the
1600          * watchdog task will continue to schedule reset tasks until
1601          * the PF is up and running.
1602          */
1603         ixgbevf_down(adapter);
1604         ixgbevf_up(adapter);
1605
1606         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1607 }
1608
1609 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1610 {
1611         struct ixgbe_hw *hw = &adapter->hw;
1612         struct net_device *netdev = adapter->netdev;
1613
1614         spin_lock(&adapter->mbx_lock);
1615
1616         if (hw->mac.ops.reset_hw(hw))
1617                 hw_dbg(hw, "PF still resetting\n");
1618         else
1619                 hw->mac.ops.init_hw(hw);
1620
1621         spin_unlock(&adapter->mbx_lock);
1622
1623         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1624                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1625                        netdev->addr_len);
1626                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1627                        netdev->addr_len);
1628         }
1629 }
1630
1631 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1632                                          int vectors)
1633 {
1634         int err, vector_threshold;
1635
1636         /* We'll want at least 2 (vector_threshold):
1637          * 1) TxQ[0] + RxQ[0] handler
1638          * 2) Other (Link Status Change, etc.)
1639          */
1640         vector_threshold = MIN_MSIX_COUNT;
1641
1642         /* The more we get, the more we will assign to Tx/Rx Cleanup
1643          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1644          * Right now, we simply care about how many we'll get; we'll
1645          * set them up later while requesting irq's.
1646          */
1647         while (vectors >= vector_threshold) {
1648                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1649                                       vectors);
1650                 if (!err) /* Success in acquiring all requested vectors. */
1651                         break;
1652                 else if (err < 0)
1653                         vectors = 0; /* Nasty failure, quit now */
1654                 else /* err == number of vectors we should try again with */
1655                         vectors = err;
1656         }
1657
1658         if (vectors < vector_threshold) {
1659                 /* Can't allocate enough MSI-X interrupts?  Oh well.
1660                  * This just means we'll go with either a single MSI
1661                  * vector or fall back to legacy interrupts.
1662                  */
1663                 hw_dbg(&adapter->hw,
1664                        "Unable to allocate MSI-X interrupts\n");
1665                 kfree(adapter->msix_entries);
1666                 adapter->msix_entries = NULL;
1667         } else {
1668                 /*
1669                  * Adjust for only the vectors we'll use, which is minimum
1670                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1671                  * vectors we were allocated.
1672                  */
1673                 adapter->num_msix_vectors = vectors;
1674         }
1675 }
1676
1677 /**
1678  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1679  * @adapter: board private structure to initialize
1680  *
1681  * This is the top level queue allocation routine.  The order here is very
1682  * important, starting with the "most" number of features turned on at once,
1683  * and ending with the smallest set of features.  This way large combinations
1684  * can be allocated if they're turned on, and smaller combinations are the
1685  * fallthrough conditions.
1686  *
1687  **/
1688 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1689 {
1690         /* Start with base case */
1691         adapter->num_rx_queues = 1;
1692         adapter->num_tx_queues = 1;
1693 }
1694
1695 /**
1696  * ixgbevf_alloc_queues - Allocate memory for all rings
1697  * @adapter: board private structure to initialize
1698  *
1699  * We allocate one ring per queue at run-time since we don't know the
1700  * number of queues at compile-time.  The polling_netdev array is
1701  * intended for Multiqueue, but should work fine with a single queue.
1702  **/
1703 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1704 {
1705         int i;
1706
1707         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1708                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1709         if (!adapter->tx_ring)
1710                 goto err_tx_ring_allocation;
1711
1712         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1713                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1714         if (!adapter->rx_ring)
1715                 goto err_rx_ring_allocation;
1716
1717         for (i = 0; i < adapter->num_tx_queues; i++) {
1718                 adapter->tx_ring[i].count = adapter->tx_ring_count;
1719                 adapter->tx_ring[i].queue_index = i;
1720                 adapter->tx_ring[i].reg_idx = i;
1721                 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1722                 adapter->tx_ring[i].netdev = adapter->netdev;
1723         }
1724
1725         for (i = 0; i < adapter->num_rx_queues; i++) {
1726                 adapter->rx_ring[i].count = adapter->rx_ring_count;
1727                 adapter->rx_ring[i].queue_index = i;
1728                 adapter->rx_ring[i].reg_idx = i;
1729                 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1730                 adapter->rx_ring[i].netdev = adapter->netdev;
1731         }
1732
1733         return 0;
1734
1735 err_rx_ring_allocation:
1736         kfree(adapter->tx_ring);
1737 err_tx_ring_allocation:
1738         return -ENOMEM;
1739 }
1740
1741 /**
1742  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1743  * @adapter: board private structure to initialize
1744  *
1745  * Attempt to configure the interrupts using the best available
1746  * capabilities of the hardware and the kernel.
1747  **/
1748 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1749 {
1750         int err = 0;
1751         int vector, v_budget;
1752
1753         /*
1754          * It's easy to be greedy for MSI-X vectors, but it really
1755          * doesn't do us much good if we have a lot more vectors
1756          * than CPU's.  So let's be conservative and only ask for
1757          * (roughly) the same number of vectors as there are CPU's.
1758          * The default is to use pairs of vectors.
1759          */
1760         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1761         v_budget = min_t(int, v_budget, num_online_cpus());
1762         v_budget += NON_Q_VECTORS;
1763
1764         /* A failure in MSI-X entry allocation isn't fatal, but it does
1765          * mean we disable MSI-X capabilities of the adapter. */
1766         adapter->msix_entries = kcalloc(v_budget,
1767                                         sizeof(struct msix_entry), GFP_KERNEL);
1768         if (!adapter->msix_entries) {
1769                 err = -ENOMEM;
1770                 goto out;
1771         }
1772
1773         for (vector = 0; vector < v_budget; vector++)
1774                 adapter->msix_entries[vector].entry = vector;
1775
1776         ixgbevf_acquire_msix_vectors(adapter, v_budget);
1777
1778 out:
1779         return err;
1780 }
1781
1782 /**
1783  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1784  * @adapter: board private structure to initialize
1785  *
1786  * We allocate one q_vector per queue interrupt.  If allocation fails we
1787  * return -ENOMEM.
1788  **/
1789 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1790 {
1791         int q_idx, num_q_vectors;
1792         struct ixgbevf_q_vector *q_vector;
1793
1794         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1795
1796         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1797                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1798                 if (!q_vector)
1799                         goto err_out;
1800                 q_vector->adapter = adapter;
1801                 q_vector->v_idx = q_idx;
1802                 netif_napi_add(adapter->netdev, &q_vector->napi,
1803                                ixgbevf_poll, 64);
1804                 adapter->q_vector[q_idx] = q_vector;
1805         }
1806
1807         return 0;
1808
1809 err_out:
1810         while (q_idx) {
1811                 q_idx--;
1812                 q_vector = adapter->q_vector[q_idx];
1813                 netif_napi_del(&q_vector->napi);
1814                 kfree(q_vector);
1815                 adapter->q_vector[q_idx] = NULL;
1816         }
1817         return -ENOMEM;
1818 }
1819
1820 /**
1821  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1822  * @adapter: board private structure to initialize
1823  *
1824  * This function frees the memory allocated to the q_vectors.  In addition if
1825  * NAPI is enabled it will delete any references to the NAPI struct prior
1826  * to freeing the q_vector.
1827  **/
1828 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1829 {
1830         int q_idx, num_q_vectors;
1831         int napi_vectors;
1832
1833         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1834         napi_vectors = adapter->num_rx_queues;
1835
1836         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1837                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1838
1839                 adapter->q_vector[q_idx] = NULL;
1840                 if (q_idx < napi_vectors)
1841                         netif_napi_del(&q_vector->napi);
1842                 kfree(q_vector);
1843         }
1844 }
1845
1846 /**
1847  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1848  * @adapter: board private structure
1849  *
1850  **/
1851 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1852 {
1853         pci_disable_msix(adapter->pdev);
1854         kfree(adapter->msix_entries);
1855         adapter->msix_entries = NULL;
1856 }
1857
1858 /**
1859  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1860  * @adapter: board private structure to initialize
1861  *
1862  **/
1863 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1864 {
1865         int err;
1866
1867         /* Number of supported queues */
1868         ixgbevf_set_num_queues(adapter);
1869
1870         err = ixgbevf_set_interrupt_capability(adapter);
1871         if (err) {
1872                 hw_dbg(&adapter->hw,
1873                        "Unable to setup interrupt capabilities\n");
1874                 goto err_set_interrupt;
1875         }
1876
1877         err = ixgbevf_alloc_q_vectors(adapter);
1878         if (err) {
1879                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
1880                        "vectors\n");
1881                 goto err_alloc_q_vectors;
1882         }
1883
1884         err = ixgbevf_alloc_queues(adapter);
1885         if (err) {
1886                 pr_err("Unable to allocate memory for queues\n");
1887                 goto err_alloc_queues;
1888         }
1889
1890         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
1891                "Tx Queue count = %u\n",
1892                (adapter->num_rx_queues > 1) ? "Enabled" :
1893                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
1894
1895         set_bit(__IXGBEVF_DOWN, &adapter->state);
1896
1897         return 0;
1898 err_alloc_queues:
1899         ixgbevf_free_q_vectors(adapter);
1900 err_alloc_q_vectors:
1901         ixgbevf_reset_interrupt_capability(adapter);
1902 err_set_interrupt:
1903         return err;
1904 }
1905
1906 /**
1907  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
1908  * @adapter: board private structure to clear interrupt scheme on
1909  *
1910  * We go through and clear interrupt specific resources and reset the structure
1911  * to pre-load conditions
1912  **/
1913 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
1914 {
1915         adapter->num_tx_queues = 0;
1916         adapter->num_rx_queues = 0;
1917
1918         ixgbevf_free_q_vectors(adapter);
1919         ixgbevf_reset_interrupt_capability(adapter);
1920 }
1921
1922 /**
1923  * ixgbevf_sw_init - Initialize general software structures
1924  * (struct ixgbevf_adapter)
1925  * @adapter: board private structure to initialize
1926  *
1927  * ixgbevf_sw_init initializes the Adapter private data structure.
1928  * Fields are initialized based on PCI device information and
1929  * OS network device settings (MTU size).
1930  **/
1931 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
1932 {
1933         struct ixgbe_hw *hw = &adapter->hw;
1934         struct pci_dev *pdev = adapter->pdev;
1935         int err;
1936
1937         /* PCI config space info */
1938
1939         hw->vendor_id = pdev->vendor;
1940         hw->device_id = pdev->device;
1941         hw->revision_id = pdev->revision;
1942         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1943         hw->subsystem_device_id = pdev->subsystem_device;
1944
1945         hw->mbx.ops.init_params(hw);
1946         hw->mac.max_tx_queues = MAX_TX_QUEUES;
1947         hw->mac.max_rx_queues = MAX_RX_QUEUES;
1948         err = hw->mac.ops.reset_hw(hw);
1949         if (err) {
1950                 dev_info(&pdev->dev,
1951                          "PF still in reset state, assigning new address\n");
1952                 eth_hw_addr_random(adapter->netdev);
1953                 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
1954                         adapter->netdev->addr_len);
1955         } else {
1956                 err = hw->mac.ops.init_hw(hw);
1957                 if (err) {
1958                         pr_err("init_shared_code failed: %d\n", err);
1959                         goto out;
1960                 }
1961                 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
1962                         adapter->netdev->addr_len);
1963         }
1964
1965         /* lock to protect mailbox accesses */
1966         spin_lock_init(&adapter->mbx_lock);
1967
1968         /* Enable dynamic interrupt throttling rates */
1969         adapter->rx_itr_setting = 1;
1970         adapter->tx_itr_setting = 1;
1971
1972         /* set default ring sizes */
1973         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
1974         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
1975
1976         set_bit(__IXGBEVF_DOWN, &adapter->state);
1977         return 0;
1978
1979 out:
1980         return err;
1981 }
1982
1983 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
1984         {                                                       \
1985                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
1986                 if (current_counter < last_counter)             \
1987                         counter += 0x100000000LL;               \
1988                 last_counter = current_counter;                 \
1989                 counter &= 0xFFFFFFFF00000000LL;                \
1990                 counter |= current_counter;                     \
1991         }
1992
1993 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1994         {                                                                \
1995                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
1996                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
1997                 u64 current_counter = (current_counter_msb << 32) |      \
1998                         current_counter_lsb;                             \
1999                 if (current_counter < last_counter)                      \
2000                         counter += 0x1000000000LL;                       \
2001                 last_counter = current_counter;                          \
2002                 counter &= 0xFFFFFFF000000000LL;                         \
2003                 counter |= current_counter;                              \
2004         }
2005 /**
2006  * ixgbevf_update_stats - Update the board statistics counters.
2007  * @adapter: board private structure
2008  **/
2009 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2010 {
2011         struct ixgbe_hw *hw = &adapter->hw;
2012
2013         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2014                                 adapter->stats.vfgprc);
2015         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2016                                 adapter->stats.vfgptc);
2017         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2018                                 adapter->stats.last_vfgorc,
2019                                 adapter->stats.vfgorc);
2020         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2021                                 adapter->stats.last_vfgotc,
2022                                 adapter->stats.vfgotc);
2023         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2024                                 adapter->stats.vfmprc);
2025 }
2026
2027 /**
2028  * ixgbevf_watchdog - Timer Call-back
2029  * @data: pointer to adapter cast into an unsigned long
2030  **/
2031 static void ixgbevf_watchdog(unsigned long data)
2032 {
2033         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2034         struct ixgbe_hw *hw = &adapter->hw;
2035         u32 eics = 0;
2036         int i;
2037
2038         /*
2039          * Do the watchdog outside of interrupt context due to the lovely
2040          * delays that some of the newer hardware requires
2041          */
2042
2043         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2044                 goto watchdog_short_circuit;
2045
2046         /* get one bit for every active tx/rx interrupt vector */
2047         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2048                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2049                 if (qv->rx.ring || qv->tx.ring)
2050                         eics |= 1 << i;
2051         }
2052
2053         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2054
2055 watchdog_short_circuit:
2056         schedule_work(&adapter->watchdog_task);
2057 }
2058
2059 /**
2060  * ixgbevf_tx_timeout - Respond to a Tx Hang
2061  * @netdev: network interface device structure
2062  **/
2063 static void ixgbevf_tx_timeout(struct net_device *netdev)
2064 {
2065         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2066
2067         /* Do the reset outside of interrupt context */
2068         schedule_work(&adapter->reset_task);
2069 }
2070
2071 static void ixgbevf_reset_task(struct work_struct *work)
2072 {
2073         struct ixgbevf_adapter *adapter;
2074         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2075
2076         /* If we're already down or resetting, just bail */
2077         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2078             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2079                 return;
2080
2081         adapter->tx_timeout_count++;
2082
2083         ixgbevf_reinit_locked(adapter);
2084 }
2085
2086 /**
2087  * ixgbevf_watchdog_task - worker thread to bring link up
2088  * @work: pointer to work_struct containing our data
2089  **/
2090 static void ixgbevf_watchdog_task(struct work_struct *work)
2091 {
2092         struct ixgbevf_adapter *adapter = container_of(work,
2093                                                        struct ixgbevf_adapter,
2094                                                        watchdog_task);
2095         struct net_device *netdev = adapter->netdev;
2096         struct ixgbe_hw *hw = &adapter->hw;
2097         u32 link_speed = adapter->link_speed;
2098         bool link_up = adapter->link_up;
2099
2100         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2101
2102         /*
2103          * Always check the link on the watchdog because we have
2104          * no LSC interrupt
2105          */
2106         if (hw->mac.ops.check_link) {
2107                 s32 need_reset;
2108
2109                 spin_lock(&adapter->mbx_lock);
2110
2111                 need_reset = hw->mac.ops.check_link(hw, &link_speed,
2112                                                     &link_up, false);
2113
2114                 spin_unlock(&adapter->mbx_lock);
2115
2116                 if (need_reset) {
2117                         adapter->link_up = link_up;
2118                         adapter->link_speed = link_speed;
2119                         netif_carrier_off(netdev);
2120                         netif_tx_stop_all_queues(netdev);
2121                         schedule_work(&adapter->reset_task);
2122                         goto pf_has_reset;
2123                 }
2124         } else {
2125                 /* always assume link is up, if no check link
2126                  * function */
2127                 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2128                 link_up = true;
2129         }
2130         adapter->link_up = link_up;
2131         adapter->link_speed = link_speed;
2132
2133         if (link_up) {
2134                 if (!netif_carrier_ok(netdev)) {
2135                         hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2136                                (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2137                                10 : 1);
2138                         netif_carrier_on(netdev);
2139                         netif_tx_wake_all_queues(netdev);
2140                 }
2141         } else {
2142                 adapter->link_up = false;
2143                 adapter->link_speed = 0;
2144                 if (netif_carrier_ok(netdev)) {
2145                         hw_dbg(&adapter->hw, "NIC Link is Down\n");
2146                         netif_carrier_off(netdev);
2147                         netif_tx_stop_all_queues(netdev);
2148                 }
2149         }
2150
2151         ixgbevf_update_stats(adapter);
2152
2153 pf_has_reset:
2154         /* Reset the timer */
2155         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2156                 mod_timer(&adapter->watchdog_timer,
2157                           round_jiffies(jiffies + (2 * HZ)));
2158
2159         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2160 }
2161
2162 /**
2163  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2164  * @adapter: board private structure
2165  * @tx_ring: Tx descriptor ring for a specific queue
2166  *
2167  * Free all transmit software resources
2168  **/
2169 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2170                                struct ixgbevf_ring *tx_ring)
2171 {
2172         struct pci_dev *pdev = adapter->pdev;
2173
2174         ixgbevf_clean_tx_ring(adapter, tx_ring);
2175
2176         vfree(tx_ring->tx_buffer_info);
2177         tx_ring->tx_buffer_info = NULL;
2178
2179         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2180                           tx_ring->dma);
2181
2182         tx_ring->desc = NULL;
2183 }
2184
2185 /**
2186  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2187  * @adapter: board private structure
2188  *
2189  * Free all transmit software resources
2190  **/
2191 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2192 {
2193         int i;
2194
2195         for (i = 0; i < adapter->num_tx_queues; i++)
2196                 if (adapter->tx_ring[i].desc)
2197                         ixgbevf_free_tx_resources(adapter,
2198                                                   &adapter->tx_ring[i]);
2199
2200 }
2201
2202 /**
2203  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2204  * @adapter: board private structure
2205  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2206  *
2207  * Return 0 on success, negative on failure
2208  **/
2209 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2210                                struct ixgbevf_ring *tx_ring)
2211 {
2212         struct pci_dev *pdev = adapter->pdev;
2213         int size;
2214
2215         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2216         tx_ring->tx_buffer_info = vzalloc(size);
2217         if (!tx_ring->tx_buffer_info)
2218                 goto err;
2219
2220         /* round up to nearest 4K */
2221         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2222         tx_ring->size = ALIGN(tx_ring->size, 4096);
2223
2224         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2225                                            &tx_ring->dma, GFP_KERNEL);
2226         if (!tx_ring->desc)
2227                 goto err;
2228
2229         tx_ring->next_to_use = 0;
2230         tx_ring->next_to_clean = 0;
2231         return 0;
2232
2233 err:
2234         vfree(tx_ring->tx_buffer_info);
2235         tx_ring->tx_buffer_info = NULL;
2236         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2237                "descriptor ring\n");
2238         return -ENOMEM;
2239 }
2240
2241 /**
2242  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2243  * @adapter: board private structure
2244  *
2245  * If this function returns with an error, then it's possible one or
2246  * more of the rings is populated (while the rest are not).  It is the
2247  * callers duty to clean those orphaned rings.
2248  *
2249  * Return 0 on success, negative on failure
2250  **/
2251 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2252 {
2253         int i, err = 0;
2254
2255         for (i = 0; i < adapter->num_tx_queues; i++) {
2256                 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2257                 if (!err)
2258                         continue;
2259                 hw_dbg(&adapter->hw,
2260                        "Allocation for Tx Queue %u failed\n", i);
2261                 break;
2262         }
2263
2264         return err;
2265 }
2266
2267 /**
2268  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2269  * @adapter: board private structure
2270  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2271  *
2272  * Returns 0 on success, negative on failure
2273  **/
2274 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2275                                struct ixgbevf_ring *rx_ring)
2276 {
2277         struct pci_dev *pdev = adapter->pdev;
2278         int size;
2279
2280         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2281         rx_ring->rx_buffer_info = vzalloc(size);
2282         if (!rx_ring->rx_buffer_info)
2283                 goto alloc_failed;
2284
2285         /* Round up to nearest 4K */
2286         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2287         rx_ring->size = ALIGN(rx_ring->size, 4096);
2288
2289         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2290                                            &rx_ring->dma, GFP_KERNEL);
2291
2292         if (!rx_ring->desc) {
2293                 hw_dbg(&adapter->hw,
2294                        "Unable to allocate memory for "
2295                        "the receive descriptor ring\n");
2296                 vfree(rx_ring->rx_buffer_info);
2297                 rx_ring->rx_buffer_info = NULL;
2298                 goto alloc_failed;
2299         }
2300
2301         rx_ring->next_to_clean = 0;
2302         rx_ring->next_to_use = 0;
2303
2304         return 0;
2305 alloc_failed:
2306         return -ENOMEM;
2307 }
2308
2309 /**
2310  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2311  * @adapter: board private structure
2312  *
2313  * If this function returns with an error, then it's possible one or
2314  * more of the rings is populated (while the rest are not).  It is the
2315  * callers duty to clean those orphaned rings.
2316  *
2317  * Return 0 on success, negative on failure
2318  **/
2319 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2320 {
2321         int i, err = 0;
2322
2323         for (i = 0; i < adapter->num_rx_queues; i++) {
2324                 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2325                 if (!err)
2326                         continue;
2327                 hw_dbg(&adapter->hw,
2328                        "Allocation for Rx Queue %u failed\n", i);
2329                 break;
2330         }
2331         return err;
2332 }
2333
2334 /**
2335  * ixgbevf_free_rx_resources - Free Rx Resources
2336  * @adapter: board private structure
2337  * @rx_ring: ring to clean the resources from
2338  *
2339  * Free all receive software resources
2340  **/
2341 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2342                                struct ixgbevf_ring *rx_ring)
2343 {
2344         struct pci_dev *pdev = adapter->pdev;
2345
2346         ixgbevf_clean_rx_ring(adapter, rx_ring);
2347
2348         vfree(rx_ring->rx_buffer_info);
2349         rx_ring->rx_buffer_info = NULL;
2350
2351         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2352                           rx_ring->dma);
2353
2354         rx_ring->desc = NULL;
2355 }
2356
2357 /**
2358  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2359  * @adapter: board private structure
2360  *
2361  * Free all receive software resources
2362  **/
2363 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2364 {
2365         int i;
2366
2367         for (i = 0; i < adapter->num_rx_queues; i++)
2368                 if (adapter->rx_ring[i].desc)
2369                         ixgbevf_free_rx_resources(adapter,
2370                                                   &adapter->rx_ring[i]);
2371 }
2372
2373 /**
2374  * ixgbevf_open - Called when a network interface is made active
2375  * @netdev: network interface device structure
2376  *
2377  * Returns 0 on success, negative value on failure
2378  *
2379  * The open entry point is called when a network interface is made
2380  * active by the system (IFF_UP).  At this point all resources needed
2381  * for transmit and receive operations are allocated, the interrupt
2382  * handler is registered with the OS, the watchdog timer is started,
2383  * and the stack is notified that the interface is ready.
2384  **/
2385 static int ixgbevf_open(struct net_device *netdev)
2386 {
2387         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2388         struct ixgbe_hw *hw = &adapter->hw;
2389         int err;
2390
2391         /* disallow open during test */
2392         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2393                 return -EBUSY;
2394
2395         if (hw->adapter_stopped) {
2396                 ixgbevf_reset(adapter);
2397                 /* if adapter is still stopped then PF isn't up and
2398                  * the vf can't start. */
2399                 if (hw->adapter_stopped) {
2400                         err = IXGBE_ERR_MBX;
2401                         pr_err("Unable to start - perhaps the PF Driver isn't "
2402                                "up yet\n");
2403                         goto err_setup_reset;
2404                 }
2405         }
2406
2407         ixgbevf_negotiate_api(adapter);
2408
2409         /* allocate transmit descriptors */
2410         err = ixgbevf_setup_all_tx_resources(adapter);
2411         if (err)
2412                 goto err_setup_tx;
2413
2414         /* allocate receive descriptors */
2415         err = ixgbevf_setup_all_rx_resources(adapter);
2416         if (err)
2417                 goto err_setup_rx;
2418
2419         ixgbevf_configure(adapter);
2420
2421         /*
2422          * Map the Tx/Rx rings to the vectors we were allotted.
2423          * if request_irq will be called in this function map_rings
2424          * must be called *before* up_complete
2425          */
2426         ixgbevf_map_rings_to_vectors(adapter);
2427
2428         ixgbevf_up_complete(adapter);
2429
2430         /* clear any pending interrupts, may auto mask */
2431         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2432         err = ixgbevf_request_irq(adapter);
2433         if (err)
2434                 goto err_req_irq;
2435
2436         ixgbevf_irq_enable(adapter);
2437
2438         return 0;
2439
2440 err_req_irq:
2441         ixgbevf_down(adapter);
2442         ixgbevf_free_irq(adapter);
2443 err_setup_rx:
2444         ixgbevf_free_all_rx_resources(adapter);
2445 err_setup_tx:
2446         ixgbevf_free_all_tx_resources(adapter);
2447         ixgbevf_reset(adapter);
2448
2449 err_setup_reset:
2450
2451         return err;
2452 }
2453
2454 /**
2455  * ixgbevf_close - Disables a network interface
2456  * @netdev: network interface device structure
2457  *
2458  * Returns 0, this is not allowed to fail
2459  *
2460  * The close entry point is called when an interface is de-activated
2461  * by the OS.  The hardware is still under the drivers control, but
2462  * needs to be disabled.  A global MAC reset is issued to stop the
2463  * hardware, and all transmit and receive resources are freed.
2464  **/
2465 static int ixgbevf_close(struct net_device *netdev)
2466 {
2467         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2468
2469         ixgbevf_down(adapter);
2470         ixgbevf_free_irq(adapter);
2471
2472         ixgbevf_free_all_tx_resources(adapter);
2473         ixgbevf_free_all_rx_resources(adapter);
2474
2475         return 0;
2476 }
2477
2478 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2479                                 u32 vlan_macip_lens, u32 type_tucmd,
2480                                 u32 mss_l4len_idx)
2481 {
2482         struct ixgbe_adv_tx_context_desc *context_desc;
2483         u16 i = tx_ring->next_to_use;
2484
2485         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2486
2487         i++;
2488         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2489
2490         /* set bits to identify this as an advanced context descriptor */
2491         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2492
2493         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
2494         context_desc->seqnum_seed       = 0;
2495         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
2496         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
2497 }
2498
2499 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2500                        struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2501 {
2502         u32 vlan_macip_lens, type_tucmd;
2503         u32 mss_l4len_idx, l4len;
2504
2505         if (!skb_is_gso(skb))
2506                 return 0;
2507
2508         if (skb_header_cloned(skb)) {
2509                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2510                 if (err)
2511                         return err;
2512         }
2513
2514         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2515         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2516
2517         if (skb->protocol == htons(ETH_P_IP)) {
2518                 struct iphdr *iph = ip_hdr(skb);
2519                 iph->tot_len = 0;
2520                 iph->check = 0;
2521                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2522                                                          iph->daddr, 0,
2523                                                          IPPROTO_TCP,
2524                                                          0);
2525                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2526         } else if (skb_is_gso_v6(skb)) {
2527                 ipv6_hdr(skb)->payload_len = 0;
2528                 tcp_hdr(skb)->check =
2529                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2530                                      &ipv6_hdr(skb)->daddr,
2531                                      0, IPPROTO_TCP, 0);
2532         }
2533
2534         /* compute header lengths */
2535         l4len = tcp_hdrlen(skb);
2536         *hdr_len += l4len;
2537         *hdr_len = skb_transport_offset(skb) + l4len;
2538
2539         /* mss_l4len_id: use 1 as index for TSO */
2540         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2541         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2542         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2543
2544         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2545         vlan_macip_lens = skb_network_header_len(skb);
2546         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2547         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2548
2549         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2550                             type_tucmd, mss_l4len_idx);
2551
2552         return 1;
2553 }
2554
2555 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2556                             struct sk_buff *skb, u32 tx_flags)
2557 {
2558
2559
2560
2561         u32 vlan_macip_lens = 0;
2562         u32 mss_l4len_idx = 0;
2563         u32 type_tucmd = 0;
2564
2565         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2566                 u8 l4_hdr = 0;
2567                 switch (skb->protocol) {
2568                 case __constant_htons(ETH_P_IP):
2569                         vlan_macip_lens |= skb_network_header_len(skb);
2570                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2571                         l4_hdr = ip_hdr(skb)->protocol;
2572                         break;
2573                 case __constant_htons(ETH_P_IPV6):
2574                         vlan_macip_lens |= skb_network_header_len(skb);
2575                         l4_hdr = ipv6_hdr(skb)->nexthdr;
2576                         break;
2577                 default:
2578                         if (unlikely(net_ratelimit())) {
2579                                 dev_warn(tx_ring->dev,
2580                                  "partial checksum but proto=%x!\n",
2581                                  skb->protocol);
2582                         }
2583                         break;
2584                 }
2585
2586                 switch (l4_hdr) {
2587                 case IPPROTO_TCP:
2588                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2589                         mss_l4len_idx = tcp_hdrlen(skb) <<
2590                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2591                         break;
2592                 case IPPROTO_SCTP:
2593                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2594                         mss_l4len_idx = sizeof(struct sctphdr) <<
2595                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2596                         break;
2597                 case IPPROTO_UDP:
2598                         mss_l4len_idx = sizeof(struct udphdr) <<
2599                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2600                         break;
2601                 default:
2602                         if (unlikely(net_ratelimit())) {
2603                                 dev_warn(tx_ring->dev,
2604                                  "partial checksum but l4 proto=%x!\n",
2605                                  l4_hdr);
2606                         }
2607                         break;
2608                 }
2609         }
2610
2611         /* vlan_macip_lens: MACLEN, VLAN tag */
2612         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2613         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2614
2615         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2616                             type_tucmd, mss_l4len_idx);
2617
2618         return (skb->ip_summed == CHECKSUM_PARTIAL);
2619 }
2620
2621 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2622                           struct sk_buff *skb, u32 tx_flags,
2623                           unsigned int first)
2624 {
2625         struct ixgbevf_tx_buffer *tx_buffer_info;
2626         unsigned int len;
2627         unsigned int total = skb->len;
2628         unsigned int offset = 0, size;
2629         int count = 0;
2630         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2631         unsigned int f;
2632         int i;
2633
2634         i = tx_ring->next_to_use;
2635
2636         len = min(skb_headlen(skb), total);
2637         while (len) {
2638                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2639                 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2640
2641                 tx_buffer_info->length = size;
2642                 tx_buffer_info->mapped_as_page = false;
2643                 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
2644                                                      skb->data + offset,
2645                                                      size, DMA_TO_DEVICE);
2646                 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
2647                         goto dma_error;
2648                 tx_buffer_info->next_to_watch = i;
2649
2650                 len -= size;
2651                 total -= size;
2652                 offset += size;
2653                 count++;
2654                 i++;
2655                 if (i == tx_ring->count)
2656                         i = 0;
2657         }
2658
2659         for (f = 0; f < nr_frags; f++) {
2660                 const struct skb_frag_struct *frag;
2661
2662                 frag = &skb_shinfo(skb)->frags[f];
2663                 len = min((unsigned int)skb_frag_size(frag), total);
2664                 offset = 0;
2665
2666                 while (len) {
2667                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
2668                         size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2669
2670                         tx_buffer_info->length = size;
2671                         tx_buffer_info->dma =
2672                                 skb_frag_dma_map(tx_ring->dev, frag,
2673                                                  offset, size, DMA_TO_DEVICE);
2674                         tx_buffer_info->mapped_as_page = true;
2675                         if (dma_mapping_error(tx_ring->dev,
2676                                               tx_buffer_info->dma))
2677                                 goto dma_error;
2678                         tx_buffer_info->next_to_watch = i;
2679
2680                         len -= size;
2681                         total -= size;
2682                         offset += size;
2683                         count++;
2684                         i++;
2685                         if (i == tx_ring->count)
2686                                 i = 0;
2687                 }
2688                 if (total == 0)
2689                         break;
2690         }
2691
2692         if (i == 0)
2693                 i = tx_ring->count - 1;
2694         else
2695                 i = i - 1;
2696         tx_ring->tx_buffer_info[i].skb = skb;
2697         tx_ring->tx_buffer_info[first].next_to_watch = i;
2698         tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2699
2700         return count;
2701
2702 dma_error:
2703         dev_err(tx_ring->dev, "TX DMA map failed\n");
2704
2705         /* clear timestamp and dma mappings for failed tx_buffer_info map */
2706         tx_buffer_info->dma = 0;
2707         tx_buffer_info->next_to_watch = 0;
2708         count--;
2709
2710         /* clear timestamp and dma mappings for remaining portion of packet */
2711         while (count >= 0) {
2712                 count--;
2713                 i--;
2714                 if (i < 0)
2715                         i += tx_ring->count;
2716                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2717                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2718         }
2719
2720         return count;
2721 }
2722
2723 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
2724                              int count, u32 paylen, u8 hdr_len)
2725 {
2726         union ixgbe_adv_tx_desc *tx_desc = NULL;
2727         struct ixgbevf_tx_buffer *tx_buffer_info;
2728         u32 olinfo_status = 0, cmd_type_len = 0;
2729         unsigned int i;
2730
2731         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2732
2733         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2734
2735         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2736
2737         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2738                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2739
2740         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2741                 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2742
2743         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2744                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2745
2746                 /* use index 1 context for tso */
2747                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2748                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2749                         olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
2750
2751         }
2752
2753         /*
2754          * Check Context must be set if Tx switch is enabled, which it
2755          * always is for case where virtual functions are running
2756          */
2757         olinfo_status |= IXGBE_ADVTXD_CC;
2758
2759         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2760
2761         i = tx_ring->next_to_use;
2762         while (count--) {
2763                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2764                 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2765                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2766                 tx_desc->read.cmd_type_len =
2767                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2768                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2769                 i++;
2770                 if (i == tx_ring->count)
2771                         i = 0;
2772         }
2773
2774         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2775
2776         tx_ring->next_to_use = i;
2777 }
2778
2779 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2780 {
2781         struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
2782
2783         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2784         /* Herbert's original patch had:
2785          *  smp_mb__after_netif_stop_queue();
2786          * but since that doesn't exist yet, just open code it. */
2787         smp_mb();
2788
2789         /* We need to check again in a case another CPU has just
2790          * made room available. */
2791         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
2792                 return -EBUSY;
2793
2794         /* A reprieve! - use start_queue because it doesn't call schedule */
2795         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2796         ++adapter->restart_queue;
2797         return 0;
2798 }
2799
2800 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2801 {
2802         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
2803                 return 0;
2804         return __ixgbevf_maybe_stop_tx(tx_ring, size);
2805 }
2806
2807 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2808 {
2809         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2810         struct ixgbevf_ring *tx_ring;
2811         unsigned int first;
2812         unsigned int tx_flags = 0;
2813         u8 hdr_len = 0;
2814         int r_idx = 0, tso;
2815         u16 count = TXD_USE_COUNT(skb_headlen(skb));
2816 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2817         unsigned short f;
2818 #endif
2819
2820         tx_ring = &adapter->tx_ring[r_idx];
2821
2822         /*
2823          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
2824          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
2825          *       + 2 desc gap to keep tail from touching head,
2826          *       + 1 desc for context descriptor,
2827          * otherwise try next time
2828          */
2829 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2830         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2831                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2832 #else
2833         count += skb_shinfo(skb)->nr_frags;
2834 #endif
2835         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
2836                 adapter->tx_busy++;
2837                 return NETDEV_TX_BUSY;
2838         }
2839
2840         if (vlan_tx_tag_present(skb)) {
2841                 tx_flags |= vlan_tx_tag_get(skb);
2842                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
2843                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
2844         }
2845
2846         first = tx_ring->next_to_use;
2847
2848         if (skb->protocol == htons(ETH_P_IP))
2849                 tx_flags |= IXGBE_TX_FLAGS_IPV4;
2850         tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
2851         if (tso < 0) {
2852                 dev_kfree_skb_any(skb);
2853                 return NETDEV_TX_OK;
2854         }
2855
2856         if (tso)
2857                 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
2858         else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
2859                 tx_flags |= IXGBE_TX_FLAGS_CSUM;
2860
2861         ixgbevf_tx_queue(tx_ring, tx_flags,
2862                          ixgbevf_tx_map(tx_ring, skb, tx_flags, first),
2863                          skb->len, hdr_len);
2864         /*
2865          * Force memory writes to complete before letting h/w
2866          * know there are new descriptors to fetch.  (Only
2867          * applicable for weak-ordered memory model archs,
2868          * such as IA-64).
2869          */
2870         wmb();
2871
2872         writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
2873
2874         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
2875
2876         return NETDEV_TX_OK;
2877 }
2878
2879 /**
2880  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
2881  * @netdev: network interface device structure
2882  * @p: pointer to an address structure
2883  *
2884  * Returns 0 on success, negative on failure
2885  **/
2886 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
2887 {
2888         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2889         struct ixgbe_hw *hw = &adapter->hw;
2890         struct sockaddr *addr = p;
2891
2892         if (!is_valid_ether_addr(addr->sa_data))
2893                 return -EADDRNOTAVAIL;
2894
2895         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2896         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2897
2898         spin_lock(&adapter->mbx_lock);
2899
2900         if (hw->mac.ops.set_rar)
2901                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2902
2903         spin_unlock(&adapter->mbx_lock);
2904
2905         return 0;
2906 }
2907
2908 /**
2909  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
2910  * @netdev: network interface device structure
2911  * @new_mtu: new value for maximum frame size
2912  *
2913  * Returns 0 on success, negative on failure
2914  **/
2915 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
2916 {
2917         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2918         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2919         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
2920
2921         if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
2922                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
2923
2924         /* MTU < 68 is an error and causes problems on some kernels */
2925         if ((new_mtu < 68) || (max_frame > max_possible_frame))
2926                 return -EINVAL;
2927
2928         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
2929                netdev->mtu, new_mtu);
2930         /* must set new MTU before calling down or up */
2931         netdev->mtu = new_mtu;
2932
2933         if (netif_running(netdev))
2934                 ixgbevf_reinit_locked(adapter);
2935
2936         return 0;
2937 }
2938
2939 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
2940 {
2941         struct net_device *netdev = pci_get_drvdata(pdev);
2942         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2943 #ifdef CONFIG_PM
2944         int retval = 0;
2945 #endif
2946
2947         netif_device_detach(netdev);
2948
2949         if (netif_running(netdev)) {
2950                 rtnl_lock();
2951                 ixgbevf_down(adapter);
2952                 ixgbevf_free_irq(adapter);
2953                 ixgbevf_free_all_tx_resources(adapter);
2954                 ixgbevf_free_all_rx_resources(adapter);
2955                 rtnl_unlock();
2956         }
2957
2958         ixgbevf_clear_interrupt_scheme(adapter);
2959
2960 #ifdef CONFIG_PM
2961         retval = pci_save_state(pdev);
2962         if (retval)
2963                 return retval;
2964
2965 #endif
2966         pci_disable_device(pdev);
2967
2968         return 0;
2969 }
2970
2971 #ifdef CONFIG_PM
2972 static int ixgbevf_resume(struct pci_dev *pdev)
2973 {
2974         struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev);
2975         struct net_device *netdev = adapter->netdev;
2976         u32 err;
2977
2978         pci_set_power_state(pdev, PCI_D0);
2979         pci_restore_state(pdev);
2980         /*
2981          * pci_restore_state clears dev->state_saved so call
2982          * pci_save_state to restore it.
2983          */
2984         pci_save_state(pdev);
2985
2986         err = pci_enable_device_mem(pdev);
2987         if (err) {
2988                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
2989                 return err;
2990         }
2991         pci_set_master(pdev);
2992
2993         rtnl_lock();
2994         err = ixgbevf_init_interrupt_scheme(adapter);
2995         rtnl_unlock();
2996         if (err) {
2997                 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
2998                 return err;
2999         }
3000
3001         ixgbevf_reset(adapter);
3002
3003         if (netif_running(netdev)) {
3004                 err = ixgbevf_open(netdev);
3005                 if (err)
3006                         return err;
3007         }
3008
3009         netif_device_attach(netdev);
3010
3011         return err;
3012 }
3013
3014 #endif /* CONFIG_PM */
3015 static void ixgbevf_shutdown(struct pci_dev *pdev)
3016 {
3017         ixgbevf_suspend(pdev, PMSG_SUSPEND);
3018 }
3019
3020 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3021                                                 struct rtnl_link_stats64 *stats)
3022 {
3023         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3024         unsigned int start;
3025         u64 bytes, packets;
3026         const struct ixgbevf_ring *ring;
3027         int i;
3028
3029         ixgbevf_update_stats(adapter);
3030
3031         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3032
3033         for (i = 0; i < adapter->num_rx_queues; i++) {
3034                 ring = &adapter->rx_ring[i];
3035                 do {
3036                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3037                         bytes = ring->total_bytes;
3038                         packets = ring->total_packets;
3039                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3040                 stats->rx_bytes += bytes;
3041                 stats->rx_packets += packets;
3042         }
3043
3044         for (i = 0; i < adapter->num_tx_queues; i++) {
3045                 ring = &adapter->tx_ring[i];
3046                 do {
3047                         start = u64_stats_fetch_begin_bh(&ring->syncp);
3048                         bytes = ring->total_bytes;
3049                         packets = ring->total_packets;
3050                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
3051                 stats->tx_bytes += bytes;
3052                 stats->tx_packets += packets;
3053         }
3054
3055         return stats;
3056 }
3057
3058 static const struct net_device_ops ixgbevf_netdev_ops = {
3059         .ndo_open               = ixgbevf_open,
3060         .ndo_stop               = ixgbevf_close,
3061         .ndo_start_xmit         = ixgbevf_xmit_frame,
3062         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3063         .ndo_get_stats64        = ixgbevf_get_stats,
3064         .ndo_validate_addr      = eth_validate_addr,
3065         .ndo_set_mac_address    = ixgbevf_set_mac,
3066         .ndo_change_mtu         = ixgbevf_change_mtu,
3067         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3068         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3069         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3070 };
3071
3072 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3073 {
3074         dev->netdev_ops = &ixgbevf_netdev_ops;
3075         ixgbevf_set_ethtool_ops(dev);
3076         dev->watchdog_timeo = 5 * HZ;
3077 }
3078
3079 /**
3080  * ixgbevf_probe - Device Initialization Routine
3081  * @pdev: PCI device information struct
3082  * @ent: entry in ixgbevf_pci_tbl
3083  *
3084  * Returns 0 on success, negative on failure
3085  *
3086  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3087  * The OS initialization, configuring of the adapter private structure,
3088  * and a hardware reset occur.
3089  **/
3090 static int __devinit ixgbevf_probe(struct pci_dev *pdev,
3091                                    const struct pci_device_id *ent)
3092 {
3093         struct net_device *netdev;
3094         struct ixgbevf_adapter *adapter = NULL;
3095         struct ixgbe_hw *hw = NULL;
3096         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3097         static int cards_found;
3098         int err, pci_using_dac;
3099
3100         err = pci_enable_device(pdev);
3101         if (err)
3102                 return err;
3103
3104         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
3105             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
3106                 pci_using_dac = 1;
3107         } else {
3108                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
3109                 if (err) {
3110                         err = dma_set_coherent_mask(&pdev->dev,
3111                                                     DMA_BIT_MASK(32));
3112                         if (err) {
3113                                 dev_err(&pdev->dev, "No usable DMA "
3114                                         "configuration, aborting\n");
3115                                 goto err_dma;
3116                         }
3117                 }
3118                 pci_using_dac = 0;
3119         }
3120
3121         err = pci_request_regions(pdev, ixgbevf_driver_name);
3122         if (err) {
3123                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3124                 goto err_pci_reg;
3125         }
3126
3127         pci_set_master(pdev);
3128
3129         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3130                                    MAX_TX_QUEUES);
3131         if (!netdev) {
3132                 err = -ENOMEM;
3133                 goto err_alloc_etherdev;
3134         }
3135
3136         SET_NETDEV_DEV(netdev, &pdev->dev);
3137
3138         pci_set_drvdata(pdev, netdev);
3139         adapter = netdev_priv(netdev);
3140
3141         adapter->netdev = netdev;
3142         adapter->pdev = pdev;
3143         hw = &adapter->hw;
3144         hw->back = adapter;
3145         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3146
3147         /*
3148          * call save state here in standalone driver because it relies on
3149          * adapter struct to exist, and needs to call netdev_priv
3150          */
3151         pci_save_state(pdev);
3152
3153         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3154                               pci_resource_len(pdev, 0));
3155         if (!hw->hw_addr) {
3156                 err = -EIO;
3157                 goto err_ioremap;
3158         }
3159
3160         ixgbevf_assign_netdev_ops(netdev);
3161
3162         adapter->bd_number = cards_found;
3163
3164         /* Setup hw api */
3165         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3166         hw->mac.type  = ii->mac;
3167
3168         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3169                sizeof(struct ixgbe_mbx_operations));
3170
3171         /* setup the private structure */
3172         err = ixgbevf_sw_init(adapter);
3173         if (err)
3174                 goto err_sw_init;
3175
3176         /* The HW MAC address was set and/or determined in sw_init */
3177         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3178
3179         if (!is_valid_ether_addr(netdev->dev_addr)) {
3180                 pr_err("invalid MAC address\n");
3181                 err = -EIO;
3182                 goto err_sw_init;
3183         }
3184
3185         netdev->hw_features = NETIF_F_SG |
3186                            NETIF_F_IP_CSUM |
3187                            NETIF_F_IPV6_CSUM |
3188                            NETIF_F_TSO |
3189                            NETIF_F_TSO6 |
3190                            NETIF_F_RXCSUM;
3191
3192         netdev->features = netdev->hw_features |
3193                            NETIF_F_HW_VLAN_TX |
3194                            NETIF_F_HW_VLAN_RX |
3195                            NETIF_F_HW_VLAN_FILTER;
3196
3197         netdev->vlan_features |= NETIF_F_TSO;
3198         netdev->vlan_features |= NETIF_F_TSO6;
3199         netdev->vlan_features |= NETIF_F_IP_CSUM;
3200         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3201         netdev->vlan_features |= NETIF_F_SG;
3202
3203         if (pci_using_dac)
3204                 netdev->features |= NETIF_F_HIGHDMA;
3205
3206         netdev->priv_flags |= IFF_UNICAST_FLT;
3207
3208         init_timer(&adapter->watchdog_timer);
3209         adapter->watchdog_timer.function = ixgbevf_watchdog;
3210         adapter->watchdog_timer.data = (unsigned long)adapter;
3211
3212         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3213         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3214
3215         err = ixgbevf_init_interrupt_scheme(adapter);
3216         if (err)
3217                 goto err_sw_init;
3218
3219         /* pick up the PCI bus settings for reporting later */
3220         if (hw->mac.ops.get_bus_info)
3221                 hw->mac.ops.get_bus_info(hw);
3222
3223         strcpy(netdev->name, "eth%d");
3224
3225         err = register_netdev(netdev);
3226         if (err)
3227                 goto err_register;
3228
3229         netif_carrier_off(netdev);
3230
3231         ixgbevf_init_last_counter_stats(adapter);
3232
3233         /* print the MAC address */
3234         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3235
3236         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3237
3238         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3239         cards_found++;
3240         return 0;
3241
3242 err_register:
3243         ixgbevf_clear_interrupt_scheme(adapter);
3244 err_sw_init:
3245         ixgbevf_reset_interrupt_capability(adapter);
3246         iounmap(hw->hw_addr);
3247 err_ioremap:
3248         free_netdev(netdev);
3249 err_alloc_etherdev:
3250         pci_release_regions(pdev);
3251 err_pci_reg:
3252 err_dma:
3253         pci_disable_device(pdev);
3254         return err;
3255 }
3256
3257 /**
3258  * ixgbevf_remove - Device Removal Routine
3259  * @pdev: PCI device information struct
3260  *
3261  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3262  * that it should release a PCI device.  The could be caused by a
3263  * Hot-Plug event, or because the driver is going to be removed from
3264  * memory.
3265  **/
3266 static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3267 {
3268         struct net_device *netdev = pci_get_drvdata(pdev);
3269         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3270
3271         set_bit(__IXGBEVF_DOWN, &adapter->state);
3272
3273         del_timer_sync(&adapter->watchdog_timer);
3274
3275         cancel_work_sync(&adapter->reset_task);
3276         cancel_work_sync(&adapter->watchdog_task);
3277
3278         if (netdev->reg_state == NETREG_REGISTERED)
3279                 unregister_netdev(netdev);
3280
3281         ixgbevf_clear_interrupt_scheme(adapter);
3282         ixgbevf_reset_interrupt_capability(adapter);
3283
3284         iounmap(adapter->hw.hw_addr);
3285         pci_release_regions(pdev);
3286
3287         hw_dbg(&adapter->hw, "Remove complete\n");
3288
3289         kfree(adapter->tx_ring);
3290         kfree(adapter->rx_ring);
3291
3292         free_netdev(netdev);
3293
3294         pci_disable_device(pdev);
3295 }
3296
3297 /**
3298  * ixgbevf_io_error_detected - called when PCI error is detected
3299  * @pdev: Pointer to PCI device
3300  * @state: The current pci connection state
3301  *
3302  * This function is called after a PCI bus error affecting
3303  * this device has been detected.
3304  */
3305 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3306                                                   pci_channel_state_t state)
3307 {
3308         struct net_device *netdev = pci_get_drvdata(pdev);
3309         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3310
3311         netif_device_detach(netdev);
3312
3313         if (state == pci_channel_io_perm_failure)
3314                 return PCI_ERS_RESULT_DISCONNECT;
3315
3316         if (netif_running(netdev))
3317                 ixgbevf_down(adapter);
3318
3319         pci_disable_device(pdev);
3320
3321         /* Request a slot slot reset. */
3322         return PCI_ERS_RESULT_NEED_RESET;
3323 }
3324
3325 /**
3326  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3327  * @pdev: Pointer to PCI device
3328  *
3329  * Restart the card from scratch, as if from a cold-boot. Implementation
3330  * resembles the first-half of the ixgbevf_resume routine.
3331  */
3332 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3333 {
3334         struct net_device *netdev = pci_get_drvdata(pdev);
3335         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3336
3337         if (pci_enable_device_mem(pdev)) {
3338                 dev_err(&pdev->dev,
3339                         "Cannot re-enable PCI device after reset.\n");
3340                 return PCI_ERS_RESULT_DISCONNECT;
3341         }
3342
3343         pci_set_master(pdev);
3344
3345         ixgbevf_reset(adapter);
3346
3347         return PCI_ERS_RESULT_RECOVERED;
3348 }
3349
3350 /**
3351  * ixgbevf_io_resume - called when traffic can start flowing again.
3352  * @pdev: Pointer to PCI device
3353  *
3354  * This callback is called when the error recovery driver tells us that
3355  * its OK to resume normal operation. Implementation resembles the
3356  * second-half of the ixgbevf_resume routine.
3357  */
3358 static void ixgbevf_io_resume(struct pci_dev *pdev)
3359 {
3360         struct net_device *netdev = pci_get_drvdata(pdev);
3361         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3362
3363         if (netif_running(netdev))
3364                 ixgbevf_up(adapter);
3365
3366         netif_device_attach(netdev);
3367 }
3368
3369 /* PCI Error Recovery (ERS) */
3370 static const struct pci_error_handlers ixgbevf_err_handler = {
3371         .error_detected = ixgbevf_io_error_detected,
3372         .slot_reset = ixgbevf_io_slot_reset,
3373         .resume = ixgbevf_io_resume,
3374 };
3375
3376 static struct pci_driver ixgbevf_driver = {
3377         .name     = ixgbevf_driver_name,
3378         .id_table = ixgbevf_pci_tbl,
3379         .probe    = ixgbevf_probe,
3380         .remove   = __devexit_p(ixgbevf_remove),
3381 #ifdef CONFIG_PM
3382         /* Power Management Hooks */
3383         .suspend  = ixgbevf_suspend,
3384         .resume   = ixgbevf_resume,
3385 #endif
3386         .shutdown = ixgbevf_shutdown,
3387         .err_handler = &ixgbevf_err_handler
3388 };
3389
3390 /**
3391  * ixgbevf_init_module - Driver Registration Routine
3392  *
3393  * ixgbevf_init_module is the first routine called when the driver is
3394  * loaded. All it does is register with the PCI subsystem.
3395  **/
3396 static int __init ixgbevf_init_module(void)
3397 {
3398         int ret;
3399         pr_info("%s - version %s\n", ixgbevf_driver_string,
3400                 ixgbevf_driver_version);
3401
3402         pr_info("%s\n", ixgbevf_copyright);
3403
3404         ret = pci_register_driver(&ixgbevf_driver);
3405         return ret;
3406 }
3407
3408 module_init(ixgbevf_init_module);
3409
3410 /**
3411  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3412  *
3413  * ixgbevf_exit_module is called just before the driver is removed
3414  * from memory.
3415  **/
3416 static void __exit ixgbevf_exit_module(void)
3417 {
3418         pci_unregister_driver(&ixgbevf_driver);
3419 }
3420
3421 #ifdef DEBUG
3422 /**
3423  * ixgbevf_get_hw_dev_name - return device name string
3424  * used by hardware layer to print debugging information
3425  **/
3426 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3427 {
3428         struct ixgbevf_adapter *adapter = hw->back;
3429         return adapter->netdev->name;
3430 }
3431
3432 #endif
3433 module_exit(ixgbevf_exit_module);
3434
3435 /* ixgbevf_main.c */