drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.6.0-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69 };
  70
  71 /* ixgbevf_pci_tbl - PCI Device ID Table
  72  *
  73  * Wildcard entries (PCI_ANY_ID) should come last
  74  * Last entry must be all 0s
  75  *
  76  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  77  *   Class, Class Mask, private data (not used) }
  78  */
  79 static struct pci_device_id ixgbevf_pci_tbl[] = {
  80         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
  81         board_82599_vf},
  82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
  83         board_X540_vf},
  84
  85         /* required last entry */
  86         {0, }
  87 };
  88 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  89
  90 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  91 MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver");
  92 MODULE_LICENSE("GPL");
  93 MODULE_VERSION(DRV_VERSION);
  94
  95 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  96 static int debug = -1;
  97 module_param(debug, int, 0);
  98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  99
 100 /* forward decls */
 101 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
 102
 103 static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw,
 104                                            struct ixgbevf_ring *rx_ring,
 105                                            u32 val)
 106 {
 107         /*
 108          * Force memory writes to complete before letting h/w
 109          * know there are new descriptors to fetch.  (Only
 110          * applicable for weak-ordered memory model archs,
 111          * such as IA-64).
 112          */
 113         wmb();
 114         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val);
 115 }
 116
 117 /**
 118  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 119  * @adapter: pointer to adapter struct
 120  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 121  * @queue: queue to map the corresponding interrupt to
 122  * @msix_vector: the vector to map to the corresponding queue
 123  *
 124  */
 125 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 126                              u8 queue, u8 msix_vector)
 127 {
 128         u32 ivar, index;
 129         struct ixgbe_hw *hw = &adapter->hw;
 130         if (direction == -1) {
 131                 /* other causes */
 132                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 133                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 134                 ivar &= ~0xFF;
 135                 ivar |= msix_vector;
 136                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 137         } else {
 138                 /* tx or rx causes */
 139                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 140                 index = ((16 * (queue & 1)) + (8 * direction));
 141                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 142                 ivar &= ~(0xFF << index);
 143                 ivar |= (msix_vector << index);
 144                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 145         }
 146 }
 147
 148 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 149                                                struct ixgbevf_tx_buffer
 150                                                *tx_buffer_info)
 151 {
 152         if (tx_buffer_info->dma) {
 153                 if (tx_buffer_info->mapped_as_page)
 154                         dma_unmap_page(tx_ring->dev,
 155                                        tx_buffer_info->dma,
 156                                        tx_buffer_info->length,
 157                                        DMA_TO_DEVICE);
 158                 else
 159                         dma_unmap_single(tx_ring->dev,
 160                                          tx_buffer_info->dma,
 161                                          tx_buffer_info->length,
 162                                          DMA_TO_DEVICE);
 163                 tx_buffer_info->dma = 0;
 164         }
 165         if (tx_buffer_info->skb) {
 166                 dev_kfree_skb_any(tx_buffer_info->skb);
 167                 tx_buffer_info->skb = NULL;
 168         }
 169         tx_buffer_info->time_stamp = 0;
 170         /* tx_buffer_info must be completely set up in the transmit path */
 171 }
 172
 173 #define IXGBE_MAX_TXD_PWR       14
 174 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 175
 176 /* Tx Descriptors needed, worst case */
 177 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 178 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 179
 180 static void ixgbevf_tx_timeout(struct net_device *netdev);
 181
 182 /**
 183  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 184  * @q_vector: board private structure
 185  * @tx_ring: tx ring to clean
 186  **/
 187 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 188                                  struct ixgbevf_ring *tx_ring)
 189 {
 190         struct ixgbevf_adapter *adapter = q_vector->adapter;
 191         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 192         struct ixgbevf_tx_buffer *tx_buffer_info;
 193         unsigned int i, eop, count = 0;
 194         unsigned int total_bytes = 0, total_packets = 0;
 195
 196         i = tx_ring->next_to_clean;
 197         eop = tx_ring->tx_buffer_info[i].next_to_watch;
 198         eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
 199
 200         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
 201                (count < tx_ring->count)) {
 202                 bool cleaned = false;
 203                 rmb(); /* read buffer_info after eop_desc */
 204                 /* eop could change between read and DD-check */
 205                 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch))
 206                         goto cont_loop;
 207                 for ( ; !cleaned; count++) {
 208                         struct sk_buff *skb;
 209                         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 210                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 211                         cleaned = (i == eop);
 212                         skb = tx_buffer_info->skb;
 213
 214                         if (cleaned && skb) {
 215                                 unsigned int segs, bytecount;
 216
 217                                 /* gso_segs is currently only valid for tcp */
 218                                 segs = skb_shinfo(skb)->gso_segs ?: 1;
 219                                 /* multiply data chunks by size of headers */
 220                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
 221                                             skb->len;
 222                                 total_packets += segs;
 223                                 total_bytes += bytecount;
 224                         }
 225
 226                         ixgbevf_unmap_and_free_tx_resource(tx_ring,
 227                                                            tx_buffer_info);
 228
 229                         tx_desc->wb.status = 0;
 230
 231                         i++;
 232                         if (i == tx_ring->count)
 233                                 i = 0;
 234                 }
 235
 236 cont_loop:
 237                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
 238                 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop);
 239         }
 240
 241         tx_ring->next_to_clean = i;
 242
 243 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 244         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
 245                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 246                 /* Make sure that anybody stopping the queue after this
 247                  * sees the new next_to_clean.
 248                  */
 249                 smp_mb();
 250                 if (__netif_subqueue_stopped(tx_ring->netdev,
 251                                              tx_ring->queue_index) &&
 252                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 253                         netif_wake_subqueue(tx_ring->netdev,
 254                                             tx_ring->queue_index);
 255                         ++adapter->restart_queue;
 256                 }
 257         }
 258
 259         u64_stats_update_begin(&tx_ring->syncp);
 260         tx_ring->total_bytes += total_bytes;
 261         tx_ring->total_packets += total_packets;
 262         u64_stats_update_end(&tx_ring->syncp);
 263
 264         return count < tx_ring->count;
 265 }
 266
 267 /**
 268  * ixgbevf_receive_skb - Send a completed packet up the stack
 269  * @q_vector: structure containing interrupt and ring information
 270  * @skb: packet to send up
 271  * @status: hardware indication of status of receive
 272  * @rx_ring: rx descriptor ring (for a specific queue) to setup
 273  * @rx_desc: rx descriptor
 274  **/
 275 static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector,
 276                                 struct sk_buff *skb, u8 status,
 277                                 struct ixgbevf_ring *ring,
 278                                 union ixgbe_adv_rx_desc *rx_desc)
 279 {
 280         struct ixgbevf_adapter *adapter = q_vector->adapter;
 281         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
 282         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
 283
 284         if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans))
 285                 __vlan_hwaccel_put_tag(skb, tag);
 286
 287         napi_gro_receive(&q_vector->napi, skb);
 288 }
 289
 290 /**
 291  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 292  * @adapter: address of board private structure
 293  * @status_err: hardware indication of status of receive
 294  * @skb: skb currently being received and modified
 295  **/
 296 static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter,
 297                                        struct ixgbevf_ring *ring,
 298                                        u32 status_err, struct sk_buff *skb)
 299 {
 300         skb_checksum_none_assert(skb);
 301
 302         /* Rx csum disabled */
 303         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 304                 return;
 305
 306         /* if IP and error */
 307         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
 308             (status_err & IXGBE_RXDADV_ERR_IPE)) {
 309                 adapter->hw_csum_rx_error++;
 310                 return;
 311         }
 312
 313         if (!(status_err & IXGBE_RXD_STAT_L4CS))
 314                 return;
 315
 316         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
 317                 adapter->hw_csum_rx_error++;
 318                 return;
 319         }
 320
 321         /* It must be a TCP or UDP packet with a valid checksum */
 322         skb->ip_summed = CHECKSUM_UNNECESSARY;
 323         adapter->hw_csum_rx_good++;
 324 }
 325
 326 /**
 327  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 328  * @adapter: address of board private structure
 329  **/
 330 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter,
 331                                      struct ixgbevf_ring *rx_ring,
 332                                      int cleaned_count)
 333 {
 334         struct pci_dev *pdev = adapter->pdev;
 335         union ixgbe_adv_rx_desc *rx_desc;
 336         struct ixgbevf_rx_buffer *bi;
 337         struct sk_buff *skb;
 338         unsigned int i = rx_ring->next_to_use;
 339
 340         bi = &rx_ring->rx_buffer_info[i];
 341
 342         while (cleaned_count--) {
 343                 rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 344                 skb = bi->skb;
 345                 if (!skb) {
 346                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 347                                                         rx_ring->rx_buf_len);
 348                         if (!skb) {
 349                                 adapter->alloc_rx_buff_failed++;
 350                                 goto no_buffers;
 351                         }
 352
 353                         bi->skb = skb;
 354                 }
 355                 if (!bi->dma) {
 356                         bi->dma = dma_map_single(&pdev->dev, skb->data,
 357                                                  rx_ring->rx_buf_len,
 358                                                  DMA_FROM_DEVICE);
 359                 }
 360                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
 361
 362                 i++;
 363                 if (i == rx_ring->count)
 364                         i = 0;
 365                 bi = &rx_ring->rx_buffer_info[i];
 366         }
 367
 368 no_buffers:
 369         if (rx_ring->next_to_use != i) {
 370                 rx_ring->next_to_use = i;
 371
 372                 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i);
 373         }
 374 }
 375
 376 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 377                                              u32 qmask)
 378 {
 379         struct ixgbe_hw *hw = &adapter->hw;
 380
 381         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 382 }
 383
 384 static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 385                                  struct ixgbevf_ring *rx_ring,
 386                                  int budget)
 387 {
 388         struct ixgbevf_adapter *adapter = q_vector->adapter;
 389         struct pci_dev *pdev = adapter->pdev;
 390         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
 391         struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer;
 392         struct sk_buff *skb;
 393         unsigned int i;
 394         u32 len, staterr;
 395         int cleaned_count = 0;
 396         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 397
 398         i = rx_ring->next_to_clean;
 399         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 400         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 401         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 402
 403         while (staterr & IXGBE_RXD_STAT_DD) {
 404                 if (!budget)
 405                         break;
 406                 budget--;
 407
 408                 rmb(); /* read descriptor and rx_buffer_info after status DD */
 409                 len = le16_to_cpu(rx_desc->wb.upper.length);
 410                 skb = rx_buffer_info->skb;
 411                 prefetch(skb->data - NET_IP_ALIGN);
 412                 rx_buffer_info->skb = NULL;
 413
 414                 if (rx_buffer_info->dma) {
 415                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
 416                                          rx_ring->rx_buf_len,
 417                                          DMA_FROM_DEVICE);
 418                         rx_buffer_info->dma = 0;
 419                         skb_put(skb, len);
 420                 }
 421
 422                 i++;
 423                 if (i == rx_ring->count)
 424                         i = 0;
 425
 426                 next_rxd = IXGBEVF_RX_DESC(rx_ring, i);
 427                 prefetch(next_rxd);
 428                 cleaned_count++;
 429
 430                 next_buffer = &rx_ring->rx_buffer_info[i];
 431
 432                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
 433                         skb->next = next_buffer->skb;
 434                         skb->next->prev = skb;
 435                         adapter->non_eop_descs++;
 436                         goto next_desc;
 437                 }
 438
 439                 /* ERR_MASK will only have valid bits if EOP set */
 440                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
 441                         dev_kfree_skb_irq(skb);
 442                         goto next_desc;
 443                 }
 444
 445                 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb);
 446
 447                 /* probably a little skewed due to removing CRC */
 448                 total_rx_bytes += skb->len;
 449                 total_rx_packets++;
 450
 451                 /*
 452                  * Work around issue of some types of VM to VM loop back
 453                  * packets not getting split correctly
 454                  */
 455                 if (staterr & IXGBE_RXD_STAT_LB) {
 456                         u32 header_fixup_len = skb_headlen(skb);
 457                         if (header_fixup_len < 14)
 458                                 skb_push(skb, header_fixup_len);
 459                 }
 460                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 461
 462                 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
 463
 464 next_desc:
 465                 rx_desc->wb.upper.status_error = 0;
 466
 467                 /* return some buffers to hardware, one at a time is too slow */
 468                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 469                         ixgbevf_alloc_rx_buffers(adapter, rx_ring,
 470                                                  cleaned_count);
 471                         cleaned_count = 0;
 472                 }
 473
 474                 /* use prefetched values */
 475                 rx_desc = next_rxd;
 476                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
 477
 478                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 479         }
 480
 481         rx_ring->next_to_clean = i;
 482         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
 483
 484         if (cleaned_count)
 485                 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count);
 486
 487         u64_stats_update_begin(&rx_ring->syncp);
 488         rx_ring->total_packets += total_rx_packets;
 489         rx_ring->total_bytes += total_rx_bytes;
 490         u64_stats_update_end(&rx_ring->syncp);
 491
 492         return !!budget;
 493 }
 494
 495 /**
 496  * ixgbevf_poll - NAPI polling calback
 497  * @napi: napi struct with our devices info in it
 498  * @budget: amount of work driver is allowed to do this pass, in packets
 499  *
 500  * This function will clean more than one or more rings associated with a
 501  * q_vector.
 502  **/
 503 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 504 {
 505         struct ixgbevf_q_vector *q_vector =
 506                 container_of(napi, struct ixgbevf_q_vector, napi);
 507         struct ixgbevf_adapter *adapter = q_vector->adapter;
 508         struct ixgbevf_ring *ring;
 509         int per_ring_budget;
 510         bool clean_complete = true;
 511
 512         ixgbevf_for_each_ring(ring, q_vector->tx)
 513                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 514
 515         /* attempt to distribute budget to each queue fairly, but don't allow
 516          * the budget to go below 1 because we'll exit polling */
 517         if (q_vector->rx.count > 1)
 518                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 519         else
 520                 per_ring_budget = budget;
 521
 522         ixgbevf_for_each_ring(ring, q_vector->rx)
 523                 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring,
 524                                                        per_ring_budget);
 525
 526         /* If all work not completed, return budget and keep polling */
 527         if (!clean_complete)
 528                 return budget;
 529         /* all work done, exit the polling mode */
 530         napi_complete(napi);
 531         if (adapter->rx_itr_setting & 1)
 532                 ixgbevf_set_itr(q_vector);
 533         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
 534                 ixgbevf_irq_enable_queues(adapter,
 535                                           1 << q_vector->v_idx);
 536
 537         return 0;
 538 }
 539
 540
 541 /**
 542  * ixgbevf_configure_msix - Configure MSI-X hardware
 543  * @adapter: board private structure
 544  *
 545  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
 546  * interrupts.
 547  **/
 548 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
 549 {
 550         struct ixgbevf_q_vector *q_vector;
 551         int q_vectors, v_idx;
 552
 553         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 554         adapter->eims_enable_mask = 0;
 555
 556         /*
 557          * Populate the IVAR table and set the ITR values to the
 558          * corresponding register.
 559          */
 560         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
 561                 struct ixgbevf_ring *ring;
 562                 q_vector = adapter->q_vector[v_idx];
 563
 564                 ixgbevf_for_each_ring(ring, q_vector->rx)
 565                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
 566
 567                 ixgbevf_for_each_ring(ring, q_vector->tx)
 568                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
 569
 570                 if (q_vector->tx.ring && !q_vector->rx.ring) {
 571                         /* tx only vector */
 572                         if (adapter->tx_itr_setting == 1)
 573                                 q_vector->itr = IXGBE_10K_ITR;
 574                         else
 575                                 q_vector->itr = adapter->tx_itr_setting;
 576                 } else {
 577                         /* rx or rx/tx vector */
 578                         if (adapter->rx_itr_setting == 1)
 579                                 q_vector->itr = IXGBE_20K_ITR;
 580                         else
 581                                 q_vector->itr = adapter->rx_itr_setting;
 582                 }
 583
 584                 /* add q_vector eims value to global eims_enable_mask */
 585                 adapter->eims_enable_mask |= 1 << v_idx;
 586
 587                 ixgbevf_write_eitr(q_vector);
 588         }
 589
 590         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
 591         /* setup eims_other and add value to global eims_enable_mask */
 592         adapter->eims_other = 1 << v_idx;
 593         adapter->eims_enable_mask |= adapter->eims_other;
 594 }
 595
 596 enum latency_range {
 597         lowest_latency = 0,
 598         low_latency = 1,
 599         bulk_latency = 2,
 600         latency_invalid = 255
 601 };
 602
 603 /**
 604  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
 605  * @q_vector: structure containing interrupt and ring information
 606  * @ring_container: structure containing ring performance data
 607  *
 608  *      Stores a new ITR value based on packets and byte
 609  *      counts during the last interrupt.  The advantage of per interrupt
 610  *      computation is faster updates and more accurate ITR for the current
 611  *      traffic pattern.  Constants in this function were computed
 612  *      based on theoretical maximum wire speed and thresholds were set based
 613  *      on testing data as well as attempting to minimize response time
 614  *      while increasing bulk throughput.
 615  **/
 616 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
 617                                struct ixgbevf_ring_container *ring_container)
 618 {
 619         int bytes = ring_container->total_bytes;
 620         int packets = ring_container->total_packets;
 621         u32 timepassed_us;
 622         u64 bytes_perint;
 623         u8 itr_setting = ring_container->itr;
 624
 625         if (packets == 0)
 626                 return;
 627
 628         /* simple throttlerate management
 629          *    0-20MB/s lowest (100000 ints/s)
 630          *   20-100MB/s low   (20000 ints/s)
 631          *  100-1249MB/s bulk (8000 ints/s)
 632          */
 633         /* what was last interrupt timeslice? */
 634         timepassed_us = q_vector->itr >> 2;
 635         bytes_perint = bytes / timepassed_us; /* bytes/usec */
 636
 637         switch (itr_setting) {
 638         case lowest_latency:
 639                 if (bytes_perint > 10)
 640                         itr_setting = low_latency;
 641                 break;
 642         case low_latency:
 643                 if (bytes_perint > 20)
 644                         itr_setting = bulk_latency;
 645                 else if (bytes_perint <= 10)
 646                         itr_setting = lowest_latency;
 647                 break;
 648         case bulk_latency:
 649                 if (bytes_perint <= 20)
 650                         itr_setting = low_latency;
 651                 break;
 652         }
 653
 654         /* clear work counters since we have the values we need */
 655         ring_container->total_bytes = 0;
 656         ring_container->total_packets = 0;
 657
 658         /* write updated itr to ring container */
 659         ring_container->itr = itr_setting;
 660 }
 661
 662 /**
 663  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 664  * @q_vector: structure containing interrupt and ring information
 665  *
 666  * This function is made to be called by ethtool and by the driver
 667  * when it needs to update VTEITR registers at runtime.  Hardware
 668  * specific quirks/differences are taken care of here.
 669  */
 670 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 671 {
 672         struct ixgbevf_adapter *adapter = q_vector->adapter;
 673         struct ixgbe_hw *hw = &adapter->hw;
 674         int v_idx = q_vector->v_idx;
 675         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 676
 677         /*
 678          * set the WDIS bit to not clear the timer bits and cause an
 679          * immediate assertion of the interrupt
 680          */
 681         itr_reg |= IXGBE_EITR_CNT_WDIS;
 682
 683         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 684 }
 685
 686 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
 687 {
 688         u32 new_itr = q_vector->itr;
 689         u8 current_itr;
 690
 691         ixgbevf_update_itr(q_vector, &q_vector->tx);
 692         ixgbevf_update_itr(q_vector, &q_vector->rx);
 693
 694         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
 695
 696         switch (current_itr) {
 697         /* counts and packets in update_itr are dependent on these numbers */
 698         case lowest_latency:
 699                 new_itr = IXGBE_100K_ITR;
 700                 break;
 701         case low_latency:
 702                 new_itr = IXGBE_20K_ITR;
 703                 break;
 704         case bulk_latency:
 705         default:
 706                 new_itr = IXGBE_8K_ITR;
 707                 break;
 708         }
 709
 710         if (new_itr != q_vector->itr) {
 711                 /* do an exponential smoothing */
 712                 new_itr = (10 * new_itr * q_vector->itr) /
 713                           ((9 * new_itr) + q_vector->itr);
 714
 715                 /* save the algorithm value here */
 716                 q_vector->itr = new_itr;
 717
 718                 ixgbevf_write_eitr(q_vector);
 719         }
 720 }
 721
 722 static irqreturn_t ixgbevf_msix_mbx(int irq, void *data)
 723 {
 724         struct ixgbevf_adapter *adapter = data;
 725         struct ixgbe_hw *hw = &adapter->hw;
 726         u32 msg;
 727         bool got_ack = false;
 728
 729         if (!hw->mbx.ops.check_for_ack(hw))
 730                 got_ack = true;
 731
 732         if (!hw->mbx.ops.check_for_msg(hw)) {
 733                 hw->mbx.ops.read(hw, &msg, 1);
 734
 735                 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG)
 736                         mod_timer(&adapter->watchdog_timer,
 737                                   round_jiffies(jiffies + 1));
 738
 739                 if (msg & IXGBE_VT_MSGTYPE_NACK)
 740                         pr_warn("Last Request of type %2.2x to PF Nacked\n",
 741                                 msg & 0xFF);
 742                 /*
 743                  * Restore the PFSTS bit in case someone is polling for a
 744                  * return message from the PF
 745                  */
 746                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS;
 747         }
 748
 749         /*
 750          * checking for the ack clears the PFACK bit.  Place
 751          * it back in the v2p_mailbox cache so that anyone
 752          * polling for an ack will not miss it
 753          */
 754         if (got_ack)
 755                 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK;
 756
 757         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
 758
 759         return IRQ_HANDLED;
 760 }
 761
 762
 763 /**
 764  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
 765  * @irq: unused
 766  * @data: pointer to our q_vector struct for this interrupt vector
 767  **/
 768 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
 769 {
 770         struct ixgbevf_q_vector *q_vector = data;
 771
 772         /* EIAM disabled interrupts (on this vector) for us */
 773         if (q_vector->rx.ring || q_vector->tx.ring)
 774                 napi_schedule(&q_vector->napi);
 775
 776         return IRQ_HANDLED;
 777 }
 778
 779 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
 780                                      int r_idx)
 781 {
 782         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 783
 784         a->rx_ring[r_idx].next = q_vector->rx.ring;
 785         q_vector->rx.ring = &a->rx_ring[r_idx];
 786         q_vector->rx.count++;
 787 }
 788
 789 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
 790                                      int t_idx)
 791 {
 792         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
 793
 794         a->tx_ring[t_idx].next = q_vector->tx.ring;
 795         q_vector->tx.ring = &a->tx_ring[t_idx];
 796         q_vector->tx.count++;
 797 }
 798
 799 /**
 800  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
 801  * @adapter: board private structure to initialize
 802  *
 803  * This function maps descriptor rings to the queue-specific vectors
 804  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 805  * one vector per ring/queue, but on a constrained vector budget, we
 806  * group the rings as "efficiently" as possible.  You would add new
 807  * mapping configurations in here.
 808  **/
 809 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
 810 {
 811         int q_vectors;
 812         int v_start = 0;
 813         int rxr_idx = 0, txr_idx = 0;
 814         int rxr_remaining = adapter->num_rx_queues;
 815         int txr_remaining = adapter->num_tx_queues;
 816         int i, j;
 817         int rqpv, tqpv;
 818         int err = 0;
 819
 820         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 821
 822         /*
 823          * The ideal configuration...
 824          * We have enough vectors to map one per queue.
 825          */
 826         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
 827                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 828                         map_vector_to_rxq(adapter, v_start, rxr_idx);
 829
 830                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 831                         map_vector_to_txq(adapter, v_start, txr_idx);
 832                 goto out;
 833         }
 834
 835         /*
 836          * If we don't have enough vectors for a 1-to-1
 837          * mapping, we'll have to group them so there are
 838          * multiple queues per vector.
 839          */
 840         /* Re-adjusting *qpv takes care of the remainder. */
 841         for (i = v_start; i < q_vectors; i++) {
 842                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 843                 for (j = 0; j < rqpv; j++) {
 844                         map_vector_to_rxq(adapter, i, rxr_idx);
 845                         rxr_idx++;
 846                         rxr_remaining--;
 847                 }
 848         }
 849         for (i = v_start; i < q_vectors; i++) {
 850                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 851                 for (j = 0; j < tqpv; j++) {
 852                         map_vector_to_txq(adapter, i, txr_idx);
 853                         txr_idx++;
 854                         txr_remaining--;
 855                 }
 856         }
 857
 858 out:
 859         return err;
 860 }
 861
 862 /**
 863  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
 864  * @adapter: board private structure
 865  *
 866  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
 867  * interrupts from the kernel.
 868  **/
 869 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
 870 {
 871         struct net_device *netdev = adapter->netdev;
 872         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 873         int vector, err;
 874         int ri = 0, ti = 0;
 875
 876         for (vector = 0; vector < q_vectors; vector++) {
 877                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
 878                 struct msix_entry *entry = &adapter->msix_entries[vector];
 879
 880                 if (q_vector->tx.ring && q_vector->rx.ring) {
 881                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 882                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
 883                         ti++;
 884                 } else if (q_vector->rx.ring) {
 885                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 886                                  "%s-%s-%d", netdev->name, "rx", ri++);
 887                 } else if (q_vector->tx.ring) {
 888                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 889                                  "%s-%s-%d", netdev->name, "tx", ti++);
 890                 } else {
 891                         /* skip this unused q_vector */
 892                         continue;
 893                 }
 894                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
 895                                   q_vector->name, q_vector);
 896                 if (err) {
 897                         hw_dbg(&adapter->hw,
 898                                "request_irq failed for MSIX interrupt "
 899                                "Error: %d\n", err);
 900                         goto free_queue_irqs;
 901                 }
 902         }
 903
 904         err = request_irq(adapter->msix_entries[vector].vector,
 905                           &ixgbevf_msix_mbx, 0, netdev->name, adapter);
 906         if (err) {
 907                 hw_dbg(&adapter->hw,
 908                        "request_irq for msix_mbx failed: %d\n", err);
 909                 goto free_queue_irqs;
 910         }
 911
 912         return 0;
 913
 914 free_queue_irqs:
 915         while (vector) {
 916                 vector--;
 917                 free_irq(adapter->msix_entries[vector].vector,
 918                          adapter->q_vector[vector]);
 919         }
 920         pci_disable_msix(adapter->pdev);
 921         kfree(adapter->msix_entries);
 922         adapter->msix_entries = NULL;
 923         return err;
 924 }
 925
 926 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
 927 {
 928         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 929
 930         for (i = 0; i < q_vectors; i++) {
 931                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
 932                 q_vector->rx.ring = NULL;
 933                 q_vector->tx.ring = NULL;
 934                 q_vector->rx.count = 0;
 935                 q_vector->tx.count = 0;
 936         }
 937 }
 938
 939 /**
 940  * ixgbevf_request_irq - initialize interrupts
 941  * @adapter: board private structure
 942  *
 943  * Attempts to configure interrupts using the best available
 944  * capabilities of the hardware and kernel.
 945  **/
 946 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
 947 {
 948         int err = 0;
 949
 950         err = ixgbevf_request_msix_irqs(adapter);
 951
 952         if (err)
 953                 hw_dbg(&adapter->hw,
 954                        "request_irq failed, Error %d\n", err);
 955
 956         return err;
 957 }
 958
 959 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
 960 {
 961         int i, q_vectors;
 962
 963         q_vectors = adapter->num_msix_vectors;
 964         i = q_vectors - 1;
 965
 966         free_irq(adapter->msix_entries[i].vector, adapter);
 967         i--;
 968
 969         for (; i >= 0; i--) {
 970                 /* free only the irqs that were actually requested */
 971                 if (!adapter->q_vector[i]->rx.ring &&
 972                     !adapter->q_vector[i]->tx.ring)
 973                         continue;
 974
 975                 free_irq(adapter->msix_entries[i].vector,
 976                          adapter->q_vector[i]);
 977         }
 978
 979         ixgbevf_reset_q_vectors(adapter);
 980 }
 981
 982 /**
 983  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
 984  * @adapter: board private structure
 985  **/
 986 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
 987 {
 988         struct ixgbe_hw *hw = &adapter->hw;
 989         int i;
 990
 991         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
 992         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
 993         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
 994
 995         IXGBE_WRITE_FLUSH(hw);
 996
 997         for (i = 0; i < adapter->num_msix_vectors; i++)
 998                 synchronize_irq(adapter->msix_entries[i].vector);
 999 }
1000
1001 /**
1002  * ixgbevf_irq_enable - Enable default interrupt generation settings
1003  * @adapter: board private structure
1004  **/
1005 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1006 {
1007         struct ixgbe_hw *hw = &adapter->hw;
1008
1009         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1010         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1011         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1012 }
1013
1014 /**
1015  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1016  * @adapter: board private structure
1017  *
1018  * Configure the Tx unit of the MAC after a reset.
1019  **/
1020 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1021 {
1022         u64 tdba;
1023         struct ixgbe_hw *hw = &adapter->hw;
1024         u32 i, j, tdlen, txctrl;
1025
1026         /* Setup the HW Tx Head and Tail descriptor pointers */
1027         for (i = 0; i < adapter->num_tx_queues; i++) {
1028                 struct ixgbevf_ring *ring = &adapter->tx_ring[i];
1029                 j = ring->reg_idx;
1030                 tdba = ring->dma;
1031                 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc);
1032                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j),
1033                                 (tdba & DMA_BIT_MASK(32)));
1034                 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32));
1035                 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen);
1036                 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0);
1037                 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0);
1038                 adapter->tx_ring[i].head = IXGBE_VFTDH(j);
1039                 adapter->tx_ring[i].tail = IXGBE_VFTDT(j);
1040                 /* Disable Tx Head Writeback RO bit, since this hoses
1041                  * bookkeeping if things aren't delivered in order.
1042                  */
1043                 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j));
1044                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
1045                 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl);
1046         }
1047 }
1048
1049 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1050
1051 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1052 {
1053         struct ixgbevf_ring *rx_ring;
1054         struct ixgbe_hw *hw = &adapter->hw;
1055         u32 srrctl;
1056
1057         rx_ring = &adapter->rx_ring[index];
1058
1059         srrctl = IXGBE_SRRCTL_DROP_EN;
1060
1061         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1062
1063         if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE)
1064                 srrctl |= IXGBEVF_RXBUFFER_2048 >>
1065                         IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1066         else
1067                 srrctl |= rx_ring->rx_buf_len >>
1068                         IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1069         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1070 }
1071
1072 /**
1073  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1074  * @adapter: board private structure
1075  *
1076  * Configure the Rx unit of the MAC after a reset.
1077  **/
1078 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1079 {
1080         u64 rdba;
1081         struct ixgbe_hw *hw = &adapter->hw;
1082         struct net_device *netdev = adapter->netdev;
1083         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1084         int i, j;
1085         u32 rdlen;
1086         int rx_buf_len;
1087
1088         /* PSRTYPE must be initialized in 82599 */
1089         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0);
1090         if (netdev->mtu <= ETH_DATA_LEN)
1091                 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
1092         else
1093                 rx_buf_len = ALIGN(max_frame, 1024);
1094
1095         rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc);
1096         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1097          * the Base and Length of the Rx Descriptor Ring */
1098         for (i = 0; i < adapter->num_rx_queues; i++) {
1099                 rdba = adapter->rx_ring[i].dma;
1100                 j = adapter->rx_ring[i].reg_idx;
1101                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j),
1102                                 (rdba & DMA_BIT_MASK(32)));
1103                 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32));
1104                 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen);
1105                 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0);
1106                 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0);
1107                 adapter->rx_ring[i].head = IXGBE_VFRDH(j);
1108                 adapter->rx_ring[i].tail = IXGBE_VFRDT(j);
1109                 adapter->rx_ring[i].rx_buf_len = rx_buf_len;
1110
1111                 ixgbevf_configure_srrctl(adapter, j);
1112         }
1113 }
1114
1115 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
1116 {
1117         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1118         struct ixgbe_hw *hw = &adapter->hw;
1119
1120         /* add VID to filter table */
1121         if (hw->mac.ops.set_vfta)
1122                 hw->mac.ops.set_vfta(hw, vid, 0, true);
1123         set_bit(vid, adapter->active_vlans);
1124
1125         return 0;
1126 }
1127
1128 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
1129 {
1130         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1131         struct ixgbe_hw *hw = &adapter->hw;
1132
1133         /* remove VID from filter table */
1134         if (hw->mac.ops.set_vfta)
1135                 hw->mac.ops.set_vfta(hw, vid, 0, false);
1136         clear_bit(vid, adapter->active_vlans);
1137
1138         return 0;
1139 }
1140
1141 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1142 {
1143         u16 vid;
1144
1145         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1146                 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid);
1147 }
1148
1149 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1150 {
1151         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1152         struct ixgbe_hw *hw = &adapter->hw;
1153         int count = 0;
1154
1155         if ((netdev_uc_count(netdev)) > 10) {
1156                 pr_err("Too many unicast filters - No Space\n");
1157                 return -ENOSPC;
1158         }
1159
1160         if (!netdev_uc_empty(netdev)) {
1161                 struct netdev_hw_addr *ha;
1162                 netdev_for_each_uc_addr(ha, netdev) {
1163                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1164                         udelay(200);
1165                 }
1166         } else {
1167                 /*
1168                  * If the list is empty then send message to PF driver to
1169                  * clear all macvlans on this VF.
1170                  */
1171                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1172         }
1173
1174         return count;
1175 }
1176
1177 /**
1178  * ixgbevf_set_rx_mode - Multicast set
1179  * @netdev: network interface device structure
1180  *
1181  * The set_rx_method entry point is called whenever the multicast address
1182  * list or the network interface flags are updated.  This routine is
1183  * responsible for configuring the hardware for proper multicast mode.
1184  **/
1185 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1186 {
1187         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1188         struct ixgbe_hw *hw = &adapter->hw;
1189
1190         /* reprogram multicast list */
1191         if (hw->mac.ops.update_mc_addr_list)
1192                 hw->mac.ops.update_mc_addr_list(hw, netdev);
1193
1194         ixgbevf_write_uc_addr_list(netdev);
1195 }
1196
1197 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1198 {
1199         int q_idx;
1200         struct ixgbevf_q_vector *q_vector;
1201         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1202
1203         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1204                 q_vector = adapter->q_vector[q_idx];
1205                 napi_enable(&q_vector->napi);
1206         }
1207 }
1208
1209 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1210 {
1211         int q_idx;
1212         struct ixgbevf_q_vector *q_vector;
1213         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1214
1215         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1216                 q_vector = adapter->q_vector[q_idx];
1217                 napi_disable(&q_vector->napi);
1218         }
1219 }
1220
1221 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1222 {
1223         struct net_device *netdev = adapter->netdev;
1224         int i;
1225
1226         ixgbevf_set_rx_mode(netdev);
1227
1228         ixgbevf_restore_vlan(adapter);
1229
1230         ixgbevf_configure_tx(adapter);
1231         ixgbevf_configure_rx(adapter);
1232         for (i = 0; i < adapter->num_rx_queues; i++) {
1233                 struct ixgbevf_ring *ring = &adapter->rx_ring[i];
1234                 ixgbevf_alloc_rx_buffers(adapter, ring,
1235                                          IXGBE_DESC_UNUSED(ring));
1236         }
1237 }
1238
1239 #define IXGBE_MAX_RX_DESC_POLL 10
1240 static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1241                                                 int rxr)
1242 {
1243         struct ixgbe_hw *hw = &adapter->hw;
1244         int j = adapter->rx_ring[rxr].reg_idx;
1245         int k;
1246
1247         for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) {
1248                 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE)
1249                         break;
1250                 else
1251                         msleep(1);
1252         }
1253         if (k >= IXGBE_MAX_RX_DESC_POLL) {
1254                 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d "
1255                        "not set within the polling period\n", rxr);
1256         }
1257
1258         ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr],
1259                                 (adapter->rx_ring[rxr].count - 1));
1260 }
1261
1262 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1263 {
1264         /* Only save pre-reset stats if there are some */
1265         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1266                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1267                         adapter->stats.base_vfgprc;
1268                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1269                         adapter->stats.base_vfgptc;
1270                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1271                         adapter->stats.base_vfgorc;
1272                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1273                         adapter->stats.base_vfgotc;
1274                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1275                         adapter->stats.base_vfmprc;
1276         }
1277 }
1278
1279 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1280 {
1281         struct ixgbe_hw *hw = &adapter->hw;
1282
1283         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1284         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1285         adapter->stats.last_vfgorc |=
1286                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1287         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1288         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1289         adapter->stats.last_vfgotc |=
1290                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1291         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1292
1293         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1294         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1295         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1296         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1297         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1298 }
1299
1300 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1301 {
1302         struct net_device *netdev = adapter->netdev;
1303         struct ixgbe_hw *hw = &adapter->hw;
1304         int i, j = 0;
1305         int num_rx_rings = adapter->num_rx_queues;
1306         u32 txdctl, rxdctl;
1307         u32 msg[2];
1308
1309         for (i = 0; i < adapter->num_tx_queues; i++) {
1310                 j = adapter->tx_ring[i].reg_idx;
1311                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1312                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
1313                 txdctl |= (8 << 16);
1314                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1315         }
1316
1317         for (i = 0; i < adapter->num_tx_queues; i++) {
1318                 j = adapter->tx_ring[i].reg_idx;
1319                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1320                 txdctl |= IXGBE_TXDCTL_ENABLE;
1321                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl);
1322         }
1323
1324         for (i = 0; i < num_rx_rings; i++) {
1325                 j = adapter->rx_ring[i].reg_idx;
1326                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j));
1327                 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1328                 if (hw->mac.type == ixgbe_mac_X540_vf) {
1329                         rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
1330                         rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) |
1331                                    IXGBE_RXDCTL_RLPML_EN);
1332                 }
1333                 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl);
1334                 ixgbevf_rx_desc_queue_enable(adapter, i);
1335         }
1336
1337         ixgbevf_configure_msix(adapter);
1338
1339         if (hw->mac.ops.set_rar) {
1340                 if (is_valid_ether_addr(hw->mac.addr))
1341                         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1342                 else
1343                         hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1344         }
1345
1346         msg[0] = IXGBE_VF_SET_LPE;
1347         msg[1] = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1348         hw->mbx.ops.write_posted(hw, msg, 2);
1349
1350         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1351         ixgbevf_napi_enable_all(adapter);
1352
1353         /* enable transmits */
1354         netif_tx_start_all_queues(netdev);
1355
1356         ixgbevf_save_reset_stats(adapter);
1357         ixgbevf_init_last_counter_stats(adapter);
1358
1359         mod_timer(&adapter->watchdog_timer, jiffies);
1360 }
1361
1362 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1363 {
1364         struct ixgbe_hw *hw = &adapter->hw;
1365
1366         ixgbevf_configure(adapter);
1367
1368         ixgbevf_up_complete(adapter);
1369
1370         /* clear any pending interrupts, may auto mask */
1371         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1372
1373         ixgbevf_irq_enable(adapter);
1374 }
1375
1376 /**
1377  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1378  * @adapter: board private structure
1379  * @rx_ring: ring to free buffers from
1380  **/
1381 static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter,
1382                                   struct ixgbevf_ring *rx_ring)
1383 {
1384         struct pci_dev *pdev = adapter->pdev;
1385         unsigned long size;
1386         unsigned int i;
1387
1388         if (!rx_ring->rx_buffer_info)
1389                 return;
1390
1391         /* Free all the Rx ring sk_buffs */
1392         for (i = 0; i < rx_ring->count; i++) {
1393                 struct ixgbevf_rx_buffer *rx_buffer_info;
1394
1395                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1396                 if (rx_buffer_info->dma) {
1397                         dma_unmap_single(&pdev->dev, rx_buffer_info->dma,
1398                                          rx_ring->rx_buf_len,
1399                                          DMA_FROM_DEVICE);
1400                         rx_buffer_info->dma = 0;
1401                 }
1402                 if (rx_buffer_info->skb) {
1403                         struct sk_buff *skb = rx_buffer_info->skb;
1404                         rx_buffer_info->skb = NULL;
1405                         do {
1406                                 struct sk_buff *this = skb;
1407                                 skb = skb->prev;
1408                                 dev_kfree_skb(this);
1409                         } while (skb);
1410                 }
1411         }
1412
1413         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1414         memset(rx_ring->rx_buffer_info, 0, size);
1415
1416         /* Zero out the descriptor ring */
1417         memset(rx_ring->desc, 0, rx_ring->size);
1418
1419         rx_ring->next_to_clean = 0;
1420         rx_ring->next_to_use = 0;
1421
1422         if (rx_ring->head)
1423                 writel(0, adapter->hw.hw_addr + rx_ring->head);
1424         if (rx_ring->tail)
1425                 writel(0, adapter->hw.hw_addr + rx_ring->tail);
1426 }
1427
1428 /**
1429  * ixgbevf_clean_tx_ring - Free Tx Buffers
1430  * @adapter: board private structure
1431  * @tx_ring: ring to be cleaned
1432  **/
1433 static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter,
1434                                   struct ixgbevf_ring *tx_ring)
1435 {
1436         struct ixgbevf_tx_buffer *tx_buffer_info;
1437         unsigned long size;
1438         unsigned int i;
1439
1440         if (!tx_ring->tx_buffer_info)
1441                 return;
1442
1443         /* Free all the Tx ring sk_buffs */
1444
1445         for (i = 0; i < tx_ring->count; i++) {
1446                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
1447                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
1448         }
1449
1450         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
1451         memset(tx_ring->tx_buffer_info, 0, size);
1452
1453         memset(tx_ring->desc, 0, tx_ring->size);
1454
1455         tx_ring->next_to_use = 0;
1456         tx_ring->next_to_clean = 0;
1457
1458         if (tx_ring->head)
1459                 writel(0, adapter->hw.hw_addr + tx_ring->head);
1460         if (tx_ring->tail)
1461                 writel(0, adapter->hw.hw_addr + tx_ring->tail);
1462 }
1463
1464 /**
1465  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
1466  * @adapter: board private structure
1467  **/
1468 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
1469 {
1470         int i;
1471
1472         for (i = 0; i < adapter->num_rx_queues; i++)
1473                 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]);
1474 }
1475
1476 /**
1477  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
1478  * @adapter: board private structure
1479  **/
1480 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
1481 {
1482         int i;
1483
1484         for (i = 0; i < adapter->num_tx_queues; i++)
1485                 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]);
1486 }
1487
1488 void ixgbevf_down(struct ixgbevf_adapter *adapter)
1489 {
1490         struct net_device *netdev = adapter->netdev;
1491         struct ixgbe_hw *hw = &adapter->hw;
1492         u32 txdctl;
1493         int i, j;
1494
1495         /* signal that we are down to the interrupt handler */
1496         set_bit(__IXGBEVF_DOWN, &adapter->state);
1497         /* disable receives */
1498
1499         netif_tx_disable(netdev);
1500
1501         msleep(10);
1502
1503         netif_tx_stop_all_queues(netdev);
1504
1505         ixgbevf_irq_disable(adapter);
1506
1507         ixgbevf_napi_disable_all(adapter);
1508
1509         del_timer_sync(&adapter->watchdog_timer);
1510         /* can't call flush scheduled work here because it can deadlock
1511          * if linkwatch_event tries to acquire the rtnl_lock which we are
1512          * holding */
1513         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
1514                 msleep(1);
1515
1516         /* disable transmits in the hardware now that interrupts are off */
1517         for (i = 0; i < adapter->num_tx_queues; i++) {
1518                 j = adapter->tx_ring[i].reg_idx;
1519                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j));
1520                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j),
1521                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
1522         }
1523
1524         netif_carrier_off(netdev);
1525
1526         if (!pci_channel_offline(adapter->pdev))
1527                 ixgbevf_reset(adapter);
1528
1529         ixgbevf_clean_all_tx_rings(adapter);
1530         ixgbevf_clean_all_rx_rings(adapter);
1531 }
1532
1533 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
1534 {
1535         struct ixgbe_hw *hw = &adapter->hw;
1536
1537         WARN_ON(in_interrupt());
1538
1539         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
1540                 msleep(1);
1541
1542         /*
1543          * Check if PF is up before re-init.  If not then skip until
1544          * later when the PF is up and ready to service requests from
1545          * the VF via mailbox.  If the VF is up and running then the
1546          * watchdog task will continue to schedule reset tasks until
1547          * the PF is up and running.
1548          */
1549         if (!hw->mac.ops.reset_hw(hw)) {
1550                 ixgbevf_down(adapter);
1551                 ixgbevf_up(adapter);
1552         }
1553
1554         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
1555 }
1556
1557 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
1558 {
1559         struct ixgbe_hw *hw = &adapter->hw;
1560         struct net_device *netdev = adapter->netdev;
1561
1562         if (hw->mac.ops.reset_hw(hw))
1563                 hw_dbg(hw, "PF still resetting\n");
1564         else
1565                 hw->mac.ops.init_hw(hw);
1566
1567         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1568                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
1569                        netdev->addr_len);
1570                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1571                        netdev->addr_len);
1572         }
1573 }
1574
1575 static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
1576                                          int vectors)
1577 {
1578         int err, vector_threshold;
1579
1580         /* We'll want at least 2 (vector_threshold):
1581          * 1) TxQ[0] + RxQ[0] handler
1582          * 2) Other (Link Status Change, etc.)
1583          */
1584         vector_threshold = MIN_MSIX_COUNT;
1585
1586         /* The more we get, the more we will assign to Tx/Rx Cleanup
1587          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1588          * Right now, we simply care about how many we'll get; we'll
1589          * set them up later while requesting irq's.
1590          */
1591         while (vectors >= vector_threshold) {
1592                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1593                                       vectors);
1594                 if (!err) /* Success in acquiring all requested vectors. */
1595                         break;
1596                 else if (err < 0)
1597                         vectors = 0; /* Nasty failure, quit now */
1598                 else /* err == number of vectors we should try again with */
1599                         vectors = err;
1600         }
1601
1602         if (vectors < vector_threshold) {
1603                 /* Can't allocate enough MSI-X interrupts?  Oh well.
1604                  * This just means we'll go with either a single MSI
1605                  * vector or fall back to legacy interrupts.
1606                  */
1607                 hw_dbg(&adapter->hw,
1608                        "Unable to allocate MSI-X interrupts\n");
1609                 kfree(adapter->msix_entries);
1610                 adapter->msix_entries = NULL;
1611         } else {
1612                 /*
1613                  * Adjust for only the vectors we'll use, which is minimum
1614                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
1615                  * vectors we were allocated.
1616                  */
1617                 adapter->num_msix_vectors = vectors;
1618         }
1619 }
1620
1621 /**
1622  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
1623  * @adapter: board private structure to initialize
1624  *
1625  * This is the top level queue allocation routine.  The order here is very
1626  * important, starting with the "most" number of features turned on at once,
1627  * and ending with the smallest set of features.  This way large combinations
1628  * can be allocated if they're turned on, and smaller combinations are the
1629  * fallthrough conditions.
1630  *
1631  **/
1632 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
1633 {
1634         /* Start with base case */
1635         adapter->num_rx_queues = 1;
1636         adapter->num_tx_queues = 1;
1637 }
1638
1639 /**
1640  * ixgbevf_alloc_queues - Allocate memory for all rings
1641  * @adapter: board private structure to initialize
1642  *
1643  * We allocate one ring per queue at run-time since we don't know the
1644  * number of queues at compile-time.  The polling_netdev array is
1645  * intended for Multiqueue, but should work fine with a single queue.
1646  **/
1647 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
1648 {
1649         int i;
1650
1651         adapter->tx_ring = kcalloc(adapter->num_tx_queues,
1652                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1653         if (!adapter->tx_ring)
1654                 goto err_tx_ring_allocation;
1655
1656         adapter->rx_ring = kcalloc(adapter->num_rx_queues,
1657                                    sizeof(struct ixgbevf_ring), GFP_KERNEL);
1658         if (!adapter->rx_ring)
1659                 goto err_rx_ring_allocation;
1660
1661         for (i = 0; i < adapter->num_tx_queues; i++) {
1662                 adapter->tx_ring[i].count = adapter->tx_ring_count;
1663                 adapter->tx_ring[i].queue_index = i;
1664                 adapter->tx_ring[i].reg_idx = i;
1665                 adapter->tx_ring[i].dev = &adapter->pdev->dev;
1666                 adapter->tx_ring[i].netdev = adapter->netdev;
1667         }
1668
1669         for (i = 0; i < adapter->num_rx_queues; i++) {
1670                 adapter->rx_ring[i].count = adapter->rx_ring_count;
1671                 adapter->rx_ring[i].queue_index = i;
1672                 adapter->rx_ring[i].reg_idx = i;
1673                 adapter->rx_ring[i].dev = &adapter->pdev->dev;
1674                 adapter->rx_ring[i].netdev = adapter->netdev;
1675         }
1676
1677         return 0;
1678
1679 err_rx_ring_allocation:
1680         kfree(adapter->tx_ring);
1681 err_tx_ring_allocation:
1682         return -ENOMEM;
1683 }
1684
1685 /**
1686  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
1687  * @adapter: board private structure to initialize
1688  *
1689  * Attempt to configure the interrupts using the best available
1690  * capabilities of the hardware and the kernel.
1691  **/
1692 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
1693 {
1694         int err = 0;
1695         int vector, v_budget;
1696
1697         /*
1698          * It's easy to be greedy for MSI-X vectors, but it really
1699          * doesn't do us much good if we have a lot more vectors
1700          * than CPU's.  So let's be conservative and only ask for
1701          * (roughly) the same number of vectors as there are CPU's.
1702          * The default is to use pairs of vectors.
1703          */
1704         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
1705         v_budget = min_t(int, v_budget, num_online_cpus());
1706         v_budget += NON_Q_VECTORS;
1707
1708         /* A failure in MSI-X entry allocation isn't fatal, but it does
1709          * mean we disable MSI-X capabilities of the adapter. */
1710         adapter->msix_entries = kcalloc(v_budget,
1711                                         sizeof(struct msix_entry), GFP_KERNEL);
1712         if (!adapter->msix_entries) {
1713                 err = -ENOMEM;
1714                 goto out;
1715         }
1716
1717         for (vector = 0; vector < v_budget; vector++)
1718                 adapter->msix_entries[vector].entry = vector;
1719
1720         ixgbevf_acquire_msix_vectors(adapter, v_budget);
1721
1722 out:
1723         return err;
1724 }
1725
1726 /**
1727  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
1728  * @adapter: board private structure to initialize
1729  *
1730  * We allocate one q_vector per queue interrupt.  If allocation fails we
1731  * return -ENOMEM.
1732  **/
1733 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
1734 {
1735         int q_idx, num_q_vectors;
1736         struct ixgbevf_q_vector *q_vector;
1737
1738         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1739
1740         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1741                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
1742                 if (!q_vector)
1743                         goto err_out;
1744                 q_vector->adapter = adapter;
1745                 q_vector->v_idx = q_idx;
1746                 netif_napi_add(adapter->netdev, &q_vector->napi,
1747                                ixgbevf_poll, 64);
1748                 adapter->q_vector[q_idx] = q_vector;
1749         }
1750
1751         return 0;
1752
1753 err_out:
1754         while (q_idx) {
1755                 q_idx--;
1756                 q_vector = adapter->q_vector[q_idx];
1757                 netif_napi_del(&q_vector->napi);
1758                 kfree(q_vector);
1759                 adapter->q_vector[q_idx] = NULL;
1760         }
1761         return -ENOMEM;
1762 }
1763
1764 /**
1765  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
1766  * @adapter: board private structure to initialize
1767  *
1768  * This function frees the memory allocated to the q_vectors.  In addition if
1769  * NAPI is enabled it will delete any references to the NAPI struct prior
1770  * to freeing the q_vector.
1771  **/
1772 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
1773 {
1774         int q_idx, num_q_vectors;
1775         int napi_vectors;
1776
1777         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1778         napi_vectors = adapter->num_rx_queues;
1779
1780         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1781                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
1782
1783                 adapter->q_vector[q_idx] = NULL;
1784                 if (q_idx < napi_vectors)
1785                         netif_napi_del(&q_vector->napi);
1786                 kfree(q_vector);
1787         }
1788 }
1789
1790 /**
1791  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
1792  * @adapter: board private structure
1793  *
1794  **/
1795 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
1796 {
1797         pci_disable_msix(adapter->pdev);
1798         kfree(adapter->msix_entries);
1799         adapter->msix_entries = NULL;
1800 }
1801
1802 /**
1803  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
1804  * @adapter: board private structure to initialize
1805  *
1806  **/
1807 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
1808 {
1809         int err;
1810
1811         /* Number of supported queues */
1812         ixgbevf_set_num_queues(adapter);
1813
1814         err = ixgbevf_set_interrupt_capability(adapter);
1815         if (err) {
1816                 hw_dbg(&adapter->hw,
1817                        "Unable to setup interrupt capabilities\n");
1818                 goto err_set_interrupt;
1819         }
1820
1821         err = ixgbevf_alloc_q_vectors(adapter);
1822         if (err) {
1823                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
1824                        "vectors\n");
1825                 goto err_alloc_q_vectors;
1826         }
1827
1828         err = ixgbevf_alloc_queues(adapter);
1829         if (err) {
1830                 pr_err("Unable to allocate memory for queues\n");
1831                 goto err_alloc_queues;
1832         }
1833
1834         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
1835                "Tx Queue count = %u\n",
1836                (adapter->num_rx_queues > 1) ? "Enabled" :
1837                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
1838
1839         set_bit(__IXGBEVF_DOWN, &adapter->state);
1840
1841         return 0;
1842 err_alloc_queues:
1843         ixgbevf_free_q_vectors(adapter);
1844 err_alloc_q_vectors:
1845         ixgbevf_reset_interrupt_capability(adapter);
1846 err_set_interrupt:
1847         return err;
1848 }
1849
1850 /**
1851  * ixgbevf_sw_init - Initialize general software structures
1852  * (struct ixgbevf_adapter)
1853  * @adapter: board private structure to initialize
1854  *
1855  * ixgbevf_sw_init initializes the Adapter private data structure.
1856  * Fields are initialized based on PCI device information and
1857  * OS network device settings (MTU size).
1858  **/
1859 static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
1860 {
1861         struct ixgbe_hw *hw = &adapter->hw;
1862         struct pci_dev *pdev = adapter->pdev;
1863         int err;
1864
1865         /* PCI config space info */
1866
1867         hw->vendor_id = pdev->vendor;
1868         hw->device_id = pdev->device;
1869         hw->revision_id = pdev->revision;
1870         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1871         hw->subsystem_device_id = pdev->subsystem_device;
1872
1873         hw->mbx.ops.init_params(hw);
1874         hw->mac.max_tx_queues = MAX_TX_QUEUES;
1875         hw->mac.max_rx_queues = MAX_RX_QUEUES;
1876         err = hw->mac.ops.reset_hw(hw);
1877         if (err) {
1878                 dev_info(&pdev->dev,
1879                          "PF still in reset state, assigning new address\n");
1880                 eth_hw_addr_random(adapter->netdev);
1881                 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr,
1882                         adapter->netdev->addr_len);
1883         } else {
1884                 err = hw->mac.ops.init_hw(hw);
1885                 if (err) {
1886                         pr_err("init_shared_code failed: %d\n", err);
1887                         goto out;
1888                 }
1889                 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr,
1890                         adapter->netdev->addr_len);
1891         }
1892
1893         /* Enable dynamic interrupt throttling rates */
1894         adapter->rx_itr_setting = 1;
1895         adapter->tx_itr_setting = 1;
1896
1897         /* set default ring sizes */
1898         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
1899         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
1900
1901         set_bit(__IXGBEVF_DOWN, &adapter->state);
1902         return 0;
1903
1904 out:
1905         return err;
1906 }
1907
1908 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
1909         {                                                       \
1910                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
1911                 if (current_counter < last_counter)             \
1912                         counter += 0x100000000LL;               \
1913                 last_counter = current_counter;                 \
1914                 counter &= 0xFFFFFFFF00000000LL;                \
1915                 counter |= current_counter;                     \
1916         }
1917
1918 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
1919         {                                                                \
1920                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
1921                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
1922                 u64 current_counter = (current_counter_msb << 32) |      \
1923                         current_counter_lsb;                             \
1924                 if (current_counter < last_counter)                      \
1925                         counter += 0x1000000000LL;                       \
1926                 last_counter = current_counter;                          \
1927                 counter &= 0xFFFFFFF000000000LL;                         \
1928                 counter |= current_counter;                              \
1929         }
1930 /**
1931  * ixgbevf_update_stats - Update the board statistics counters.
1932  * @adapter: board private structure
1933  **/
1934 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
1935 {
1936         struct ixgbe_hw *hw = &adapter->hw;
1937
1938         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
1939                                 adapter->stats.vfgprc);
1940         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
1941                                 adapter->stats.vfgptc);
1942         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
1943                                 adapter->stats.last_vfgorc,
1944                                 adapter->stats.vfgorc);
1945         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
1946                                 adapter->stats.last_vfgotc,
1947                                 adapter->stats.vfgotc);
1948         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
1949                                 adapter->stats.vfmprc);
1950 }
1951
1952 /**
1953  * ixgbevf_watchdog - Timer Call-back
1954  * @data: pointer to adapter cast into an unsigned long
1955  **/
1956 static void ixgbevf_watchdog(unsigned long data)
1957 {
1958         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
1959         struct ixgbe_hw *hw = &adapter->hw;
1960         u32 eics = 0;
1961         int i;
1962
1963         /*
1964          * Do the watchdog outside of interrupt context due to the lovely
1965          * delays that some of the newer hardware requires
1966          */
1967
1968         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
1969                 goto watchdog_short_circuit;
1970
1971         /* get one bit for every active tx/rx interrupt vector */
1972         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
1973                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
1974                 if (qv->rx.ring || qv->tx.ring)
1975                         eics |= 1 << i;
1976         }
1977
1978         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
1979
1980 watchdog_short_circuit:
1981         schedule_work(&adapter->watchdog_task);
1982 }
1983
1984 /**
1985  * ixgbevf_tx_timeout - Respond to a Tx Hang
1986  * @netdev: network interface device structure
1987  **/
1988 static void ixgbevf_tx_timeout(struct net_device *netdev)
1989 {
1990         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1991
1992         /* Do the reset outside of interrupt context */
1993         schedule_work(&adapter->reset_task);
1994 }
1995
1996 static void ixgbevf_reset_task(struct work_struct *work)
1997 {
1998         struct ixgbevf_adapter *adapter;
1999         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2000
2001         /* If we're already down or resetting, just bail */
2002         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2003             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2004                 return;
2005
2006         adapter->tx_timeout_count++;
2007
2008         ixgbevf_reinit_locked(adapter);
2009 }
2010
2011 /**
2012  * ixgbevf_watchdog_task - worker thread to bring link up
2013  * @work: pointer to work_struct containing our data
2014  **/
2015 static void ixgbevf_watchdog_task(struct work_struct *work)
2016 {
2017         struct ixgbevf_adapter *adapter = container_of(work,
2018                                                        struct ixgbevf_adapter,
2019                                                        watchdog_task);
2020         struct net_device *netdev = adapter->netdev;
2021         struct ixgbe_hw *hw = &adapter->hw;
2022         u32 link_speed = adapter->link_speed;
2023         bool link_up = adapter->link_up;
2024
2025         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2026
2027         /*
2028          * Always check the link on the watchdog because we have
2029          * no LSC interrupt
2030          */
2031         if (hw->mac.ops.check_link) {
2032                 if ((hw->mac.ops.check_link(hw, &link_speed,
2033                                             &link_up, false)) != 0) {
2034                         adapter->link_up = link_up;
2035                         adapter->link_speed = link_speed;
2036                         netif_carrier_off(netdev);
2037                         netif_tx_stop_all_queues(netdev);
2038                         schedule_work(&adapter->reset_task);
2039                         goto pf_has_reset;
2040                 }
2041         } else {
2042                 /* always assume link is up, if no check link
2043                  * function */
2044                 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
2045                 link_up = true;
2046         }
2047         adapter->link_up = link_up;
2048         adapter->link_speed = link_speed;
2049
2050         if (link_up) {
2051                 if (!netif_carrier_ok(netdev)) {
2052                         hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n",
2053                                (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ?
2054                                10 : 1);
2055                         netif_carrier_on(netdev);
2056                         netif_tx_wake_all_queues(netdev);
2057                 }
2058         } else {
2059                 adapter->link_up = false;
2060                 adapter->link_speed = 0;
2061                 if (netif_carrier_ok(netdev)) {
2062                         hw_dbg(&adapter->hw, "NIC Link is Down\n");
2063                         netif_carrier_off(netdev);
2064                         netif_tx_stop_all_queues(netdev);
2065                 }
2066         }
2067
2068         ixgbevf_update_stats(adapter);
2069
2070 pf_has_reset:
2071         /* Reset the timer */
2072         if (!test_bit(__IXGBEVF_DOWN, &adapter->state))
2073                 mod_timer(&adapter->watchdog_timer,
2074                           round_jiffies(jiffies + (2 * HZ)));
2075
2076         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2077 }
2078
2079 /**
2080  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2081  * @adapter: board private structure
2082  * @tx_ring: Tx descriptor ring for a specific queue
2083  *
2084  * Free all transmit software resources
2085  **/
2086 void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter,
2087                                struct ixgbevf_ring *tx_ring)
2088 {
2089         struct pci_dev *pdev = adapter->pdev;
2090
2091         ixgbevf_clean_tx_ring(adapter, tx_ring);
2092
2093         vfree(tx_ring->tx_buffer_info);
2094         tx_ring->tx_buffer_info = NULL;
2095
2096         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2097                           tx_ring->dma);
2098
2099         tx_ring->desc = NULL;
2100 }
2101
2102 /**
2103  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2104  * @adapter: board private structure
2105  *
2106  * Free all transmit software resources
2107  **/
2108 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2109 {
2110         int i;
2111
2112         for (i = 0; i < adapter->num_tx_queues; i++)
2113                 if (adapter->tx_ring[i].desc)
2114                         ixgbevf_free_tx_resources(adapter,
2115                                                   &adapter->tx_ring[i]);
2116
2117 }
2118
2119 /**
2120  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2121  * @adapter: board private structure
2122  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2123  *
2124  * Return 0 on success, negative on failure
2125  **/
2126 int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter,
2127                                struct ixgbevf_ring *tx_ring)
2128 {
2129         struct pci_dev *pdev = adapter->pdev;
2130         int size;
2131
2132         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2133         tx_ring->tx_buffer_info = vzalloc(size);
2134         if (!tx_ring->tx_buffer_info)
2135                 goto err;
2136
2137         /* round up to nearest 4K */
2138         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2139         tx_ring->size = ALIGN(tx_ring->size, 4096);
2140
2141         tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
2142                                            &tx_ring->dma, GFP_KERNEL);
2143         if (!tx_ring->desc)
2144                 goto err;
2145
2146         tx_ring->next_to_use = 0;
2147         tx_ring->next_to_clean = 0;
2148         return 0;
2149
2150 err:
2151         vfree(tx_ring->tx_buffer_info);
2152         tx_ring->tx_buffer_info = NULL;
2153         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2154                "descriptor ring\n");
2155         return -ENOMEM;
2156 }
2157
2158 /**
2159  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2160  * @adapter: board private structure
2161  *
2162  * If this function returns with an error, then it's possible one or
2163  * more of the rings is populated (while the rest are not).  It is the
2164  * callers duty to clean those orphaned rings.
2165  *
2166  * Return 0 on success, negative on failure
2167  **/
2168 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2169 {
2170         int i, err = 0;
2171
2172         for (i = 0; i < adapter->num_tx_queues; i++) {
2173                 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]);
2174                 if (!err)
2175                         continue;
2176                 hw_dbg(&adapter->hw,
2177                        "Allocation for Tx Queue %u failed\n", i);
2178                 break;
2179         }
2180
2181         return err;
2182 }
2183
2184 /**
2185  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2186  * @adapter: board private structure
2187  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2188  *
2189  * Returns 0 on success, negative on failure
2190  **/
2191 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
2192                                struct ixgbevf_ring *rx_ring)
2193 {
2194         struct pci_dev *pdev = adapter->pdev;
2195         int size;
2196
2197         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2198         rx_ring->rx_buffer_info = vzalloc(size);
2199         if (!rx_ring->rx_buffer_info)
2200                 goto alloc_failed;
2201
2202         /* Round up to nearest 4K */
2203         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2204         rx_ring->size = ALIGN(rx_ring->size, 4096);
2205
2206         rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
2207                                            &rx_ring->dma, GFP_KERNEL);
2208
2209         if (!rx_ring->desc) {
2210                 hw_dbg(&adapter->hw,
2211                        "Unable to allocate memory for "
2212                        "the receive descriptor ring\n");
2213                 vfree(rx_ring->rx_buffer_info);
2214                 rx_ring->rx_buffer_info = NULL;
2215                 goto alloc_failed;
2216         }
2217
2218         rx_ring->next_to_clean = 0;
2219         rx_ring->next_to_use = 0;
2220
2221         return 0;
2222 alloc_failed:
2223         return -ENOMEM;
2224 }
2225
2226 /**
2227  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2228  * @adapter: board private structure
2229  *
2230  * If this function returns with an error, then it's possible one or
2231  * more of the rings is populated (while the rest are not).  It is the
2232  * callers duty to clean those orphaned rings.
2233  *
2234  * Return 0 on success, negative on failure
2235  **/
2236 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2237 {
2238         int i, err = 0;
2239
2240         for (i = 0; i < adapter->num_rx_queues; i++) {
2241                 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]);
2242                 if (!err)
2243                         continue;
2244                 hw_dbg(&adapter->hw,
2245                        "Allocation for Rx Queue %u failed\n", i);
2246                 break;
2247         }
2248         return err;
2249 }
2250
2251 /**
2252  * ixgbevf_free_rx_resources - Free Rx Resources
2253  * @adapter: board private structure
2254  * @rx_ring: ring to clean the resources from
2255  *
2256  * Free all receive software resources
2257  **/
2258 void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter,
2259                                struct ixgbevf_ring *rx_ring)
2260 {
2261         struct pci_dev *pdev = adapter->pdev;
2262
2263         ixgbevf_clean_rx_ring(adapter, rx_ring);
2264
2265         vfree(rx_ring->rx_buffer_info);
2266         rx_ring->rx_buffer_info = NULL;
2267
2268         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2269                           rx_ring->dma);
2270
2271         rx_ring->desc = NULL;
2272 }
2273
2274 /**
2275  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2276  * @adapter: board private structure
2277  *
2278  * Free all receive software resources
2279  **/
2280 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2281 {
2282         int i;
2283
2284         for (i = 0; i < adapter->num_rx_queues; i++)
2285                 if (adapter->rx_ring[i].desc)
2286                         ixgbevf_free_rx_resources(adapter,
2287                                                   &adapter->rx_ring[i]);
2288 }
2289
2290 /**
2291  * ixgbevf_open - Called when a network interface is made active
2292  * @netdev: network interface device structure
2293  *
2294  * Returns 0 on success, negative value on failure
2295  *
2296  * The open entry point is called when a network interface is made
2297  * active by the system (IFF_UP).  At this point all resources needed
2298  * for transmit and receive operations are allocated, the interrupt
2299  * handler is registered with the OS, the watchdog timer is started,
2300  * and the stack is notified that the interface is ready.
2301  **/
2302 static int ixgbevf_open(struct net_device *netdev)
2303 {
2304         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2305         struct ixgbe_hw *hw = &adapter->hw;
2306         int err;
2307
2308         /* disallow open during test */
2309         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2310                 return -EBUSY;
2311
2312         if (hw->adapter_stopped) {
2313                 ixgbevf_reset(adapter);
2314                 /* if adapter is still stopped then PF isn't up and
2315                  * the vf can't start. */
2316                 if (hw->adapter_stopped) {
2317                         err = IXGBE_ERR_MBX;
2318                         pr_err("Unable to start - perhaps the PF Driver isn't "
2319                                "up yet\n");
2320                         goto err_setup_reset;
2321                 }
2322         }
2323
2324         /* allocate transmit descriptors */
2325         err = ixgbevf_setup_all_tx_resources(adapter);
2326         if (err)
2327                 goto err_setup_tx;
2328
2329         /* allocate receive descriptors */
2330         err = ixgbevf_setup_all_rx_resources(adapter);
2331         if (err)
2332                 goto err_setup_rx;
2333
2334         ixgbevf_configure(adapter);
2335
2336         /*
2337          * Map the Tx/Rx rings to the vectors we were allotted.
2338          * if request_irq will be called in this function map_rings
2339          * must be called *before* up_complete
2340          */
2341         ixgbevf_map_rings_to_vectors(adapter);
2342
2343         ixgbevf_up_complete(adapter);
2344
2345         /* clear any pending interrupts, may auto mask */
2346         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2347         err = ixgbevf_request_irq(adapter);
2348         if (err)
2349                 goto err_req_irq;
2350
2351         ixgbevf_irq_enable(adapter);
2352
2353         return 0;
2354
2355 err_req_irq:
2356         ixgbevf_down(adapter);
2357         ixgbevf_free_irq(adapter);
2358 err_setup_rx:
2359         ixgbevf_free_all_rx_resources(adapter);
2360 err_setup_tx:
2361         ixgbevf_free_all_tx_resources(adapter);
2362         ixgbevf_reset(adapter);
2363
2364 err_setup_reset:
2365
2366         return err;
2367 }
2368
2369 /**
2370  * ixgbevf_close - Disables a network interface
2371  * @netdev: network interface device structure
2372  *
2373  * Returns 0, this is not allowed to fail
2374  *
2375  * The close entry point is called when an interface is de-activated
2376  * by the OS.  The hardware is still under the drivers control, but
2377  * needs to be disabled.  A global MAC reset is issued to stop the
2378  * hardware, and all transmit and receive resources are freed.
2379  **/
2380 static int ixgbevf_close(struct net_device *netdev)
2381 {
2382         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2383
2384         ixgbevf_down(adapter);
2385         ixgbevf_free_irq(adapter);
2386
2387         ixgbevf_free_all_tx_resources(adapter);
2388         ixgbevf_free_all_rx_resources(adapter);
2389
2390         return 0;
2391 }
2392
2393 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
2394                                 u32 vlan_macip_lens, u32 type_tucmd,
2395                                 u32 mss_l4len_idx)
2396 {
2397         struct ixgbe_adv_tx_context_desc *context_desc;
2398         u16 i = tx_ring->next_to_use;
2399
2400         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
2401
2402         i++;
2403         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2404
2405         /* set bits to identify this as an advanced context descriptor */
2406         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2407
2408         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
2409         context_desc->seqnum_seed       = 0;
2410         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
2411         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
2412 }
2413
2414 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
2415                        struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2416 {
2417         u32 vlan_macip_lens, type_tucmd;
2418         u32 mss_l4len_idx, l4len;
2419
2420         if (!skb_is_gso(skb))
2421                 return 0;
2422
2423         if (skb_header_cloned(skb)) {
2424                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2425                 if (err)
2426                         return err;
2427         }
2428
2429         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2430         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
2431
2432         if (skb->protocol == htons(ETH_P_IP)) {
2433                 struct iphdr *iph = ip_hdr(skb);
2434                 iph->tot_len = 0;
2435                 iph->check = 0;
2436                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2437                                                          iph->daddr, 0,
2438                                                          IPPROTO_TCP,
2439                                                          0);
2440                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2441         } else if (skb_is_gso_v6(skb)) {
2442                 ipv6_hdr(skb)->payload_len = 0;
2443                 tcp_hdr(skb)->check =
2444                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2445                                      &ipv6_hdr(skb)->daddr,
2446                                      0, IPPROTO_TCP, 0);
2447         }
2448
2449         /* compute header lengths */
2450         l4len = tcp_hdrlen(skb);
2451         *hdr_len += l4len;
2452         *hdr_len = skb_transport_offset(skb) + l4len;
2453
2454         /* mss_l4len_id: use 1 as index for TSO */
2455         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
2456         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
2457         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
2458
2459         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
2460         vlan_macip_lens = skb_network_header_len(skb);
2461         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2462         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2463
2464         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2465                             type_tucmd, mss_l4len_idx);
2466
2467         return 1;
2468 }
2469
2470 static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
2471                             struct sk_buff *skb, u32 tx_flags)
2472 {
2473
2474
2475
2476         u32 vlan_macip_lens = 0;
2477         u32 mss_l4len_idx = 0;
2478         u32 type_tucmd = 0;
2479
2480         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2481                 u8 l4_hdr = 0;
2482                 switch (skb->protocol) {
2483                 case __constant_htons(ETH_P_IP):
2484                         vlan_macip_lens |= skb_network_header_len(skb);
2485                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
2486                         l4_hdr = ip_hdr(skb)->protocol;
2487                         break;
2488                 case __constant_htons(ETH_P_IPV6):
2489                         vlan_macip_lens |= skb_network_header_len(skb);
2490                         l4_hdr = ipv6_hdr(skb)->nexthdr;
2491                         break;
2492                 default:
2493                         if (unlikely(net_ratelimit())) {
2494                                 dev_warn(tx_ring->dev,
2495                                  "partial checksum but proto=%x!\n",
2496                                  skb->protocol);
2497                         }
2498                         break;
2499                 }
2500
2501                 switch (l4_hdr) {
2502                 case IPPROTO_TCP:
2503                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2504                         mss_l4len_idx = tcp_hdrlen(skb) <<
2505                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2506                         break;
2507                 case IPPROTO_SCTP:
2508                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
2509                         mss_l4len_idx = sizeof(struct sctphdr) <<
2510                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2511                         break;
2512                 case IPPROTO_UDP:
2513                         mss_l4len_idx = sizeof(struct udphdr) <<
2514                                         IXGBE_ADVTXD_L4LEN_SHIFT;
2515                         break;
2516                 default:
2517                         if (unlikely(net_ratelimit())) {
2518                                 dev_warn(tx_ring->dev,
2519                                  "partial checksum but l4 proto=%x!\n",
2520                                  l4_hdr);
2521                         }
2522                         break;
2523                 }
2524         }
2525
2526         /* vlan_macip_lens: MACLEN, VLAN tag */
2527         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
2528         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
2529
2530         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
2531                             type_tucmd, mss_l4len_idx);
2532
2533         return (skb->ip_summed == CHECKSUM_PARTIAL);
2534 }
2535
2536 static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
2537                           struct sk_buff *skb, u32 tx_flags,
2538                           unsigned int first)
2539 {
2540         struct ixgbevf_tx_buffer *tx_buffer_info;
2541         unsigned int len;
2542         unsigned int total = skb->len;
2543         unsigned int offset = 0, size;
2544         int count = 0;
2545         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
2546         unsigned int f;
2547         int i;
2548
2549         i = tx_ring->next_to_use;
2550
2551         len = min(skb_headlen(skb), total);
2552         while (len) {
2553                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2554                 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2555
2556                 tx_buffer_info->length = size;
2557                 tx_buffer_info->mapped_as_page = false;
2558                 tx_buffer_info->dma = dma_map_single(tx_ring->dev,
2559                                                      skb->data + offset,
2560                                                      size, DMA_TO_DEVICE);
2561                 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma))
2562                         goto dma_error;
2563                 tx_buffer_info->next_to_watch = i;
2564
2565                 len -= size;
2566                 total -= size;
2567                 offset += size;
2568                 count++;
2569                 i++;
2570                 if (i == tx_ring->count)
2571                         i = 0;
2572         }
2573
2574         for (f = 0; f < nr_frags; f++) {
2575                 const struct skb_frag_struct *frag;
2576
2577                 frag = &skb_shinfo(skb)->frags[f];
2578                 len = min((unsigned int)skb_frag_size(frag), total);
2579                 offset = 0;
2580
2581                 while (len) {
2582                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
2583                         size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD);
2584
2585                         tx_buffer_info->length = size;
2586                         tx_buffer_info->dma =
2587                                 skb_frag_dma_map(tx_ring->dev, frag,
2588                                                  offset, size, DMA_TO_DEVICE);
2589                         tx_buffer_info->mapped_as_page = true;
2590                         if (dma_mapping_error(tx_ring->dev,
2591                                               tx_buffer_info->dma))
2592                                 goto dma_error;
2593                         tx_buffer_info->next_to_watch = i;
2594
2595                         len -= size;
2596                         total -= size;
2597                         offset += size;
2598                         count++;
2599                         i++;
2600                         if (i == tx_ring->count)
2601                                 i = 0;
2602                 }
2603                 if (total == 0)
2604                         break;
2605         }
2606
2607         if (i == 0)
2608                 i = tx_ring->count - 1;
2609         else
2610                 i = i - 1;
2611         tx_ring->tx_buffer_info[i].skb = skb;
2612         tx_ring->tx_buffer_info[first].next_to_watch = i;
2613         tx_ring->tx_buffer_info[first].time_stamp = jiffies;
2614
2615         return count;
2616
2617 dma_error:
2618         dev_err(tx_ring->dev, "TX DMA map failed\n");
2619
2620         /* clear timestamp and dma mappings for failed tx_buffer_info map */
2621         tx_buffer_info->dma = 0;
2622         tx_buffer_info->next_to_watch = 0;
2623         count--;
2624
2625         /* clear timestamp and dma mappings for remaining portion of packet */
2626         while (count >= 0) {
2627                 count--;
2628                 i--;
2629                 if (i < 0)
2630                         i += tx_ring->count;
2631                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2632                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2633         }
2634
2635         return count;
2636 }
2637
2638 static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags,
2639                              int count, u32 paylen, u8 hdr_len)
2640 {
2641         union ixgbe_adv_tx_desc *tx_desc = NULL;
2642         struct ixgbevf_tx_buffer *tx_buffer_info;
2643         u32 olinfo_status = 0, cmd_type_len = 0;
2644         unsigned int i;
2645
2646         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
2647
2648         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
2649
2650         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
2651
2652         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
2653                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2654
2655         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
2656                 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM;
2657
2658         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
2659                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2660
2661                 /* use index 1 context for tso */
2662                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
2663                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
2664                         olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM;
2665
2666         }
2667
2668         /*
2669          * Check Context must be set if Tx switch is enabled, which it
2670          * always is for case where virtual functions are running
2671          */
2672         olinfo_status |= IXGBE_ADVTXD_CC;
2673
2674         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
2675
2676         i = tx_ring->next_to_use;
2677         while (count--) {
2678                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2679                 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2680                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
2681                 tx_desc->read.cmd_type_len =
2682                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
2683                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2684                 i++;
2685                 if (i == tx_ring->count)
2686                         i = 0;
2687         }
2688
2689         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
2690
2691         tx_ring->next_to_use = i;
2692 }
2693
2694 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2695 {
2696         struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
2697
2698         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2699         /* Herbert's original patch had:
2700          *  smp_mb__after_netif_stop_queue();
2701          * but since that doesn't exist yet, just open code it. */
2702         smp_mb();
2703
2704         /* We need to check again in a case another CPU has just
2705          * made room available. */
2706         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
2707                 return -EBUSY;
2708
2709         /* A reprieve! - use start_queue because it doesn't call schedule */
2710         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2711         ++adapter->restart_queue;
2712         return 0;
2713 }
2714
2715 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
2716 {
2717         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
2718                 return 0;
2719         return __ixgbevf_maybe_stop_tx(tx_ring, size);
2720 }
2721
2722 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2723 {
2724         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2725         struct ixgbevf_ring *tx_ring;
2726         unsigned int first;
2727         unsigned int tx_flags = 0;
2728         u8 hdr_len = 0;
2729         int r_idx = 0, tso;
2730         u16 count = TXD_USE_COUNT(skb_headlen(skb));
2731 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2732         unsigned short f;
2733 #endif
2734
2735         tx_ring = &adapter->tx_ring[r_idx];
2736
2737         /*
2738          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
2739          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
2740          *       + 2 desc gap to keep tail from touching head,
2741          *       + 1 desc for context descriptor,
2742          * otherwise try next time
2743          */
2744 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
2745         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2746                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2747 #else
2748         count += skb_shinfo(skb)->nr_frags;
2749 #endif
2750         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
2751                 adapter->tx_busy++;
2752                 return NETDEV_TX_BUSY;
2753         }
2754
2755         if (vlan_tx_tag_present(skb)) {
2756                 tx_flags |= vlan_tx_tag_get(skb);
2757                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
2758                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
2759         }
2760
2761         first = tx_ring->next_to_use;
2762
2763         if (skb->protocol == htons(ETH_P_IP))
2764                 tx_flags |= IXGBE_TX_FLAGS_IPV4;
2765         tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len);
2766         if (tso < 0) {
2767                 dev_kfree_skb_any(skb);
2768                 return NETDEV_TX_OK;
2769         }
2770
2771         if (tso)
2772                 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM;
2773         else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags))
2774                 tx_flags |= IXGBE_TX_FLAGS_CSUM;
2775
2776         ixgbevf_tx_queue(tx_ring, tx_flags,
2777                          ixgbevf_tx_map(tx_ring, skb, tx_flags, first),
2778                          skb->len, hdr_len);
2779         /*
2780          * Force memory writes to complete before letting h/w
2781          * know there are new descriptors to fetch.  (Only
2782          * applicable for weak-ordered memory model archs,
2783          * such as IA-64).
2784          */
2785         wmb();
2786
2787         writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail);
2788
2789         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
2790
2791         return NETDEV_TX_OK;
2792 }
2793
2794 /**
2795  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
2796  * @netdev: network interface device structure
2797  * @p: pointer to an address structure
2798  *
2799  * Returns 0 on success, negative on failure
2800  **/
2801 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
2802 {
2803         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2804         struct ixgbe_hw *hw = &adapter->hw;
2805         struct sockaddr *addr = p;
2806
2807         if (!is_valid_ether_addr(addr->sa_data))
2808                 return -EADDRNOTAVAIL;
2809
2810         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2811         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2812
2813         if (hw->mac.ops.set_rar)
2814                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2815
2816         return 0;
2817 }
2818
2819 /**
2820  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
2821  * @netdev: network interface device structure
2822  * @new_mtu: new value for maximum frame size
2823  *
2824  * Returns 0 on success, negative on failure
2825  **/
2826 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
2827 {
2828         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2829         struct ixgbe_hw *hw = &adapter->hw;
2830         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2831         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
2832         u32 msg[2];
2833
2834         if (adapter->hw.mac.type == ixgbe_mac_X540_vf)
2835                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
2836
2837         /* MTU < 68 is an error and causes problems on some kernels */
2838         if ((new_mtu < 68) || (max_frame > max_possible_frame))
2839                 return -EINVAL;
2840
2841         hw_dbg(&adapter->hw, "changing MTU from %d to %d\n",
2842                netdev->mtu, new_mtu);
2843         /* must set new MTU before calling down or up */
2844         netdev->mtu = new_mtu;
2845
2846         if (!netif_running(netdev)) {
2847                 msg[0] = IXGBE_VF_SET_LPE;
2848                 msg[1] = max_frame;
2849                 hw->mbx.ops.write_posted(hw, msg, 2);
2850         }
2851
2852         if (netif_running(netdev))
2853                 ixgbevf_reinit_locked(adapter);
2854
2855         return 0;
2856 }
2857
2858 static void ixgbevf_shutdown(struct pci_dev *pdev)
2859 {
2860         struct net_device *netdev = pci_get_drvdata(pdev);
2861         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2862
2863         netif_device_detach(netdev);
2864
2865         if (netif_running(netdev)) {
2866                 ixgbevf_down(adapter);
2867                 ixgbevf_free_irq(adapter);
2868                 ixgbevf_free_all_tx_resources(adapter);
2869                 ixgbevf_free_all_rx_resources(adapter);
2870         }
2871
2872         pci_save_state(pdev);
2873
2874         pci_disable_device(pdev);
2875 }
2876
2877 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
2878                                                 struct rtnl_link_stats64 *stats)
2879 {
2880         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2881         unsigned int start;
2882         u64 bytes, packets;
2883         const struct ixgbevf_ring *ring;
2884         int i;
2885
2886         ixgbevf_update_stats(adapter);
2887
2888         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
2889
2890         for (i = 0; i < adapter->num_rx_queues; i++) {
2891                 ring = &adapter->rx_ring[i];
2892                 do {
2893                         start = u64_stats_fetch_begin_bh(&ring->syncp);
2894                         bytes = ring->total_bytes;
2895                         packets = ring->total_packets;
2896                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
2897                 stats->rx_bytes += bytes;
2898                 stats->rx_packets += packets;
2899         }
2900
2901         for (i = 0; i < adapter->num_tx_queues; i++) {
2902                 ring = &adapter->tx_ring[i];
2903                 do {
2904                         start = u64_stats_fetch_begin_bh(&ring->syncp);
2905                         bytes = ring->total_bytes;
2906                         packets = ring->total_packets;
2907                 } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
2908                 stats->tx_bytes += bytes;
2909                 stats->tx_packets += packets;
2910         }
2911
2912         return stats;
2913 }
2914
2915 static const struct net_device_ops ixgbe_netdev_ops = {
2916         .ndo_open               = ixgbevf_open,
2917         .ndo_stop               = ixgbevf_close,
2918         .ndo_start_xmit         = ixgbevf_xmit_frame,
2919         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
2920         .ndo_get_stats64        = ixgbevf_get_stats,
2921         .ndo_validate_addr      = eth_validate_addr,
2922         .ndo_set_mac_address    = ixgbevf_set_mac,
2923         .ndo_change_mtu         = ixgbevf_change_mtu,
2924         .ndo_tx_timeout         = ixgbevf_tx_timeout,
2925         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
2926         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
2927 };
2928
2929 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
2930 {
2931         dev->netdev_ops = &ixgbe_netdev_ops;
2932         ixgbevf_set_ethtool_ops(dev);
2933         dev->watchdog_timeo = 5 * HZ;
2934 }
2935
2936 /**
2937  * ixgbevf_probe - Device Initialization Routine
2938  * @pdev: PCI device information struct
2939  * @ent: entry in ixgbevf_pci_tbl
2940  *
2941  * Returns 0 on success, negative on failure
2942  *
2943  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
2944  * The OS initialization, configuring of the adapter private structure,
2945  * and a hardware reset occur.
2946  **/
2947 static int __devinit ixgbevf_probe(struct pci_dev *pdev,
2948                                    const struct pci_device_id *ent)
2949 {
2950         struct net_device *netdev;
2951         struct ixgbevf_adapter *adapter = NULL;
2952         struct ixgbe_hw *hw = NULL;
2953         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
2954         static int cards_found;
2955         int err, pci_using_dac;
2956
2957         err = pci_enable_device(pdev);
2958         if (err)
2959                 return err;
2960
2961         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
2962             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
2963                 pci_using_dac = 1;
2964         } else {
2965                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2966                 if (err) {
2967                         err = dma_set_coherent_mask(&pdev->dev,
2968                                                     DMA_BIT_MASK(32));
2969                         if (err) {
2970                                 dev_err(&pdev->dev, "No usable DMA "
2971                                         "configuration, aborting\n");
2972                                 goto err_dma;
2973                         }
2974                 }
2975                 pci_using_dac = 0;
2976         }
2977
2978         err = pci_request_regions(pdev, ixgbevf_driver_name);
2979         if (err) {
2980                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
2981                 goto err_pci_reg;
2982         }
2983
2984         pci_set_master(pdev);
2985
2986         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
2987                                    MAX_TX_QUEUES);
2988         if (!netdev) {
2989                 err = -ENOMEM;
2990                 goto err_alloc_etherdev;
2991         }
2992
2993         SET_NETDEV_DEV(netdev, &pdev->dev);
2994
2995         pci_set_drvdata(pdev, netdev);
2996         adapter = netdev_priv(netdev);
2997
2998         adapter->netdev = netdev;
2999         adapter->pdev = pdev;
3000         hw = &adapter->hw;
3001         hw->back = adapter;
3002         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3003
3004         /*
3005          * call save state here in standalone driver because it relies on
3006          * adapter struct to exist, and needs to call netdev_priv
3007          */
3008         pci_save_state(pdev);
3009
3010         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3011                               pci_resource_len(pdev, 0));
3012         if (!hw->hw_addr) {
3013                 err = -EIO;
3014                 goto err_ioremap;
3015         }
3016
3017         ixgbevf_assign_netdev_ops(netdev);
3018
3019         adapter->bd_number = cards_found;
3020
3021         /* Setup hw api */
3022         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3023         hw->mac.type  = ii->mac;
3024
3025         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3026                sizeof(struct ixgbe_mbx_operations));
3027
3028         /* setup the private structure */
3029         err = ixgbevf_sw_init(adapter);
3030         if (err)
3031                 goto err_sw_init;
3032
3033         /* The HW MAC address was set and/or determined in sw_init */
3034         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
3035
3036         if (!is_valid_ether_addr(netdev->dev_addr)) {
3037                 pr_err("invalid MAC address\n");
3038                 err = -EIO;
3039                 goto err_sw_init;
3040         }
3041
3042         netdev->hw_features = NETIF_F_SG |
3043                            NETIF_F_IP_CSUM |
3044                            NETIF_F_IPV6_CSUM |
3045                            NETIF_F_TSO |
3046                            NETIF_F_TSO6 |
3047                            NETIF_F_RXCSUM;
3048
3049         netdev->features = netdev->hw_features |
3050                            NETIF_F_HW_VLAN_TX |
3051                            NETIF_F_HW_VLAN_RX |
3052                            NETIF_F_HW_VLAN_FILTER;
3053
3054         netdev->vlan_features |= NETIF_F_TSO;
3055         netdev->vlan_features |= NETIF_F_TSO6;
3056         netdev->vlan_features |= NETIF_F_IP_CSUM;
3057         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3058         netdev->vlan_features |= NETIF_F_SG;
3059
3060         if (pci_using_dac)
3061                 netdev->features |= NETIF_F_HIGHDMA;
3062
3063         netdev->priv_flags |= IFF_UNICAST_FLT;
3064
3065         init_timer(&adapter->watchdog_timer);
3066         adapter->watchdog_timer.function = ixgbevf_watchdog;
3067         adapter->watchdog_timer.data = (unsigned long)adapter;
3068
3069         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3070         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3071
3072         err = ixgbevf_init_interrupt_scheme(adapter);
3073         if (err)
3074                 goto err_sw_init;
3075
3076         /* pick up the PCI bus settings for reporting later */
3077         if (hw->mac.ops.get_bus_info)
3078                 hw->mac.ops.get_bus_info(hw);
3079
3080         strcpy(netdev->name, "eth%d");
3081
3082         err = register_netdev(netdev);
3083         if (err)
3084                 goto err_register;
3085
3086         netif_carrier_off(netdev);
3087
3088         ixgbevf_init_last_counter_stats(adapter);
3089
3090         /* print the MAC address */
3091         hw_dbg(hw, "%pM\n", netdev->dev_addr);
3092
3093         hw_dbg(hw, "MAC: %d\n", hw->mac.type);
3094
3095         hw_dbg(hw, "Intel(R) 82599 Virtual Function\n");
3096         cards_found++;
3097         return 0;
3098
3099 err_register:
3100 err_sw_init:
3101         ixgbevf_reset_interrupt_capability(adapter);
3102         iounmap(hw->hw_addr);
3103 err_ioremap:
3104         free_netdev(netdev);
3105 err_alloc_etherdev:
3106         pci_release_regions(pdev);
3107 err_pci_reg:
3108 err_dma:
3109         pci_disable_device(pdev);
3110         return err;
3111 }
3112
3113 /**
3114  * ixgbevf_remove - Device Removal Routine
3115  * @pdev: PCI device information struct
3116  *
3117  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3118  * that it should release a PCI device.  The could be caused by a
3119  * Hot-Plug event, or because the driver is going to be removed from
3120  * memory.
3121  **/
3122 static void __devexit ixgbevf_remove(struct pci_dev *pdev)
3123 {
3124         struct net_device *netdev = pci_get_drvdata(pdev);
3125         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3126
3127         set_bit(__IXGBEVF_DOWN, &adapter->state);
3128
3129         del_timer_sync(&adapter->watchdog_timer);
3130
3131         cancel_work_sync(&adapter->reset_task);
3132         cancel_work_sync(&adapter->watchdog_task);
3133
3134         if (netdev->reg_state == NETREG_REGISTERED)
3135                 unregister_netdev(netdev);
3136
3137         ixgbevf_reset_interrupt_capability(adapter);
3138
3139         iounmap(adapter->hw.hw_addr);
3140         pci_release_regions(pdev);
3141
3142         hw_dbg(&adapter->hw, "Remove complete\n");
3143
3144         kfree(adapter->tx_ring);
3145         kfree(adapter->rx_ring);
3146
3147         free_netdev(netdev);
3148
3149         pci_disable_device(pdev);
3150 }
3151
3152 static struct pci_driver ixgbevf_driver = {
3153         .name     = ixgbevf_driver_name,
3154         .id_table = ixgbevf_pci_tbl,
3155         .probe    = ixgbevf_probe,
3156         .remove   = __devexit_p(ixgbevf_remove),
3157         .shutdown = ixgbevf_shutdown,
3158 };
3159
3160 /**
3161  * ixgbevf_init_module - Driver Registration Routine
3162  *
3163  * ixgbevf_init_module is the first routine called when the driver is
3164  * loaded. All it does is register with the PCI subsystem.
3165  **/
3166 static int __init ixgbevf_init_module(void)
3167 {
3168         int ret;
3169         pr_info("%s - version %s\n", ixgbevf_driver_string,
3170                 ixgbevf_driver_version);
3171
3172         pr_info("%s\n", ixgbevf_copyright);
3173
3174         ret = pci_register_driver(&ixgbevf_driver);
3175         return ret;
3176 }
3177
3178 module_init(ixgbevf_init_module);
3179
3180 /**
3181  * ixgbevf_exit_module - Driver Exit Cleanup Routine
3182  *
3183  * ixgbevf_exit_module is called just before the driver is removed
3184  * from memory.
3185  **/
3186 static void __exit ixgbevf_exit_module(void)
3187 {
3188         pci_unregister_driver(&ixgbevf_driver);
3189 }
3190
3191 #ifdef DEBUG
3192 /**
3193  * ixgbevf_get_hw_dev_name - return device name string
3194  * used by hardware layer to print debugging information
3195  **/
3196 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
3197 {
3198         struct ixgbevf_adapter *adapter = hw->back;
3199         return adapter->netdev->name;
3200 }
3201
3202 #endif
3203 module_exit(ixgbevf_exit_module);
3204
3205 /* ixgbevf_main.c */