drivers/net/ethernet/intel/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  29
  30 #include <linux/module.h>
  31 #include <linux/types.h>
  32 #include <linux/init.h>
  33 #include <linux/bitops.h>
  34 #include <linux/vmalloc.h>
  35 #include <linux/pagemap.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/ipv6.h>
  38 #include <linux/slab.h>
  39 #include <net/checksum.h>
  40 #include <net/ip6_checksum.h>
  41 #include <linux/net_tstamp.h>
  42 #include <linux/mii.h>
  43 #include <linux/ethtool.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/pci.h>
  47 #include <linux/pci-aspm.h>
  48 #include <linux/delay.h>
  49 #include <linux/interrupt.h>
  50 #include <linux/ip.h>
  51 #include <linux/tcp.h>
  52 #include <linux/sctp.h>
  53 #include <linux/if_ether.h>
  54 #include <linux/aer.h>
  55 #include <linux/prefetch.h>
  56 #include <linux/pm_runtime.h>
  57 #ifdef CONFIG_IGB_DCA
  58 #include <linux/dca.h>
  59 #endif
  60 #include "igb.h"
  61
  62 #define MAJ 4
  63 #define MIN 0
  64 #define BUILD 1
  65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  66 __stringify(BUILD) "-k"
  67 char igb_driver_name[] = "igb";
  68 char igb_driver_version[] = DRV_VERSION;
  69 static const char igb_driver_string[] =
  70                                 "Intel(R) Gigabit Ethernet Network Driver";
  71 static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
  72
  73 static const struct e1000_info *igb_info_tbl[] = {
  74         [board_82575] = &e1000_82575_info,
  75 };
  76
  77 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  99         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
 100         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
 101         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
 102         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
 103         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
 104         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
 105         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
 106         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
 107         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
 108         /* required last entry */
 109         {0, }
 110 };
 111
 112 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 113
 114 void igb_reset(struct igb_adapter *);
 115 static int igb_setup_all_tx_resources(struct igb_adapter *);
 116 static int igb_setup_all_rx_resources(struct igb_adapter *);
 117 static void igb_free_all_tx_resources(struct igb_adapter *);
 118 static void igb_free_all_rx_resources(struct igb_adapter *);
 119 static void igb_setup_mrqc(struct igb_adapter *);
 120 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 121 static void __devexit igb_remove(struct pci_dev *pdev);
 122 static int igb_sw_init(struct igb_adapter *);
 123 static int igb_open(struct net_device *);
 124 static int igb_close(struct net_device *);
 125 static void igb_configure_tx(struct igb_adapter *);
 126 static void igb_configure_rx(struct igb_adapter *);
 127 static void igb_clean_all_tx_rings(struct igb_adapter *);
 128 static void igb_clean_all_rx_rings(struct igb_adapter *);
 129 static void igb_clean_tx_ring(struct igb_ring *);
 130 static void igb_clean_rx_ring(struct igb_ring *);
 131 static void igb_set_rx_mode(struct net_device *);
 132 static void igb_update_phy_info(unsigned long);
 133 static void igb_watchdog(unsigned long);
 134 static void igb_watchdog_task(struct work_struct *);
 135 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 136 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
 137                                                  struct rtnl_link_stats64 *stats);
 138 static int igb_change_mtu(struct net_device *, int);
 139 static int igb_set_mac(struct net_device *, void *);
 140 static void igb_set_uta(struct igb_adapter *adapter);
 141 static irqreturn_t igb_intr(int irq, void *);
 142 static irqreturn_t igb_intr_msi(int irq, void *);
 143 static irqreturn_t igb_msix_other(int irq, void *);
 144 static irqreturn_t igb_msix_ring(int irq, void *);
 145 #ifdef CONFIG_IGB_DCA
 146 static void igb_update_dca(struct igb_q_vector *);
 147 static void igb_setup_dca(struct igb_adapter *);
 148 #endif /* CONFIG_IGB_DCA */
 149 static int igb_poll(struct napi_struct *, int);
 150 static bool igb_clean_tx_irq(struct igb_q_vector *);
 151 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
 152 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 153 static void igb_tx_timeout(struct net_device *);
 154 static void igb_reset_task(struct work_struct *);
 155 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
 156 static int igb_vlan_rx_add_vid(struct net_device *, u16);
 157 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
 158 static void igb_restore_vlan(struct igb_adapter *);
 159 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 160 static void igb_ping_all_vfs(struct igb_adapter *);
 161 static void igb_msg_task(struct igb_adapter *);
 162 static void igb_vmm_control(struct igb_adapter *);
 163 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 164 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 165 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 166 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 167                                int vf, u16 vlan, u8 qos);
 168 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 169 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 170                                  struct ifla_vf_info *ivi);
 171 static void igb_check_vf_rate_limit(struct igb_adapter *);
 172
 173 #ifdef CONFIG_PCI_IOV
 174 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 175 static bool igb_vfs_are_assigned(struct igb_adapter *adapter);
 176 #endif
 177
 178 #ifdef CONFIG_PM
 179 #ifdef CONFIG_PM_SLEEP
 180 static int igb_suspend(struct device *);
 181 #endif
 182 static int igb_resume(struct device *);
 183 #ifdef CONFIG_PM_RUNTIME
 184 static int igb_runtime_suspend(struct device *dev);
 185 static int igb_runtime_resume(struct device *dev);
 186 static int igb_runtime_idle(struct device *dev);
 187 #endif
 188 static const struct dev_pm_ops igb_pm_ops = {
 189         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 190         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 191                         igb_runtime_idle)
 192 };
 193 #endif
 194 static void igb_shutdown(struct pci_dev *);
 195 #ifdef CONFIG_IGB_DCA
 196 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 197 static struct notifier_block dca_notifier = {
 198         .notifier_call  = igb_notify_dca,
 199         .next           = NULL,
 200         .priority       = 0
 201 };
 202 #endif
 203 #ifdef CONFIG_NET_POLL_CONTROLLER
 204 /* for netdump / net console */
 205 static void igb_netpoll(struct net_device *);
 206 #endif
 207 #ifdef CONFIG_PCI_IOV
 208 static unsigned int max_vfs = 0;
 209 module_param(max_vfs, uint, 0);
 210 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 211                  "per physical function");
 212 #endif /* CONFIG_PCI_IOV */
 213
 214 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 215                      pci_channel_state_t);
 216 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 217 static void igb_io_resume(struct pci_dev *);
 218
 219 static const struct pci_error_handlers igb_err_handler = {
 220         .error_detected = igb_io_error_detected,
 221         .slot_reset = igb_io_slot_reset,
 222         .resume = igb_io_resume,
 223 };
 224
 225 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 226
 227 static struct pci_driver igb_driver = {
 228         .name     = igb_driver_name,
 229         .id_table = igb_pci_tbl,
 230         .probe    = igb_probe,
 231         .remove   = __devexit_p(igb_remove),
 232 #ifdef CONFIG_PM
 233         .driver.pm = &igb_pm_ops,
 234 #endif
 235         .shutdown = igb_shutdown,
 236         .err_handler = &igb_err_handler
 237 };
 238
 239 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 240 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 241 MODULE_LICENSE("GPL");
 242 MODULE_VERSION(DRV_VERSION);
 243
 244 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
 245 static int debug = -1;
 246 module_param(debug, int, 0);
 247 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 248
 249 struct igb_reg_info {
 250         u32 ofs;
 251         char *name;
 252 };
 253
 254 static const struct igb_reg_info igb_reg_info_tbl[] = {
 255
 256         /* General Registers */
 257         {E1000_CTRL, "CTRL"},
 258         {E1000_STATUS, "STATUS"},
 259         {E1000_CTRL_EXT, "CTRL_EXT"},
 260
 261         /* Interrupt Registers */
 262         {E1000_ICR, "ICR"},
 263
 264         /* RX Registers */
 265         {E1000_RCTL, "RCTL"},
 266         {E1000_RDLEN(0), "RDLEN"},
 267         {E1000_RDH(0), "RDH"},
 268         {E1000_RDT(0), "RDT"},
 269         {E1000_RXDCTL(0), "RXDCTL"},
 270         {E1000_RDBAL(0), "RDBAL"},
 271         {E1000_RDBAH(0), "RDBAH"},
 272
 273         /* TX Registers */
 274         {E1000_TCTL, "TCTL"},
 275         {E1000_TDBAL(0), "TDBAL"},
 276         {E1000_TDBAH(0), "TDBAH"},
 277         {E1000_TDLEN(0), "TDLEN"},
 278         {E1000_TDH(0), "TDH"},
 279         {E1000_TDT(0), "TDT"},
 280         {E1000_TXDCTL(0), "TXDCTL"},
 281         {E1000_TDFH, "TDFH"},
 282         {E1000_TDFT, "TDFT"},
 283         {E1000_TDFHS, "TDFHS"},
 284         {E1000_TDFPC, "TDFPC"},
 285
 286         /* List Terminator */
 287         {}
 288 };
 289
 290 /*
 291  * igb_regdump - register printout routine
 292  */
 293 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
 294 {
 295         int n = 0;
 296         char rname[16];
 297         u32 regs[8];
 298
 299         switch (reginfo->ofs) {
 300         case E1000_RDLEN(0):
 301                 for (n = 0; n < 4; n++)
 302                         regs[n] = rd32(E1000_RDLEN(n));
 303                 break;
 304         case E1000_RDH(0):
 305                 for (n = 0; n < 4; n++)
 306                         regs[n] = rd32(E1000_RDH(n));
 307                 break;
 308         case E1000_RDT(0):
 309                 for (n = 0; n < 4; n++)
 310                         regs[n] = rd32(E1000_RDT(n));
 311                 break;
 312         case E1000_RXDCTL(0):
 313                 for (n = 0; n < 4; n++)
 314                         regs[n] = rd32(E1000_RXDCTL(n));
 315                 break;
 316         case E1000_RDBAL(0):
 317                 for (n = 0; n < 4; n++)
 318                         regs[n] = rd32(E1000_RDBAL(n));
 319                 break;
 320         case E1000_RDBAH(0):
 321                 for (n = 0; n < 4; n++)
 322                         regs[n] = rd32(E1000_RDBAH(n));
 323                 break;
 324         case E1000_TDBAL(0):
 325                 for (n = 0; n < 4; n++)
 326                         regs[n] = rd32(E1000_RDBAL(n));
 327                 break;
 328         case E1000_TDBAH(0):
 329                 for (n = 0; n < 4; n++)
 330                         regs[n] = rd32(E1000_TDBAH(n));
 331                 break;
 332         case E1000_TDLEN(0):
 333                 for (n = 0; n < 4; n++)
 334                         regs[n] = rd32(E1000_TDLEN(n));
 335                 break;
 336         case E1000_TDH(0):
 337                 for (n = 0; n < 4; n++)
 338                         regs[n] = rd32(E1000_TDH(n));
 339                 break;
 340         case E1000_TDT(0):
 341                 for (n = 0; n < 4; n++)
 342                         regs[n] = rd32(E1000_TDT(n));
 343                 break;
 344         case E1000_TXDCTL(0):
 345                 for (n = 0; n < 4; n++)
 346                         regs[n] = rd32(E1000_TXDCTL(n));
 347                 break;
 348         default:
 349                 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
 350                 return;
 351         }
 352
 353         snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
 354         pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
 355                 regs[2], regs[3]);
 356 }
 357
 358 /*
 359  * igb_dump - Print registers, tx-rings and rx-rings
 360  */
 361 static void igb_dump(struct igb_adapter *adapter)
 362 {
 363         struct net_device *netdev = adapter->netdev;
 364         struct e1000_hw *hw = &adapter->hw;
 365         struct igb_reg_info *reginfo;
 366         struct igb_ring *tx_ring;
 367         union e1000_adv_tx_desc *tx_desc;
 368         struct my_u0 { u64 a; u64 b; } *u0;
 369         struct igb_ring *rx_ring;
 370         union e1000_adv_rx_desc *rx_desc;
 371         u32 staterr;
 372         u16 i, n;
 373
 374         if (!netif_msg_hw(adapter))
 375                 return;
 376
 377         /* Print netdevice Info */
 378         if (netdev) {
 379                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 380                 pr_info("Device Name     state            trans_start      "
 381                         "last_rx\n");
 382                 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
 383                         netdev->state, netdev->trans_start, netdev->last_rx);
 384         }
 385
 386         /* Print Registers */
 387         dev_info(&adapter->pdev->dev, "Register Dump\n");
 388         pr_info(" Register Name   Value\n");
 389         for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
 390              reginfo->name; reginfo++) {
 391                 igb_regdump(hw, reginfo);
 392         }
 393
 394         /* Print TX Ring Summary */
 395         if (!netdev || !netif_running(netdev))
 396                 goto exit;
 397
 398         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 399         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 400         for (n = 0; n < adapter->num_tx_queues; n++) {
 401                 struct igb_tx_buffer *buffer_info;
 402                 tx_ring = adapter->tx_ring[n];
 403                 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 404                 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
 405                         n, tx_ring->next_to_use, tx_ring->next_to_clean,
 406                         (u64)dma_unmap_addr(buffer_info, dma),
 407                         dma_unmap_len(buffer_info, len),
 408                         buffer_info->next_to_watch,
 409                         (u64)buffer_info->time_stamp);
 410         }
 411
 412         /* Print TX Rings */
 413         if (!netif_msg_tx_done(adapter))
 414                 goto rx_ring_summary;
 415
 416         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 417
 418         /* Transmit Descriptor Formats
 419          *
 420          * Advanced Transmit Descriptor
 421          *   +--------------------------------------------------------------+
 422          * 0 |         Buffer Address [63:0]                                |
 423          *   +--------------------------------------------------------------+
 424          * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
 425          *   +--------------------------------------------------------------+
 426          *   63      46 45    40 39 38 36 35 32 31   24             15       0
 427          */
 428
 429         for (n = 0; n < adapter->num_tx_queues; n++) {
 430                 tx_ring = adapter->tx_ring[n];
 431                 pr_info("------------------------------------\n");
 432                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 433                 pr_info("------------------------------------\n");
 434                 pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] "
 435                         "[bi->dma       ] leng  ntw timestamp        "
 436                         "bi->skb\n");
 437
 438                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 439                         const char *next_desc;
 440                         struct igb_tx_buffer *buffer_info;
 441                         tx_desc = IGB_TX_DESC(tx_ring, i);
 442                         buffer_info = &tx_ring->tx_buffer_info[i];
 443                         u0 = (struct my_u0 *)tx_desc;
 444                         if (i == tx_ring->next_to_use &&
 445                             i == tx_ring->next_to_clean)
 446                                 next_desc = " NTC/U";
 447                         else if (i == tx_ring->next_to_use)
 448                                 next_desc = " NTU";
 449                         else if (i == tx_ring->next_to_clean)
 450                                 next_desc = " NTC";
 451                         else
 452                                 next_desc = "";
 453
 454                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
 455                                 " %04X  %p %016llX %p%s\n", i,
 456                                 le64_to_cpu(u0->a),
 457                                 le64_to_cpu(u0->b),
 458                                 (u64)dma_unmap_addr(buffer_info, dma),
 459                                 dma_unmap_len(buffer_info, len),
 460                                 buffer_info->next_to_watch,
 461                                 (u64)buffer_info->time_stamp,
 462                                 buffer_info->skb, next_desc);
 463
 464                         if (netif_msg_pktdata(adapter) && buffer_info->skb)
 465                                 print_hex_dump(KERN_INFO, "",
 466                                         DUMP_PREFIX_ADDRESS,
 467                                         16, 1, buffer_info->skb->data,
 468                                         dma_unmap_len(buffer_info, len),
 469                                         true);
 470                 }
 471         }
 472
 473         /* Print RX Rings Summary */
 474 rx_ring_summary:
 475         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 476         pr_info("Queue [NTU] [NTC]\n");
 477         for (n = 0; n < adapter->num_rx_queues; n++) {
 478                 rx_ring = adapter->rx_ring[n];
 479                 pr_info(" %5d %5X %5X\n",
 480                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 481         }
 482
 483         /* Print RX Rings */
 484         if (!netif_msg_rx_status(adapter))
 485                 goto exit;
 486
 487         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 488
 489         /* Advanced Receive Descriptor (Read) Format
 490          *    63                                           1        0
 491          *    +-----------------------------------------------------+
 492          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 493          *    +----------------------------------------------+------+
 494          *  8 |       Header Buffer Address [63:1]           |  DD  |
 495          *    +-----------------------------------------------------+
 496          *
 497          *
 498          * Advanced Receive Descriptor (Write-Back) Format
 499          *
 500          *   63       48 47    32 31  30      21 20 17 16   4 3     0
 501          *   +------------------------------------------------------+
 502          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 503          *   | Checksum   Ident  |   |           |    | Type | Type |
 504          *   +------------------------------------------------------+
 505          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 506          *   +------------------------------------------------------+
 507          *   63       48 47    32 31            20 19               0
 508          */
 509
 510         for (n = 0; n < adapter->num_rx_queues; n++) {
 511                 rx_ring = adapter->rx_ring[n];
 512                 pr_info("------------------------------------\n");
 513                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 514                 pr_info("------------------------------------\n");
 515                 pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] "
 516                         "[bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
 517                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] -----"
 518                         "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
 519
 520                 for (i = 0; i < rx_ring->count; i++) {
 521                         const char *next_desc;
 522                         struct igb_rx_buffer *buffer_info;
 523                         buffer_info = &rx_ring->rx_buffer_info[i];
 524                         rx_desc = IGB_RX_DESC(rx_ring, i);
 525                         u0 = (struct my_u0 *)rx_desc;
 526                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 527
 528                         if (i == rx_ring->next_to_use)
 529                                 next_desc = " NTU";
 530                         else if (i == rx_ring->next_to_clean)
 531                                 next_desc = " NTC";
 532                         else
 533                                 next_desc = "";
 534
 535                         if (staterr & E1000_RXD_STAT_DD) {
 536                                 /* Descriptor Done */
 537                                 pr_info("%s[0x%03X]     %016llX %016llX -------"
 538                                         "--------- %p%s\n", "RWB", i,
 539                                         le64_to_cpu(u0->a),
 540                                         le64_to_cpu(u0->b),
 541                                         buffer_info->skb, next_desc);
 542                         } else {
 543                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX"
 544                                         " %p%s\n", "R  ", i,
 545                                         le64_to_cpu(u0->a),
 546                                         le64_to_cpu(u0->b),
 547                                         (u64)buffer_info->dma,
 548                                         buffer_info->skb, next_desc);
 549
 550                                 if (netif_msg_pktdata(adapter) &&
 551                                     buffer_info->dma && buffer_info->skb) {
 552                                         print_hex_dump(KERN_INFO, "",
 553                                                   DUMP_PREFIX_ADDRESS,
 554                                                   16, 1, buffer_info->skb->data,
 555                                                   IGB_RX_HDR_LEN, true);
 556                                         print_hex_dump(KERN_INFO, "",
 557                                           DUMP_PREFIX_ADDRESS,
 558                                           16, 1,
 559                                           page_address(buffer_info->page) +
 560                                                       buffer_info->page_offset,
 561                                           PAGE_SIZE/2, true);
 562                                 }
 563                         }
 564                 }
 565         }
 566
 567 exit:
 568         return;
 569 }
 570
 571 /**
 572  * igb_get_hw_dev - return device
 573  * used by hardware layer to print debugging information
 574  **/
 575 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 576 {
 577         struct igb_adapter *adapter = hw->back;
 578         return adapter->netdev;
 579 }
 580
 581 /**
 582  * igb_init_module - Driver Registration Routine
 583  *
 584  * igb_init_module is the first routine called when the driver is
 585  * loaded. All it does is register with the PCI subsystem.
 586  **/
 587 static int __init igb_init_module(void)
 588 {
 589         int ret;
 590         pr_info("%s - version %s\n",
 591                igb_driver_string, igb_driver_version);
 592
 593         pr_info("%s\n", igb_copyright);
 594
 595 #ifdef CONFIG_IGB_DCA
 596         dca_register_notify(&dca_notifier);
 597 #endif
 598         ret = pci_register_driver(&igb_driver);
 599         return ret;
 600 }
 601
 602 module_init(igb_init_module);
 603
 604 /**
 605  * igb_exit_module - Driver Exit Cleanup Routine
 606  *
 607  * igb_exit_module is called just before the driver is removed
 608  * from memory.
 609  **/
 610 static void __exit igb_exit_module(void)
 611 {
 612 #ifdef CONFIG_IGB_DCA
 613         dca_unregister_notify(&dca_notifier);
 614 #endif
 615         pci_unregister_driver(&igb_driver);
 616 }
 617
 618 module_exit(igb_exit_module);
 619
 620 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 621 /**
 622  * igb_cache_ring_register - Descriptor ring to register mapping
 623  * @adapter: board private structure to initialize
 624  *
 625  * Once we know the feature-set enabled for the device, we'll cache
 626  * the register offset the descriptor ring is assigned to.
 627  **/
 628 static void igb_cache_ring_register(struct igb_adapter *adapter)
 629 {
 630         int i = 0, j = 0;
 631         u32 rbase_offset = adapter->vfs_allocated_count;
 632
 633         switch (adapter->hw.mac.type) {
 634         case e1000_82576:
 635                 /* The queues are allocated for virtualization such that VF 0
 636                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 637                  * In order to avoid collision we start at the first free queue
 638                  * and continue consuming queues in the same sequence
 639                  */
 640                 if (adapter->vfs_allocated_count) {
 641                         for (; i < adapter->rss_queues; i++)
 642                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 643                                                                Q_IDX_82576(i);
 644                 }
 645         case e1000_82575:
 646         case e1000_82580:
 647         case e1000_i350:
 648         case e1000_i210:
 649         case e1000_i211:
 650         default:
 651                 for (; i < adapter->num_rx_queues; i++)
 652                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 653                 for (; j < adapter->num_tx_queues; j++)
 654                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 655                 break;
 656         }
 657 }
 658
 659 static void igb_free_queues(struct igb_adapter *adapter)
 660 {
 661         int i;
 662
 663         for (i = 0; i < adapter->num_tx_queues; i++) {
 664                 kfree(adapter->tx_ring[i]);
 665                 adapter->tx_ring[i] = NULL;
 666         }
 667         for (i = 0; i < adapter->num_rx_queues; i++) {
 668                 kfree(adapter->rx_ring[i]);
 669                 adapter->rx_ring[i] = NULL;
 670         }
 671         adapter->num_rx_queues = 0;
 672         adapter->num_tx_queues = 0;
 673 }
 674
 675 /**
 676  * igb_alloc_queues - Allocate memory for all rings
 677  * @adapter: board private structure to initialize
 678  *
 679  * We allocate one ring per queue at run-time since we don't know the
 680  * number of queues at compile-time.
 681  **/
 682 static int igb_alloc_queues(struct igb_adapter *adapter)
 683 {
 684         struct igb_ring *ring;
 685         int i;
 686
 687         for (i = 0; i < adapter->num_tx_queues; i++) {
 688                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 689                 if (!ring)
 690                         goto err;
 691                 ring->count = adapter->tx_ring_count;
 692                 ring->queue_index = i;
 693                 ring->dev = &adapter->pdev->dev;
 694                 ring->netdev = adapter->netdev;
 695                 /* For 82575, context index must be unique per ring. */
 696                 if (adapter->hw.mac.type == e1000_82575)
 697                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
 698                 adapter->tx_ring[i] = ring;
 699         }
 700
 701         for (i = 0; i < adapter->num_rx_queues; i++) {
 702                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 703                 if (!ring)
 704                         goto err;
 705                 ring->count = adapter->rx_ring_count;
 706                 ring->queue_index = i;
 707                 ring->dev = &adapter->pdev->dev;
 708                 ring->netdev = adapter->netdev;
 709                 /* set flag indicating ring supports SCTP checksum offload */
 710                 if (adapter->hw.mac.type >= e1000_82576)
 711                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
 712
 713                 /*
 714                  * On i350, i210, and i211, loopback VLAN packets
 715                  * have the tag byte-swapped.
 716                  * */
 717                 if (adapter->hw.mac.type >= e1000_i350)
 718                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
 719
 720                 adapter->rx_ring[i] = ring;
 721         }
 722
 723         igb_cache_ring_register(adapter);
 724
 725         return 0;
 726
 727 err:
 728         igb_free_queues(adapter);
 729
 730         return -ENOMEM;
 731 }
 732
 733 /**
 734  *  igb_write_ivar - configure ivar for given MSI-X vector
 735  *  @hw: pointer to the HW structure
 736  *  @msix_vector: vector number we are allocating to a given ring
 737  *  @index: row index of IVAR register to write within IVAR table
 738  *  @offset: column offset of in IVAR, should be multiple of 8
 739  *
 740  *  This function is intended to handle the writing of the IVAR register
 741  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 742  *  each containing an cause allocation for an Rx and Tx ring, and a
 743  *  variable number of rows depending on the number of queues supported.
 744  **/
 745 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 746                            int index, int offset)
 747 {
 748         u32 ivar = array_rd32(E1000_IVAR0, index);
 749
 750         /* clear any bits that are currently set */
 751         ivar &= ~((u32)0xFF << offset);
 752
 753         /* write vector and valid bit */
 754         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 755
 756         array_wr32(E1000_IVAR0, index, ivar);
 757 }
 758
 759 #define IGB_N0_QUEUE -1
 760 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 761 {
 762         struct igb_adapter *adapter = q_vector->adapter;
 763         struct e1000_hw *hw = &adapter->hw;
 764         int rx_queue = IGB_N0_QUEUE;
 765         int tx_queue = IGB_N0_QUEUE;
 766         u32 msixbm = 0;
 767
 768         if (q_vector->rx.ring)
 769                 rx_queue = q_vector->rx.ring->reg_idx;
 770         if (q_vector->tx.ring)
 771                 tx_queue = q_vector->tx.ring->reg_idx;
 772
 773         switch (hw->mac.type) {
 774         case e1000_82575:
 775                 /* The 82575 assigns vectors using a bitmask, which matches the
 776                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 777                    or more queues to a vector, we write the appropriate bits
 778                    into the MSIXBM register for that vector. */
 779                 if (rx_queue > IGB_N0_QUEUE)
 780                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 781                 if (tx_queue > IGB_N0_QUEUE)
 782                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 783                 if (!adapter->msix_entries && msix_vector == 0)
 784                         msixbm |= E1000_EIMS_OTHER;
 785                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 786                 q_vector->eims_value = msixbm;
 787                 break;
 788         case e1000_82576:
 789                 /*
 790                  * 82576 uses a table that essentially consists of 2 columns
 791                  * with 8 rows.  The ordering is column-major so we use the
 792                  * lower 3 bits as the row index, and the 4th bit as the
 793                  * column offset.
 794                  */
 795                 if (rx_queue > IGB_N0_QUEUE)
 796                         igb_write_ivar(hw, msix_vector,
 797                                        rx_queue & 0x7,
 798                                        (rx_queue & 0x8) << 1);
 799                 if (tx_queue > IGB_N0_QUEUE)
 800                         igb_write_ivar(hw, msix_vector,
 801                                        tx_queue & 0x7,
 802                                        ((tx_queue & 0x8) << 1) + 8);
 803                 q_vector->eims_value = 1 << msix_vector;
 804                 break;
 805         case e1000_82580:
 806         case e1000_i350:
 807         case e1000_i210:
 808         case e1000_i211:
 809                 /*
 810                  * On 82580 and newer adapters the scheme is similar to 82576
 811                  * however instead of ordering column-major we have things
 812                  * ordered row-major.  So we traverse the table by using
 813                  * bit 0 as the column offset, and the remaining bits as the
 814                  * row index.
 815                  */
 816                 if (rx_queue > IGB_N0_QUEUE)
 817                         igb_write_ivar(hw, msix_vector,
 818                                        rx_queue >> 1,
 819                                        (rx_queue & 0x1) << 4);
 820                 if (tx_queue > IGB_N0_QUEUE)
 821                         igb_write_ivar(hw, msix_vector,
 822                                        tx_queue >> 1,
 823                                        ((tx_queue & 0x1) << 4) + 8);
 824                 q_vector->eims_value = 1 << msix_vector;
 825                 break;
 826         default:
 827                 BUG();
 828                 break;
 829         }
 830
 831         /* add q_vector eims value to global eims_enable_mask */
 832         adapter->eims_enable_mask |= q_vector->eims_value;
 833
 834         /* configure q_vector to set itr on first interrupt */
 835         q_vector->set_itr = 1;
 836 }
 837
 838 /**
 839  * igb_configure_msix - Configure MSI-X hardware
 840  *
 841  * igb_configure_msix sets up the hardware to properly
 842  * generate MSI-X interrupts.
 843  **/
 844 static void igb_configure_msix(struct igb_adapter *adapter)
 845 {
 846         u32 tmp;
 847         int i, vector = 0;
 848         struct e1000_hw *hw = &adapter->hw;
 849
 850         adapter->eims_enable_mask = 0;
 851
 852         /* set vector for other causes, i.e. link changes */
 853         switch (hw->mac.type) {
 854         case e1000_82575:
 855                 tmp = rd32(E1000_CTRL_EXT);
 856                 /* enable MSI-X PBA support*/
 857                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 858
 859                 /* Auto-Mask interrupts upon ICR read. */
 860                 tmp |= E1000_CTRL_EXT_EIAME;
 861                 tmp |= E1000_CTRL_EXT_IRCA;
 862
 863                 wr32(E1000_CTRL_EXT, tmp);
 864
 865                 /* enable msix_other interrupt */
 866                 array_wr32(E1000_MSIXBM(0), vector++,
 867                                       E1000_EIMS_OTHER);
 868                 adapter->eims_other = E1000_EIMS_OTHER;
 869
 870                 break;
 871
 872         case e1000_82576:
 873         case e1000_82580:
 874         case e1000_i350:
 875         case e1000_i210:
 876         case e1000_i211:
 877                 /* Turn on MSI-X capability first, or our settings
 878                  * won't stick.  And it will take days to debug. */
 879                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 880                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 881                                 E1000_GPIE_NSICR);
 882
 883                 /* enable msix_other interrupt */
 884                 adapter->eims_other = 1 << vector;
 885                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 886
 887                 wr32(E1000_IVAR_MISC, tmp);
 888                 break;
 889         default:
 890                 /* do nothing, since nothing else supports MSI-X */
 891                 break;
 892         } /* switch (hw->mac.type) */
 893
 894         adapter->eims_enable_mask |= adapter->eims_other;
 895
 896         for (i = 0; i < adapter->num_q_vectors; i++)
 897                 igb_assign_vector(adapter->q_vector[i], vector++);
 898
 899         wrfl();
 900 }
 901
 902 /**
 903  * igb_request_msix - Initialize MSI-X interrupts
 904  *
 905  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 906  * kernel.
 907  **/
 908 static int igb_request_msix(struct igb_adapter *adapter)
 909 {
 910         struct net_device *netdev = adapter->netdev;
 911         struct e1000_hw *hw = &adapter->hw;
 912         int i, err = 0, vector = 0;
 913
 914         err = request_irq(adapter->msix_entries[vector].vector,
 915                           igb_msix_other, 0, netdev->name, adapter);
 916         if (err)
 917                 goto out;
 918         vector++;
 919
 920         for (i = 0; i < adapter->num_q_vectors; i++) {
 921                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 922
 923                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 924
 925                 if (q_vector->rx.ring && q_vector->tx.ring)
 926                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 927                                 q_vector->rx.ring->queue_index);
 928                 else if (q_vector->tx.ring)
 929                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 930                                 q_vector->tx.ring->queue_index);
 931                 else if (q_vector->rx.ring)
 932                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 933                                 q_vector->rx.ring->queue_index);
 934                 else
 935                         sprintf(q_vector->name, "%s-unused", netdev->name);
 936
 937                 err = request_irq(adapter->msix_entries[vector].vector,
 938                                   igb_msix_ring, 0, q_vector->name,
 939                                   q_vector);
 940                 if (err)
 941                         goto out;
 942                 vector++;
 943         }
 944
 945         igb_configure_msix(adapter);
 946         return 0;
 947 out:
 948         return err;
 949 }
 950
 951 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 952 {
 953         if (adapter->msix_entries) {
 954                 pci_disable_msix(adapter->pdev);
 955                 kfree(adapter->msix_entries);
 956                 adapter->msix_entries = NULL;
 957         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 958                 pci_disable_msi(adapter->pdev);
 959         }
 960 }
 961
 962 /**
 963  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 964  * @adapter: board private structure to initialize
 965  *
 966  * This function frees the memory allocated to the q_vectors.  In addition if
 967  * NAPI is enabled it will delete any references to the NAPI struct prior
 968  * to freeing the q_vector.
 969  **/
 970 static void igb_free_q_vectors(struct igb_adapter *adapter)
 971 {
 972         int v_idx;
 973
 974         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 975                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 976                 adapter->q_vector[v_idx] = NULL;
 977                 if (!q_vector)
 978                         continue;
 979                 netif_napi_del(&q_vector->napi);
 980                 kfree(q_vector);
 981         }
 982         adapter->num_q_vectors = 0;
 983 }
 984
 985 /**
 986  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 987  *
 988  * This function resets the device so that it has 0 rx queues, tx queues, and
 989  * MSI-X interrupts allocated.
 990  */
 991 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 992 {
 993         igb_free_queues(adapter);
 994         igb_free_q_vectors(adapter);
 995         igb_reset_interrupt_capability(adapter);
 996 }
 997
 998 /**
 999  * igb_set_interrupt_capability - set MSI or MSI-X if supported
1000  *
1001  * Attempt to configure interrupts using the best available
1002  * capabilities of the hardware and kernel.
1003  **/
1004 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1005 {
1006         int err;
1007         int numvecs, i;
1008
1009         /* Number of supported queues. */
1010         adapter->num_rx_queues = adapter->rss_queues;
1011         if (adapter->vfs_allocated_count)
1012                 adapter->num_tx_queues = 1;
1013         else
1014                 adapter->num_tx_queues = adapter->rss_queues;
1015
1016         /* start with one vector for every rx queue */
1017         numvecs = adapter->num_rx_queues;
1018
1019         /* if tx handler is separate add 1 for every tx queue */
1020         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1021                 numvecs += adapter->num_tx_queues;
1022
1023         /* store the number of vectors reserved for queues */
1024         adapter->num_q_vectors = numvecs;
1025
1026         /* add 1 vector for link status interrupts */
1027         numvecs++;
1028         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1029                                         GFP_KERNEL);
1030
1031         if (!adapter->msix_entries)
1032                 goto msi_only;
1033
1034         for (i = 0; i < numvecs; i++)
1035                 adapter->msix_entries[i].entry = i;
1036
1037         err = pci_enable_msix(adapter->pdev,
1038                               adapter->msix_entries,
1039                               numvecs);
1040         if (err == 0)
1041                 goto out;
1042
1043         igb_reset_interrupt_capability(adapter);
1044
1045         /* If we can't do MSI-X, try MSI */
1046 msi_only:
1047 #ifdef CONFIG_PCI_IOV
1048         /* disable SR-IOV for non MSI-X configurations */
1049         if (adapter->vf_data) {
1050                 struct e1000_hw *hw = &adapter->hw;
1051                 /* disable iov and allow time for transactions to clear */
1052                 pci_disable_sriov(adapter->pdev);
1053                 msleep(500);
1054
1055                 kfree(adapter->vf_data);
1056                 adapter->vf_data = NULL;
1057                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1058                 wrfl();
1059                 msleep(100);
1060                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1061         }
1062 #endif
1063         adapter->vfs_allocated_count = 0;
1064         adapter->rss_queues = 1;
1065         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1066         adapter->num_rx_queues = 1;
1067         adapter->num_tx_queues = 1;
1068         adapter->num_q_vectors = 1;
1069         if (!pci_enable_msi(adapter->pdev))
1070                 adapter->flags |= IGB_FLAG_HAS_MSI;
1071 out:
1072         /* Notify the stack of the (possibly) reduced queue counts. */
1073         rtnl_lock();
1074         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1075         err = netif_set_real_num_rx_queues(adapter->netdev,
1076                 adapter->num_rx_queues);
1077         rtnl_unlock();
1078         return err;
1079 }
1080
1081 /**
1082  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1083  * @adapter: board private structure to initialize
1084  *
1085  * We allocate one q_vector per queue interrupt.  If allocation fails we
1086  * return -ENOMEM.
1087  **/
1088 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1089 {
1090         struct igb_q_vector *q_vector;
1091         struct e1000_hw *hw = &adapter->hw;
1092         int v_idx;
1093
1094         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1095                 q_vector = kzalloc(sizeof(struct igb_q_vector),
1096                                    GFP_KERNEL);
1097                 if (!q_vector)
1098                         goto err_out;
1099                 q_vector->adapter = adapter;
1100                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1101                 q_vector->itr_val = IGB_START_ITR;
1102                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1103                 adapter->q_vector[v_idx] = q_vector;
1104         }
1105
1106         return 0;
1107
1108 err_out:
1109         igb_free_q_vectors(adapter);
1110         return -ENOMEM;
1111 }
1112
1113 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1114                                       int ring_idx, int v_idx)
1115 {
1116         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1117
1118         q_vector->rx.ring = adapter->rx_ring[ring_idx];
1119         q_vector->rx.ring->q_vector = q_vector;
1120         q_vector->rx.count++;
1121         q_vector->itr_val = adapter->rx_itr_setting;
1122         if (q_vector->itr_val && q_vector->itr_val <= 3)
1123                 q_vector->itr_val = IGB_START_ITR;
1124 }
1125
1126 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1127                                       int ring_idx, int v_idx)
1128 {
1129         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1130
1131         q_vector->tx.ring = adapter->tx_ring[ring_idx];
1132         q_vector->tx.ring->q_vector = q_vector;
1133         q_vector->tx.count++;
1134         q_vector->itr_val = adapter->tx_itr_setting;
1135         q_vector->tx.work_limit = adapter->tx_work_limit;
1136         if (q_vector->itr_val && q_vector->itr_val <= 3)
1137                 q_vector->itr_val = IGB_START_ITR;
1138 }
1139
1140 /**
1141  * igb_map_ring_to_vector - maps allocated queues to vectors
1142  *
1143  * This function maps the recently allocated queues to vectors.
1144  **/
1145 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1146 {
1147         int i;
1148         int v_idx = 0;
1149
1150         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1151             (adapter->num_q_vectors < adapter->num_tx_queues))
1152                 return -ENOMEM;
1153
1154         if (adapter->num_q_vectors >=
1155             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1156                 for (i = 0; i < adapter->num_rx_queues; i++)
1157                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1158                 for (i = 0; i < adapter->num_tx_queues; i++)
1159                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1160         } else {
1161                 for (i = 0; i < adapter->num_rx_queues; i++) {
1162                         if (i < adapter->num_tx_queues)
1163                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1164                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1165                 }
1166                 for (; i < adapter->num_tx_queues; i++)
1167                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1168         }
1169         return 0;
1170 }
1171
1172 /**
1173  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1174  *
1175  * This function initializes the interrupts and allocates all of the queues.
1176  **/
1177 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1178 {
1179         struct pci_dev *pdev = adapter->pdev;
1180         int err;
1181
1182         err = igb_set_interrupt_capability(adapter);
1183         if (err)
1184                 return err;
1185
1186         err = igb_alloc_q_vectors(adapter);
1187         if (err) {
1188                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1189                 goto err_alloc_q_vectors;
1190         }
1191
1192         err = igb_alloc_queues(adapter);
1193         if (err) {
1194                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1195                 goto err_alloc_queues;
1196         }
1197
1198         err = igb_map_ring_to_vector(adapter);
1199         if (err) {
1200                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1201                 goto err_map_queues;
1202         }
1203
1204
1205         return 0;
1206 err_map_queues:
1207         igb_free_queues(adapter);
1208 err_alloc_queues:
1209         igb_free_q_vectors(adapter);
1210 err_alloc_q_vectors:
1211         igb_reset_interrupt_capability(adapter);
1212         return err;
1213 }
1214
1215 /**
1216  * igb_request_irq - initialize interrupts
1217  *
1218  * Attempts to configure interrupts using the best available
1219  * capabilities of the hardware and kernel.
1220  **/
1221 static int igb_request_irq(struct igb_adapter *adapter)
1222 {
1223         struct net_device *netdev = adapter->netdev;
1224         struct pci_dev *pdev = adapter->pdev;
1225         int err = 0;
1226
1227         if (adapter->msix_entries) {
1228                 err = igb_request_msix(adapter);
1229                 if (!err)
1230                         goto request_done;
1231                 /* fall back to MSI */
1232                 igb_clear_interrupt_scheme(adapter);
1233                 if (!pci_enable_msi(pdev))
1234                         adapter->flags |= IGB_FLAG_HAS_MSI;
1235                 igb_free_all_tx_resources(adapter);
1236                 igb_free_all_rx_resources(adapter);
1237                 adapter->num_tx_queues = 1;
1238                 adapter->num_rx_queues = 1;
1239                 adapter->num_q_vectors = 1;
1240                 err = igb_alloc_q_vectors(adapter);
1241                 if (err) {
1242                         dev_err(&pdev->dev,
1243                                 "Unable to allocate memory for vectors\n");
1244                         goto request_done;
1245                 }
1246                 err = igb_alloc_queues(adapter);
1247                 if (err) {
1248                         dev_err(&pdev->dev,
1249                                 "Unable to allocate memory for queues\n");
1250                         igb_free_q_vectors(adapter);
1251                         goto request_done;
1252                 }
1253                 igb_setup_all_tx_resources(adapter);
1254                 igb_setup_all_rx_resources(adapter);
1255         }
1256
1257         igb_assign_vector(adapter->q_vector[0], 0);
1258
1259         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1260                 err = request_irq(pdev->irq, igb_intr_msi, 0,
1261                                   netdev->name, adapter);
1262                 if (!err)
1263                         goto request_done;
1264
1265                 /* fall back to legacy interrupts */
1266                 igb_reset_interrupt_capability(adapter);
1267                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1268         }
1269
1270         err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1271                           netdev->name, adapter);
1272
1273         if (err)
1274                 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1275                         err);
1276
1277 request_done:
1278         return err;
1279 }
1280
1281 static void igb_free_irq(struct igb_adapter *adapter)
1282 {
1283         if (adapter->msix_entries) {
1284                 int vector = 0, i;
1285
1286                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1287
1288                 for (i = 0; i < adapter->num_q_vectors; i++)
1289                         free_irq(adapter->msix_entries[vector++].vector,
1290                                  adapter->q_vector[i]);
1291         } else {
1292                 free_irq(adapter->pdev->irq, adapter);
1293         }
1294 }
1295
1296 /**
1297  * igb_irq_disable - Mask off interrupt generation on the NIC
1298  * @adapter: board private structure
1299  **/
1300 static void igb_irq_disable(struct igb_adapter *adapter)
1301 {
1302         struct e1000_hw *hw = &adapter->hw;
1303
1304         /*
1305          * we need to be careful when disabling interrupts.  The VFs are also
1306          * mapped into these registers and so clearing the bits can cause
1307          * issues on the VF drivers so we only need to clear what we set
1308          */
1309         if (adapter->msix_entries) {
1310                 u32 regval = rd32(E1000_EIAM);
1311                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1312                 wr32(E1000_EIMC, adapter->eims_enable_mask);
1313                 regval = rd32(E1000_EIAC);
1314                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1315         }
1316
1317         wr32(E1000_IAM, 0);
1318         wr32(E1000_IMC, ~0);
1319         wrfl();
1320         if (adapter->msix_entries) {
1321                 int i;
1322                 for (i = 0; i < adapter->num_q_vectors; i++)
1323                         synchronize_irq(adapter->msix_entries[i].vector);
1324         } else {
1325                 synchronize_irq(adapter->pdev->irq);
1326         }
1327 }
1328
1329 /**
1330  * igb_irq_enable - Enable default interrupt generation settings
1331  * @adapter: board private structure
1332  **/
1333 static void igb_irq_enable(struct igb_adapter *adapter)
1334 {
1335         struct e1000_hw *hw = &adapter->hw;
1336
1337         if (adapter->msix_entries) {
1338                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1339                 u32 regval = rd32(E1000_EIAC);
1340                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1341                 regval = rd32(E1000_EIAM);
1342                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1343                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1344                 if (adapter->vfs_allocated_count) {
1345                         wr32(E1000_MBVFIMR, 0xFF);
1346                         ims |= E1000_IMS_VMMB;
1347                 }
1348                 wr32(E1000_IMS, ims);
1349         } else {
1350                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1351                                 E1000_IMS_DRSTA);
1352                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1353                                 E1000_IMS_DRSTA);
1354         }
1355 }
1356
1357 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1358 {
1359         struct e1000_hw *hw = &adapter->hw;
1360         u16 vid = adapter->hw.mng_cookie.vlan_id;
1361         u16 old_vid = adapter->mng_vlan_id;
1362
1363         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1364                 /* add VID to filter table */
1365                 igb_vfta_set(hw, vid, true);
1366                 adapter->mng_vlan_id = vid;
1367         } else {
1368                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1369         }
1370
1371         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1372             (vid != old_vid) &&
1373             !test_bit(old_vid, adapter->active_vlans)) {
1374                 /* remove VID from filter table */
1375                 igb_vfta_set(hw, old_vid, false);
1376         }
1377 }
1378
1379 /**
1380  * igb_release_hw_control - release control of the h/w to f/w
1381  * @adapter: address of board private structure
1382  *
1383  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1384  * For ASF and Pass Through versions of f/w this means that the
1385  * driver is no longer loaded.
1386  *
1387  **/
1388 static void igb_release_hw_control(struct igb_adapter *adapter)
1389 {
1390         struct e1000_hw *hw = &adapter->hw;
1391         u32 ctrl_ext;
1392
1393         /* Let firmware take over control of h/w */
1394         ctrl_ext = rd32(E1000_CTRL_EXT);
1395         wr32(E1000_CTRL_EXT,
1396                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1397 }
1398
1399 /**
1400  * igb_get_hw_control - get control of the h/w from f/w
1401  * @adapter: address of board private structure
1402  *
1403  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1404  * For ASF and Pass Through versions of f/w this means that
1405  * the driver is loaded.
1406  *
1407  **/
1408 static void igb_get_hw_control(struct igb_adapter *adapter)
1409 {
1410         struct e1000_hw *hw = &adapter->hw;
1411         u32 ctrl_ext;
1412
1413         /* Let firmware know the driver has taken over */
1414         ctrl_ext = rd32(E1000_CTRL_EXT);
1415         wr32(E1000_CTRL_EXT,
1416                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1417 }
1418
1419 /**
1420  * igb_configure - configure the hardware for RX and TX
1421  * @adapter: private board structure
1422  **/
1423 static void igb_configure(struct igb_adapter *adapter)
1424 {
1425         struct net_device *netdev = adapter->netdev;
1426         int i;
1427
1428         igb_get_hw_control(adapter);
1429         igb_set_rx_mode(netdev);
1430
1431         igb_restore_vlan(adapter);
1432
1433         igb_setup_tctl(adapter);
1434         igb_setup_mrqc(adapter);
1435         igb_setup_rctl(adapter);
1436
1437         igb_configure_tx(adapter);
1438         igb_configure_rx(adapter);
1439
1440         igb_rx_fifo_flush_82575(&adapter->hw);
1441
1442         /* call igb_desc_unused which always leaves
1443          * at least 1 descriptor unused to make sure
1444          * next_to_use != next_to_clean */
1445         for (i = 0; i < adapter->num_rx_queues; i++) {
1446                 struct igb_ring *ring = adapter->rx_ring[i];
1447                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1448         }
1449 }
1450
1451 /**
1452  * igb_power_up_link - Power up the phy/serdes link
1453  * @adapter: address of board private structure
1454  **/
1455 void igb_power_up_link(struct igb_adapter *adapter)
1456 {
1457         igb_reset_phy(&adapter->hw);
1458
1459         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1460                 igb_power_up_phy_copper(&adapter->hw);
1461         else
1462                 igb_power_up_serdes_link_82575(&adapter->hw);
1463 }
1464
1465 /**
1466  * igb_power_down_link - Power down the phy/serdes link
1467  * @adapter: address of board private structure
1468  */
1469 static void igb_power_down_link(struct igb_adapter *adapter)
1470 {
1471         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1472                 igb_power_down_phy_copper_82575(&adapter->hw);
1473         else
1474                 igb_shutdown_serdes_link_82575(&adapter->hw);
1475 }
1476
1477 /**
1478  * igb_up - Open the interface and prepare it to handle traffic
1479  * @adapter: board private structure
1480  **/
1481 int igb_up(struct igb_adapter *adapter)
1482 {
1483         struct e1000_hw *hw = &adapter->hw;
1484         int i;
1485
1486         /* hardware has been reset, we need to reload some things */
1487         igb_configure(adapter);
1488
1489         clear_bit(__IGB_DOWN, &adapter->state);
1490
1491         for (i = 0; i < adapter->num_q_vectors; i++)
1492                 napi_enable(&(adapter->q_vector[i]->napi));
1493
1494         if (adapter->msix_entries)
1495                 igb_configure_msix(adapter);
1496         else
1497                 igb_assign_vector(adapter->q_vector[0], 0);
1498
1499         /* Clear any pending interrupts. */
1500         rd32(E1000_ICR);
1501         igb_irq_enable(adapter);
1502
1503         /* notify VFs that reset has been completed */
1504         if (adapter->vfs_allocated_count) {
1505                 u32 reg_data = rd32(E1000_CTRL_EXT);
1506                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1507                 wr32(E1000_CTRL_EXT, reg_data);
1508         }
1509
1510         netif_tx_start_all_queues(adapter->netdev);
1511
1512         /* start the watchdog. */
1513         hw->mac.get_link_status = 1;
1514         schedule_work(&adapter->watchdog_task);
1515
1516         return 0;
1517 }
1518
1519 void igb_down(struct igb_adapter *adapter)
1520 {
1521         struct net_device *netdev = adapter->netdev;
1522         struct e1000_hw *hw = &adapter->hw;
1523         u32 tctl, rctl;
1524         int i;
1525
1526         /* signal that we're down so the interrupt handler does not
1527          * reschedule our watchdog timer */
1528         set_bit(__IGB_DOWN, &adapter->state);
1529
1530         /* disable receives in the hardware */
1531         rctl = rd32(E1000_RCTL);
1532         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1533         /* flush and sleep below */
1534
1535         netif_tx_stop_all_queues(netdev);
1536
1537         /* disable transmits in the hardware */
1538         tctl = rd32(E1000_TCTL);
1539         tctl &= ~E1000_TCTL_EN;
1540         wr32(E1000_TCTL, tctl);
1541         /* flush both disables and wait for them to finish */
1542         wrfl();
1543         msleep(10);
1544
1545         for (i = 0; i < adapter->num_q_vectors; i++)
1546                 napi_disable(&(adapter->q_vector[i]->napi));
1547
1548         igb_irq_disable(adapter);
1549
1550         del_timer_sync(&adapter->watchdog_timer);
1551         del_timer_sync(&adapter->phy_info_timer);
1552
1553         netif_carrier_off(netdev);
1554
1555         /* record the stats before reset*/
1556         spin_lock(&adapter->stats64_lock);
1557         igb_update_stats(adapter, &adapter->stats64);
1558         spin_unlock(&adapter->stats64_lock);
1559
1560         adapter->link_speed = 0;
1561         adapter->link_duplex = 0;
1562
1563         if (!pci_channel_offline(adapter->pdev))
1564                 igb_reset(adapter);
1565         igb_clean_all_tx_rings(adapter);
1566         igb_clean_all_rx_rings(adapter);
1567 #ifdef CONFIG_IGB_DCA
1568
1569         /* since we reset the hardware DCA settings were cleared */
1570         igb_setup_dca(adapter);
1571 #endif
1572 }
1573
1574 void igb_reinit_locked(struct igb_adapter *adapter)
1575 {
1576         WARN_ON(in_interrupt());
1577         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1578                 msleep(1);
1579         igb_down(adapter);
1580         igb_up(adapter);
1581         clear_bit(__IGB_RESETTING, &adapter->state);
1582 }
1583
1584 void igb_reset(struct igb_adapter *adapter)
1585 {
1586         struct pci_dev *pdev = adapter->pdev;
1587         struct e1000_hw *hw = &adapter->hw;
1588         struct e1000_mac_info *mac = &hw->mac;
1589         struct e1000_fc_info *fc = &hw->fc;
1590         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1591         u16 hwm;
1592
1593         /* Repartition Pba for greater than 9k mtu
1594          * To take effect CTRL.RST is required.
1595          */
1596         switch (mac->type) {
1597         case e1000_i350:
1598         case e1000_82580:
1599                 pba = rd32(E1000_RXPBS);
1600                 pba = igb_rxpbs_adjust_82580(pba);
1601                 break;
1602         case e1000_82576:
1603                 pba = rd32(E1000_RXPBS);
1604                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1605                 break;
1606         case e1000_82575:
1607         case e1000_i210:
1608         case e1000_i211:
1609         default:
1610                 pba = E1000_PBA_34K;
1611                 break;
1612         }
1613
1614         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1615             (mac->type < e1000_82576)) {
1616                 /* adjust PBA for jumbo frames */
1617                 wr32(E1000_PBA, pba);
1618
1619                 /* To maintain wire speed transmits, the Tx FIFO should be
1620                  * large enough to accommodate two full transmit packets,
1621                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1622                  * the Rx FIFO should be large enough to accommodate at least
1623                  * one full receive packet and is similarly rounded up and
1624                  * expressed in KB. */
1625                 pba = rd32(E1000_PBA);
1626                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1627                 tx_space = pba >> 16;
1628                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1629                 pba &= 0xffff;
1630                 /* the tx fifo also stores 16 bytes of information about the tx
1631                  * but don't include ethernet FCS because hardware appends it */
1632                 min_tx_space = (adapter->max_frame_size +
1633                                 sizeof(union e1000_adv_tx_desc) -
1634                                 ETH_FCS_LEN) * 2;
1635                 min_tx_space = ALIGN(min_tx_space, 1024);
1636                 min_tx_space >>= 10;
1637                 /* software strips receive CRC, so leave room for it */
1638                 min_rx_space = adapter->max_frame_size;
1639                 min_rx_space = ALIGN(min_rx_space, 1024);
1640                 min_rx_space >>= 10;
1641
1642                 /* If current Tx allocation is less than the min Tx FIFO size,
1643                  * and the min Tx FIFO size is less than the current Rx FIFO
1644                  * allocation, take space away from current Rx allocation */
1645                 if (tx_space < min_tx_space &&
1646                     ((min_tx_space - tx_space) < pba)) {
1647                         pba = pba - (min_tx_space - tx_space);
1648
1649                         /* if short on rx space, rx wins and must trump tx
1650                          * adjustment */
1651                         if (pba < min_rx_space)
1652                                 pba = min_rx_space;
1653                 }
1654                 wr32(E1000_PBA, pba);
1655         }
1656
1657         /* flow control settings */
1658         /* The high water mark must be low enough to fit one full frame
1659          * (or the size used for early receive) above it in the Rx FIFO.
1660          * Set it to the lower of:
1661          * - 90% of the Rx FIFO size, or
1662          * - the full Rx FIFO size minus one full frame */
1663         hwm = min(((pba << 10) * 9 / 10),
1664                         ((pba << 10) - 2 * adapter->max_frame_size));
1665
1666         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1667         fc->low_water = fc->high_water - 16;
1668         fc->pause_time = 0xFFFF;
1669         fc->send_xon = 1;
1670         fc->current_mode = fc->requested_mode;
1671
1672         /* disable receive for all VFs and wait one second */
1673         if (adapter->vfs_allocated_count) {
1674                 int i;
1675                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1676                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1677
1678                 /* ping all the active vfs to let them know we are going down */
1679                 igb_ping_all_vfs(adapter);
1680
1681                 /* disable transmits and receives */
1682                 wr32(E1000_VFRE, 0);
1683                 wr32(E1000_VFTE, 0);
1684         }
1685
1686         /* Allow time for pending master requests to run */
1687         hw->mac.ops.reset_hw(hw);
1688         wr32(E1000_WUC, 0);
1689
1690         if (hw->mac.ops.init_hw(hw))
1691                 dev_err(&pdev->dev, "Hardware Error\n");
1692
1693         /*
1694          * Flow control settings reset on hardware reset, so guarantee flow
1695          * control is off when forcing speed.
1696          */
1697         if (!hw->mac.autoneg)
1698                 igb_force_mac_fc(hw);
1699
1700         igb_init_dmac(adapter, pba);
1701         if (!netif_running(adapter->netdev))
1702                 igb_power_down_link(adapter);
1703
1704         igb_update_mng_vlan(adapter);
1705
1706         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1707         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1708
1709 #ifdef CONFIG_IGB_PTP
1710         /* Re-enable PTP, where applicable. */
1711         igb_ptp_reset(adapter);
1712 #endif /* CONFIG_IGB_PTP */
1713
1714         igb_get_phy_info(hw);
1715 }
1716
1717 static netdev_features_t igb_fix_features(struct net_device *netdev,
1718         netdev_features_t features)
1719 {
1720         /*
1721          * Since there is no support for separate rx/tx vlan accel
1722          * enable/disable make sure tx flag is always in same state as rx.
1723          */
1724         if (features & NETIF_F_HW_VLAN_RX)
1725                 features |= NETIF_F_HW_VLAN_TX;
1726         else
1727                 features &= ~NETIF_F_HW_VLAN_TX;
1728
1729         return features;
1730 }
1731
1732 static int igb_set_features(struct net_device *netdev,
1733         netdev_features_t features)
1734 {
1735         netdev_features_t changed = netdev->features ^ features;
1736         struct igb_adapter *adapter = netdev_priv(netdev);
1737
1738         if (changed & NETIF_F_HW_VLAN_RX)
1739                 igb_vlan_mode(netdev, features);
1740
1741         if (!(changed & NETIF_F_RXALL))
1742                 return 0;
1743
1744         netdev->features = features;
1745
1746         if (netif_running(netdev))
1747                 igb_reinit_locked(adapter);
1748         else
1749                 igb_reset(adapter);
1750
1751         return 0;
1752 }
1753
1754 static const struct net_device_ops igb_netdev_ops = {
1755         .ndo_open               = igb_open,
1756         .ndo_stop               = igb_close,
1757         .ndo_start_xmit         = igb_xmit_frame,
1758         .ndo_get_stats64        = igb_get_stats64,
1759         .ndo_set_rx_mode        = igb_set_rx_mode,
1760         .ndo_set_mac_address    = igb_set_mac,
1761         .ndo_change_mtu         = igb_change_mtu,
1762         .ndo_do_ioctl           = igb_ioctl,
1763         .ndo_tx_timeout         = igb_tx_timeout,
1764         .ndo_validate_addr      = eth_validate_addr,
1765         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1766         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1767         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1768         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1769         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1770         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1771 #ifdef CONFIG_NET_POLL_CONTROLLER
1772         .ndo_poll_controller    = igb_netpoll,
1773 #endif
1774         .ndo_fix_features       = igb_fix_features,
1775         .ndo_set_features       = igb_set_features,
1776 };
1777
1778 /**
1779  * igb_set_fw_version - Configure version string for ethtool
1780  * @adapter: adapter struct
1781  *
1782  **/
1783 void igb_set_fw_version(struct igb_adapter *adapter)
1784 {
1785         struct e1000_hw *hw = &adapter->hw;
1786         u16 eeprom_verh, eeprom_verl, comb_verh, comb_verl, comb_offset;
1787         u16 major, build, patch, fw_version;
1788         u32 etrack_id;
1789
1790         hw->nvm.ops.read(hw, 5, 1, &fw_version);
1791         if (adapter->hw.mac.type != e1000_i211) {
1792                 hw->nvm.ops.read(hw, NVM_ETRACK_WORD, 1, &eeprom_verh);
1793                 hw->nvm.ops.read(hw, (NVM_ETRACK_WORD + 1), 1, &eeprom_verl);
1794                 etrack_id = (eeprom_verh << IGB_ETRACK_SHIFT) | eeprom_verl;
1795
1796                 /* combo image version needs to be found */
1797                 hw->nvm.ops.read(hw, NVM_COMB_VER_PTR, 1, &comb_offset);
1798                 if ((comb_offset != 0x0) &&
1799                     (comb_offset != IGB_NVM_VER_INVALID)) {
1800                         hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset
1801                                          + 1), 1, &comb_verh);
1802                         hw->nvm.ops.read(hw, (NVM_COMB_VER_OFF + comb_offset),
1803                                          1, &comb_verl);
1804
1805                         /* Only display Option Rom if it exists and is valid */
1806                         if ((comb_verh && comb_verl) &&
1807                             ((comb_verh != IGB_NVM_VER_INVALID) &&
1808                              (comb_verl != IGB_NVM_VER_INVALID))) {
1809                                 major = comb_verl >> IGB_COMB_VER_SHFT;
1810                                 build = (comb_verl << IGB_COMB_VER_SHFT) |
1811                                         (comb_verh >> IGB_COMB_VER_SHFT);
1812                                 patch = comb_verh & IGB_COMB_VER_MASK;
1813                                 snprintf(adapter->fw_version,
1814                                          sizeof(adapter->fw_version),
1815                                          "%d.%d%d, 0x%08x, %d.%d.%d",
1816                                          (fw_version & IGB_MAJOR_MASK) >>
1817                                          IGB_MAJOR_SHIFT,
1818                                          (fw_version & IGB_MINOR_MASK) >>
1819                                          IGB_MINOR_SHIFT,
1820                                          (fw_version & IGB_BUILD_MASK),
1821                                          etrack_id, major, build, patch);
1822                                 goto out;
1823                         }
1824                 }
1825                 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1826                          "%d.%d%d, 0x%08x",
1827                          (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1828                          (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1829                          (fw_version & IGB_BUILD_MASK), etrack_id);
1830         } else {
1831                 snprintf(adapter->fw_version, sizeof(adapter->fw_version),
1832                          "%d.%d%d",
1833                          (fw_version & IGB_MAJOR_MASK) >> IGB_MAJOR_SHIFT,
1834                          (fw_version & IGB_MINOR_MASK) >> IGB_MINOR_SHIFT,
1835                          (fw_version & IGB_BUILD_MASK));
1836         }
1837 out:
1838         return;
1839 }
1840
1841 /**
1842  * igb_probe - Device Initialization Routine
1843  * @pdev: PCI device information struct
1844  * @ent: entry in igb_pci_tbl
1845  *
1846  * Returns 0 on success, negative on failure
1847  *
1848  * igb_probe initializes an adapter identified by a pci_dev structure.
1849  * The OS initialization, configuring of the adapter private structure,
1850  * and a hardware reset occur.
1851  **/
1852 static int __devinit igb_probe(struct pci_dev *pdev,
1853                                const struct pci_device_id *ent)
1854 {
1855         struct net_device *netdev;
1856         struct igb_adapter *adapter;
1857         struct e1000_hw *hw;
1858         u16 eeprom_data = 0;
1859         s32 ret_val;
1860         static int global_quad_port_a; /* global quad port a indication */
1861         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1862         unsigned long mmio_start, mmio_len;
1863         int err, pci_using_dac;
1864         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1865         u8 part_str[E1000_PBANUM_LENGTH];
1866
1867         /* Catch broken hardware that put the wrong VF device ID in
1868          * the PCIe SR-IOV capability.
1869          */
1870         if (pdev->is_virtfn) {
1871                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1872                         pci_name(pdev), pdev->vendor, pdev->device);
1873                 return -EINVAL;
1874         }
1875
1876         err = pci_enable_device_mem(pdev);
1877         if (err)
1878                 return err;
1879
1880         pci_using_dac = 0;
1881         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1882         if (!err) {
1883                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1884                 if (!err)
1885                         pci_using_dac = 1;
1886         } else {
1887                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1888                 if (err) {
1889                         err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1890                         if (err) {
1891                                 dev_err(&pdev->dev, "No usable DMA "
1892                                         "configuration, aborting\n");
1893                                 goto err_dma;
1894                         }
1895                 }
1896         }
1897
1898         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1899                                            IORESOURCE_MEM),
1900                                            igb_driver_name);
1901         if (err)
1902                 goto err_pci_reg;
1903
1904         pci_enable_pcie_error_reporting(pdev);
1905
1906         pci_set_master(pdev);
1907         pci_save_state(pdev);
1908
1909         err = -ENOMEM;
1910         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1911                                    IGB_MAX_TX_QUEUES);
1912         if (!netdev)
1913                 goto err_alloc_etherdev;
1914
1915         SET_NETDEV_DEV(netdev, &pdev->dev);
1916
1917         pci_set_drvdata(pdev, netdev);
1918         adapter = netdev_priv(netdev);
1919         adapter->netdev = netdev;
1920         adapter->pdev = pdev;
1921         hw = &adapter->hw;
1922         hw->back = adapter;
1923         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
1924
1925         mmio_start = pci_resource_start(pdev, 0);
1926         mmio_len = pci_resource_len(pdev, 0);
1927
1928         err = -EIO;
1929         hw->hw_addr = ioremap(mmio_start, mmio_len);
1930         if (!hw->hw_addr)
1931                 goto err_ioremap;
1932
1933         netdev->netdev_ops = &igb_netdev_ops;
1934         igb_set_ethtool_ops(netdev);
1935         netdev->watchdog_timeo = 5 * HZ;
1936
1937         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1938
1939         netdev->mem_start = mmio_start;
1940         netdev->mem_end = mmio_start + mmio_len;
1941
1942         /* PCI config space info */
1943         hw->vendor_id = pdev->vendor;
1944         hw->device_id = pdev->device;
1945         hw->revision_id = pdev->revision;
1946         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1947         hw->subsystem_device_id = pdev->subsystem_device;
1948
1949         /* Copy the default MAC, PHY and NVM function pointers */
1950         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1951         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1952         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1953         /* Initialize skew-specific constants */
1954         err = ei->get_invariants(hw);
1955         if (err)
1956                 goto err_sw_init;
1957
1958         /* setup the private structure */
1959         err = igb_sw_init(adapter);
1960         if (err)
1961                 goto err_sw_init;
1962
1963         igb_get_bus_info_pcie(hw);
1964
1965         hw->phy.autoneg_wait_to_complete = false;
1966
1967         /* Copper options */
1968         if (hw->phy.media_type == e1000_media_type_copper) {
1969                 hw->phy.mdix = AUTO_ALL_MODES;
1970                 hw->phy.disable_polarity_correction = false;
1971                 hw->phy.ms_type = e1000_ms_hw_default;
1972         }
1973
1974         if (igb_check_reset_block(hw))
1975                 dev_info(&pdev->dev,
1976                         "PHY reset is blocked due to SOL/IDER session.\n");
1977
1978         /*
1979          * features is initialized to 0 in allocation, it might have bits
1980          * set by igb_sw_init so we should use an or instead of an
1981          * assignment.
1982          */
1983         netdev->features |= NETIF_F_SG |
1984                             NETIF_F_IP_CSUM |
1985                             NETIF_F_IPV6_CSUM |
1986                             NETIF_F_TSO |
1987                             NETIF_F_TSO6 |
1988                             NETIF_F_RXHASH |
1989                             NETIF_F_RXCSUM |
1990                             NETIF_F_HW_VLAN_RX |
1991                             NETIF_F_HW_VLAN_TX;
1992
1993         /* copy netdev features into list of user selectable features */
1994         netdev->hw_features |= netdev->features;
1995         netdev->hw_features |= NETIF_F_RXALL;
1996
1997         /* set this bit last since it cannot be part of hw_features */
1998         netdev->features |= NETIF_F_HW_VLAN_FILTER;
1999
2000         netdev->vlan_features |= NETIF_F_TSO |
2001                                  NETIF_F_TSO6 |
2002                                  NETIF_F_IP_CSUM |
2003                                  NETIF_F_IPV6_CSUM |
2004                                  NETIF_F_SG;
2005
2006         netdev->priv_flags |= IFF_SUPP_NOFCS;
2007
2008         if (pci_using_dac) {
2009                 netdev->features |= NETIF_F_HIGHDMA;
2010                 netdev->vlan_features |= NETIF_F_HIGHDMA;
2011         }
2012
2013         if (hw->mac.type >= e1000_82576) {
2014                 netdev->hw_features |= NETIF_F_SCTP_CSUM;
2015                 netdev->features |= NETIF_F_SCTP_CSUM;
2016         }
2017
2018         netdev->priv_flags |= IFF_UNICAST_FLT;
2019
2020         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2021
2022         /* before reading the NVM, reset the controller to put the device in a
2023          * known good starting state */
2024         hw->mac.ops.reset_hw(hw);
2025
2026         /*
2027          * make sure the NVM is good , i211 parts have special NVM that
2028          * doesn't contain a checksum
2029          */
2030         if (hw->mac.type != e1000_i211) {
2031                 if (hw->nvm.ops.validate(hw) < 0) {
2032                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2033                         err = -EIO;
2034                         goto err_eeprom;
2035                 }
2036         }
2037
2038         /* copy the MAC address out of the NVM */
2039         if (hw->mac.ops.read_mac_addr(hw))
2040                 dev_err(&pdev->dev, "NVM Read Error\n");
2041
2042         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2043         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2044
2045         if (!is_valid_ether_addr(netdev->perm_addr)) {
2046                 dev_err(&pdev->dev, "Invalid MAC Address\n");
2047                 err = -EIO;
2048                 goto err_eeprom;
2049         }
2050
2051         /* get firmware version for ethtool -i */
2052         igb_set_fw_version(adapter);
2053
2054         setup_timer(&adapter->watchdog_timer, igb_watchdog,
2055                     (unsigned long) adapter);
2056         setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2057                     (unsigned long) adapter);
2058
2059         INIT_WORK(&adapter->reset_task, igb_reset_task);
2060         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2061
2062         /* Initialize link properties that are user-changeable */
2063         adapter->fc_autoneg = true;
2064         hw->mac.autoneg = true;
2065         hw->phy.autoneg_advertised = 0x2f;
2066
2067         hw->fc.requested_mode = e1000_fc_default;
2068         hw->fc.current_mode = e1000_fc_default;
2069
2070         igb_validate_mdi_setting(hw);
2071
2072         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2073          * enable the ACPI Magic Packet filter
2074          */
2075
2076         if (hw->bus.func == 0)
2077                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2078         else if (hw->mac.type >= e1000_82580)
2079                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2080                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2081                                  &eeprom_data);
2082         else if (hw->bus.func == 1)
2083                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2084
2085         if (eeprom_data & eeprom_apme_mask)
2086                 adapter->eeprom_wol |= E1000_WUFC_MAG;
2087
2088         /* now that we have the eeprom settings, apply the special cases where
2089          * the eeprom may be wrong or the board simply won't support wake on
2090          * lan on a particular port */
2091         switch (pdev->device) {
2092         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2093                 adapter->eeprom_wol = 0;
2094                 break;
2095         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2096         case E1000_DEV_ID_82576_FIBER:
2097         case E1000_DEV_ID_82576_SERDES:
2098                 /* Wake events only supported on port A for dual fiber
2099                  * regardless of eeprom setting */
2100                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2101                         adapter->eeprom_wol = 0;
2102                 break;
2103         case E1000_DEV_ID_82576_QUAD_COPPER:
2104         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2105                 /* if quad port adapter, disable WoL on all but port A */
2106                 if (global_quad_port_a != 0)
2107                         adapter->eeprom_wol = 0;
2108                 else
2109                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2110                 /* Reset for multiple quad port adapters */
2111                 if (++global_quad_port_a == 4)
2112                         global_quad_port_a = 0;
2113                 break;
2114         }
2115
2116         /* initialize the wol settings based on the eeprom settings */
2117         adapter->wol = adapter->eeprom_wol;
2118         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2119
2120         /* reset the hardware with the new settings */
2121         igb_reset(adapter);
2122
2123         /* let the f/w know that the h/w is now under the control of the
2124          * driver. */
2125         igb_get_hw_control(adapter);
2126
2127         strcpy(netdev->name, "eth%d");
2128         err = register_netdev(netdev);
2129         if (err)
2130                 goto err_register;
2131
2132         /* carrier off reporting is important to ethtool even BEFORE open */
2133         netif_carrier_off(netdev);
2134
2135 #ifdef CONFIG_IGB_DCA
2136         if (dca_add_requester(&pdev->dev) == 0) {
2137                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2138                 dev_info(&pdev->dev, "DCA enabled\n");
2139                 igb_setup_dca(adapter);
2140         }
2141
2142 #endif
2143
2144 #ifdef CONFIG_IGB_PTP
2145         /* do hw tstamp init after resetting */
2146         igb_ptp_init(adapter);
2147 #endif /* CONFIG_IGB_PTP */
2148
2149         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2150         /* print bus type/speed/width info */
2151         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2152                  netdev->name,
2153                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2154                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2155                                                             "unknown"),
2156                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2157                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2158                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2159                    "unknown"),
2160                  netdev->dev_addr);
2161
2162         ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2163         if (ret_val)
2164                 strcpy(part_str, "Unknown");
2165         dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2166         dev_info(&pdev->dev,
2167                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2168                 adapter->msix_entries ? "MSI-X" :
2169                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2170                 adapter->num_rx_queues, adapter->num_tx_queues);
2171         switch (hw->mac.type) {
2172         case e1000_i350:
2173         case e1000_i210:
2174         case e1000_i211:
2175                 igb_set_eee_i350(hw);
2176                 break;
2177         default:
2178                 break;
2179         }
2180
2181         pm_runtime_put_noidle(&pdev->dev);
2182         return 0;
2183
2184 err_register:
2185         igb_release_hw_control(adapter);
2186 err_eeprom:
2187         if (!igb_check_reset_block(hw))
2188                 igb_reset_phy(hw);
2189
2190         if (hw->flash_address)
2191                 iounmap(hw->flash_address);
2192 err_sw_init:
2193         igb_clear_interrupt_scheme(adapter);
2194         iounmap(hw->hw_addr);
2195 err_ioremap:
2196         free_netdev(netdev);
2197 err_alloc_etherdev:
2198         pci_release_selected_regions(pdev,
2199                                      pci_select_bars(pdev, IORESOURCE_MEM));
2200 err_pci_reg:
2201 err_dma:
2202         pci_disable_device(pdev);
2203         return err;
2204 }
2205
2206 /**
2207  * igb_remove - Device Removal Routine
2208  * @pdev: PCI device information struct
2209  *
2210  * igb_remove is called by the PCI subsystem to alert the driver
2211  * that it should release a PCI device.  The could be caused by a
2212  * Hot-Plug event, or because the driver is going to be removed from
2213  * memory.
2214  **/
2215 static void __devexit igb_remove(struct pci_dev *pdev)
2216 {
2217         struct net_device *netdev = pci_get_drvdata(pdev);
2218         struct igb_adapter *adapter = netdev_priv(netdev);
2219         struct e1000_hw *hw = &adapter->hw;
2220
2221         pm_runtime_get_noresume(&pdev->dev);
2222 #ifdef CONFIG_IGB_PTP
2223         igb_ptp_stop(adapter);
2224 #endif /* CONFIG_IGB_PTP */
2225
2226         /*
2227          * The watchdog timer may be rescheduled, so explicitly
2228          * disable watchdog from being rescheduled.
2229          */
2230         set_bit(__IGB_DOWN, &adapter->state);
2231         del_timer_sync(&adapter->watchdog_timer);
2232         del_timer_sync(&adapter->phy_info_timer);
2233
2234         cancel_work_sync(&adapter->reset_task);
2235         cancel_work_sync(&adapter->watchdog_task);
2236
2237 #ifdef CONFIG_IGB_DCA
2238         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2239                 dev_info(&pdev->dev, "DCA disabled\n");
2240                 dca_remove_requester(&pdev->dev);
2241                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2242                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2243         }
2244 #endif
2245
2246         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2247          * would have already happened in close and is redundant. */
2248         igb_release_hw_control(adapter);
2249
2250         unregister_netdev(netdev);
2251
2252         igb_clear_interrupt_scheme(adapter);
2253
2254 #ifdef CONFIG_PCI_IOV
2255         /* reclaim resources allocated to VFs */
2256         if (adapter->vf_data) {
2257                 /* disable iov and allow time for transactions to clear */
2258                 if (igb_vfs_are_assigned(adapter)) {
2259                         dev_info(&pdev->dev, "Unloading driver while VFs are assigned - VFs will not be deallocated\n");
2260                 } else {
2261                         pci_disable_sriov(pdev);
2262                         msleep(500);
2263                 }
2264
2265                 kfree(adapter->vf_data);
2266                 adapter->vf_data = NULL;
2267                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2268                 wrfl();
2269                 msleep(100);
2270                 dev_info(&pdev->dev, "IOV Disabled\n");
2271         }
2272 #endif
2273
2274         iounmap(hw->hw_addr);
2275         if (hw->flash_address)
2276                 iounmap(hw->flash_address);
2277         pci_release_selected_regions(pdev,
2278                                      pci_select_bars(pdev, IORESOURCE_MEM));
2279
2280         kfree(adapter->shadow_vfta);
2281         free_netdev(netdev);
2282
2283         pci_disable_pcie_error_reporting(pdev);
2284
2285         pci_disable_device(pdev);
2286 }
2287
2288 /**
2289  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2290  * @adapter: board private structure to initialize
2291  *
2292  * This function initializes the vf specific data storage and then attempts to
2293  * allocate the VFs.  The reason for ordering it this way is because it is much
2294  * mor expensive time wise to disable SR-IOV than it is to allocate and free
2295  * the memory for the VFs.
2296  **/
2297 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2298 {
2299 #ifdef CONFIG_PCI_IOV
2300         struct pci_dev *pdev = adapter->pdev;
2301         struct e1000_hw *hw = &adapter->hw;
2302         int old_vfs = pci_num_vf(adapter->pdev);
2303         int i;
2304
2305         /* Virtualization features not supported on i210 family. */
2306         if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2307                 return;
2308
2309         if (old_vfs) {
2310                 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2311                          "max_vfs setting of %d\n", old_vfs, max_vfs);
2312                 adapter->vfs_allocated_count = old_vfs;
2313         }
2314
2315         if (!adapter->vfs_allocated_count)
2316                 return;
2317
2318         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2319                                 sizeof(struct vf_data_storage), GFP_KERNEL);
2320
2321         /* if allocation failed then we do not support SR-IOV */
2322         if (!adapter->vf_data) {
2323                 adapter->vfs_allocated_count = 0;
2324                 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2325                         "Data Storage\n");
2326                 goto out;
2327         }
2328
2329         if (!old_vfs) {
2330                 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2331                         goto err_out;
2332         }
2333         dev_info(&pdev->dev, "%d VFs allocated\n",
2334                  adapter->vfs_allocated_count);
2335         for (i = 0; i < adapter->vfs_allocated_count; i++)
2336                 igb_vf_configure(adapter, i);
2337
2338         /* DMA Coalescing is not supported in IOV mode. */
2339         adapter->flags &= ~IGB_FLAG_DMAC;
2340         goto out;
2341 err_out:
2342         kfree(adapter->vf_data);
2343         adapter->vf_data = NULL;
2344         adapter->vfs_allocated_count = 0;
2345 out:
2346         return;
2347 #endif /* CONFIG_PCI_IOV */
2348 }
2349
2350 /**
2351  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2352  * @adapter: board private structure to initialize
2353  *
2354  * igb_sw_init initializes the Adapter private data structure.
2355  * Fields are initialized based on PCI device information and
2356  * OS network device settings (MTU size).
2357  **/
2358 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2359 {
2360         struct e1000_hw *hw = &adapter->hw;
2361         struct net_device *netdev = adapter->netdev;
2362         struct pci_dev *pdev = adapter->pdev;
2363         u32 max_rss_queues;
2364
2365         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2366
2367         /* set default ring sizes */
2368         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2369         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2370
2371         /* set default ITR values */
2372         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2373         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2374
2375         /* set default work limits */
2376         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2377
2378         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2379                                   VLAN_HLEN;
2380         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2381
2382         spin_lock_init(&adapter->stats64_lock);
2383 #ifdef CONFIG_PCI_IOV
2384         switch (hw->mac.type) {
2385         case e1000_82576:
2386         case e1000_i350:
2387                 if (max_vfs > 7) {
2388                         dev_warn(&pdev->dev,
2389                                  "Maximum of 7 VFs per PF, using max\n");
2390                         adapter->vfs_allocated_count = 7;
2391                 } else
2392                         adapter->vfs_allocated_count = max_vfs;
2393                 break;
2394         default:
2395                 break;
2396         }
2397 #endif /* CONFIG_PCI_IOV */
2398
2399         /* Determine the maximum number of RSS queues supported. */
2400         switch (hw->mac.type) {
2401         case e1000_i211:
2402                 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
2403                 break;
2404         case e1000_82575:
2405         case e1000_i210:
2406                 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
2407                 break;
2408         case e1000_i350:
2409                 /* I350 cannot do RSS and SR-IOV at the same time */
2410                 if (!!adapter->vfs_allocated_count) {
2411                         max_rss_queues = 1;
2412                         break;
2413                 }
2414                 /* fall through */
2415         case e1000_82576:
2416                 if (!!adapter->vfs_allocated_count) {
2417                         max_rss_queues = 2;
2418                         break;
2419                 }
2420                 /* fall through */
2421         case e1000_82580:
2422         default:
2423                 max_rss_queues = IGB_MAX_RX_QUEUES;
2424                 break;
2425         }
2426
2427         adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
2428
2429         /* Determine if we need to pair queues. */
2430         switch (hw->mac.type) {
2431         case e1000_82575:
2432         case e1000_i211:
2433                 /* Device supports enough interrupts without queue pairing. */
2434                 break;
2435         case e1000_82576:
2436                 /*
2437                  * If VFs are going to be allocated with RSS queues then we
2438                  * should pair the queues in order to conserve interrupts due
2439                  * to limited supply.
2440                  */
2441                 if ((adapter->rss_queues > 1) &&
2442                     (adapter->vfs_allocated_count > 6))
2443                         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2444                 /* fall through */
2445         case e1000_82580:
2446         case e1000_i350:
2447         case e1000_i210:
2448         default:
2449                 /*
2450                  * If rss_queues > half of max_rss_queues, pair the queues in
2451                  * order to conserve interrupts due to limited supply.
2452                  */
2453                 if (adapter->rss_queues > (max_rss_queues / 2))
2454                         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2455                 break;
2456         }
2457
2458         /* Setup and initialize a copy of the hw vlan table array */
2459         adapter->shadow_vfta = kzalloc(sizeof(u32) *
2460                                 E1000_VLAN_FILTER_TBL_SIZE,
2461                                 GFP_ATOMIC);
2462
2463         /* This call may decrease the number of queues */
2464         if (igb_init_interrupt_scheme(adapter)) {
2465                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2466                 return -ENOMEM;
2467         }
2468
2469         igb_probe_vfs(adapter);
2470
2471         /* Explicitly disable IRQ since the NIC can be in any state. */
2472         igb_irq_disable(adapter);
2473
2474         if (hw->mac.type >= e1000_i350)
2475                 adapter->flags &= ~IGB_FLAG_DMAC;
2476
2477         set_bit(__IGB_DOWN, &adapter->state);
2478         return 0;
2479 }
2480
2481 /**
2482  * igb_open - Called when a network interface is made active
2483  * @netdev: network interface device structure
2484  *
2485  * Returns 0 on success, negative value on failure
2486  *
2487  * The open entry point is called when a network interface is made
2488  * active by the system (IFF_UP).  At this point all resources needed
2489  * for transmit and receive operations are allocated, the interrupt
2490  * handler is registered with the OS, the watchdog timer is started,
2491  * and the stack is notified that the interface is ready.
2492  **/
2493 static int __igb_open(struct net_device *netdev, bool resuming)
2494 {
2495         struct igb_adapter *adapter = netdev_priv(netdev);
2496         struct e1000_hw *hw = &adapter->hw;
2497         struct pci_dev *pdev = adapter->pdev;
2498         int err;
2499         int i;
2500
2501         /* disallow open during test */
2502         if (test_bit(__IGB_TESTING, &adapter->state)) {
2503                 WARN_ON(resuming);
2504                 return -EBUSY;
2505         }
2506
2507         if (!resuming)
2508                 pm_runtime_get_sync(&pdev->dev);
2509
2510         netif_carrier_off(netdev);
2511
2512         /* allocate transmit descriptors */
2513         err = igb_setup_all_tx_resources(adapter);
2514         if (err)
2515                 goto err_setup_tx;
2516
2517         /* allocate receive descriptors */
2518         err = igb_setup_all_rx_resources(adapter);
2519         if (err)
2520                 goto err_setup_rx;
2521
2522         igb_power_up_link(adapter);
2523
2524         /* before we allocate an interrupt, we must be ready to handle it.
2525          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2526          * as soon as we call pci_request_irq, so we have to setup our
2527          * clean_rx handler before we do so.  */
2528         igb_configure(adapter);
2529
2530         err = igb_request_irq(adapter);
2531         if (err)
2532                 goto err_req_irq;
2533
2534         /* From here on the code is the same as igb_up() */
2535         clear_bit(__IGB_DOWN, &adapter->state);
2536
2537         for (i = 0; i < adapter->num_q_vectors; i++)
2538                 napi_enable(&(adapter->q_vector[i]->napi));
2539
2540         /* Clear any pending interrupts. */
2541         rd32(E1000_ICR);
2542
2543         igb_irq_enable(adapter);
2544
2545         /* notify VFs that reset has been completed */
2546         if (adapter->vfs_allocated_count) {
2547                 u32 reg_data = rd32(E1000_CTRL_EXT);
2548                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2549                 wr32(E1000_CTRL_EXT, reg_data);
2550         }
2551
2552         netif_tx_start_all_queues(netdev);
2553
2554         if (!resuming)
2555                 pm_runtime_put(&pdev->dev);
2556
2557         /* start the watchdog. */
2558         hw->mac.get_link_status = 1;
2559         schedule_work(&adapter->watchdog_task);
2560
2561         return 0;
2562
2563 err_req_irq:
2564         igb_release_hw_control(adapter);
2565         igb_power_down_link(adapter);
2566         igb_free_all_rx_resources(adapter);
2567 err_setup_rx:
2568         igb_free_all_tx_resources(adapter);
2569 err_setup_tx:
2570         igb_reset(adapter);
2571         if (!resuming)
2572                 pm_runtime_put(&pdev->dev);
2573
2574         return err;
2575 }
2576
2577 static int igb_open(struct net_device *netdev)
2578 {
2579         return __igb_open(netdev, false);
2580 }
2581
2582 /**
2583  * igb_close - Disables a network interface
2584  * @netdev: network interface device structure
2585  *
2586  * Returns 0, this is not allowed to fail
2587  *
2588  * The close entry point is called when an interface is de-activated
2589  * by the OS.  The hardware is still under the driver's control, but
2590  * needs to be disabled.  A global MAC reset is issued to stop the
2591  * hardware, and all transmit and receive resources are freed.
2592  **/
2593 static int __igb_close(struct net_device *netdev, bool suspending)
2594 {
2595         struct igb_adapter *adapter = netdev_priv(netdev);
2596         struct pci_dev *pdev = adapter->pdev;
2597
2598         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2599
2600         if (!suspending)
2601                 pm_runtime_get_sync(&pdev->dev);
2602
2603         igb_down(adapter);
2604         igb_free_irq(adapter);
2605
2606         igb_free_all_tx_resources(adapter);
2607         igb_free_all_rx_resources(adapter);
2608
2609         if (!suspending)
2610                 pm_runtime_put_sync(&pdev->dev);
2611         return 0;
2612 }
2613
2614 static int igb_close(struct net_device *netdev)
2615 {
2616         return __igb_close(netdev, false);
2617 }
2618
2619 /**
2620  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2621  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2622  *
2623  * Return 0 on success, negative on failure
2624  **/
2625 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2626 {
2627         struct device *dev = tx_ring->dev;
2628         int size;
2629
2630         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2631
2632         tx_ring->tx_buffer_info = vzalloc(size);
2633         if (!tx_ring->tx_buffer_info)
2634                 goto err;
2635
2636         /* round up to nearest 4K */
2637         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2638         tx_ring->size = ALIGN(tx_ring->size, 4096);
2639
2640         tx_ring->desc = dma_alloc_coherent(dev,
2641                                            tx_ring->size,
2642                                            &tx_ring->dma,
2643                                            GFP_KERNEL);
2644         if (!tx_ring->desc)
2645                 goto err;
2646
2647         tx_ring->next_to_use = 0;
2648         tx_ring->next_to_clean = 0;
2649
2650         return 0;
2651
2652 err:
2653         vfree(tx_ring->tx_buffer_info);
2654         tx_ring->tx_buffer_info = NULL;
2655         dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2656         return -ENOMEM;
2657 }
2658
2659 /**
2660  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2661  *                                (Descriptors) for all queues
2662  * @adapter: board private structure
2663  *
2664  * Return 0 on success, negative on failure
2665  **/
2666 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2667 {
2668         struct pci_dev *pdev = adapter->pdev;
2669         int i, err = 0;
2670
2671         for (i = 0; i < adapter->num_tx_queues; i++) {
2672                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2673                 if (err) {
2674                         dev_err(&pdev->dev,
2675                                 "Allocation for Tx Queue %u failed\n", i);
2676                         for (i--; i >= 0; i--)
2677                                 igb_free_tx_resources(adapter->tx_ring[i]);
2678                         break;
2679                 }
2680         }
2681
2682         return err;
2683 }
2684
2685 /**
2686  * igb_setup_tctl - configure the transmit control registers
2687  * @adapter: Board private structure
2688  **/
2689 void igb_setup_tctl(struct igb_adapter *adapter)
2690 {
2691         struct e1000_hw *hw = &adapter->hw;
2692         u32 tctl;
2693
2694         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2695         wr32(E1000_TXDCTL(0), 0);
2696
2697         /* Program the Transmit Control Register */
2698         tctl = rd32(E1000_TCTL);
2699         tctl &= ~E1000_TCTL_CT;
2700         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2701                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2702
2703         igb_config_collision_dist(hw);
2704
2705         /* Enable transmits */
2706         tctl |= E1000_TCTL_EN;
2707
2708         wr32(E1000_TCTL, tctl);
2709 }
2710
2711 /**
2712  * igb_configure_tx_ring - Configure transmit ring after Reset
2713  * @adapter: board private structure
2714  * @ring: tx ring to configure
2715  *
2716  * Configure a transmit ring after a reset.
2717  **/
2718 void igb_configure_tx_ring(struct igb_adapter *adapter,
2719                            struct igb_ring *ring)
2720 {
2721         struct e1000_hw *hw = &adapter->hw;
2722         u32 txdctl = 0;
2723         u64 tdba = ring->dma;
2724         int reg_idx = ring->reg_idx;
2725
2726         /* disable the queue */
2727         wr32(E1000_TXDCTL(reg_idx), 0);
2728         wrfl();
2729         mdelay(10);
2730
2731         wr32(E1000_TDLEN(reg_idx),
2732                         ring->count * sizeof(union e1000_adv_tx_desc));
2733         wr32(E1000_TDBAL(reg_idx),
2734                         tdba & 0x00000000ffffffffULL);
2735         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2736
2737         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2738         wr32(E1000_TDH(reg_idx), 0);
2739         writel(0, ring->tail);
2740
2741         txdctl |= IGB_TX_PTHRESH;
2742         txdctl |= IGB_TX_HTHRESH << 8;
2743         txdctl |= IGB_TX_WTHRESH << 16;
2744
2745         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2746         wr32(E1000_TXDCTL(reg_idx), txdctl);
2747 }
2748
2749 /**
2750  * igb_configure_tx - Configure transmit Unit after Reset
2751  * @adapter: board private structure
2752  *
2753  * Configure the Tx unit of the MAC after a reset.
2754  **/
2755 static void igb_configure_tx(struct igb_adapter *adapter)
2756 {
2757         int i;
2758
2759         for (i = 0; i < adapter->num_tx_queues; i++)
2760                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2761 }
2762
2763 /**
2764  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2765  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2766  *
2767  * Returns 0 on success, negative on failure
2768  **/
2769 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2770 {
2771         struct device *dev = rx_ring->dev;
2772         int size;
2773
2774         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2775
2776         rx_ring->rx_buffer_info = vzalloc(size);
2777         if (!rx_ring->rx_buffer_info)
2778                 goto err;
2779
2780
2781         /* Round up to nearest 4K */
2782         rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
2783         rx_ring->size = ALIGN(rx_ring->size, 4096);
2784
2785         rx_ring->desc = dma_alloc_coherent(dev,
2786                                            rx_ring->size,
2787                                            &rx_ring->dma,
2788                                            GFP_KERNEL);
2789         if (!rx_ring->desc)
2790                 goto err;
2791
2792         rx_ring->next_to_clean = 0;
2793         rx_ring->next_to_use = 0;
2794
2795         return 0;
2796
2797 err:
2798         vfree(rx_ring->rx_buffer_info);
2799         rx_ring->rx_buffer_info = NULL;
2800         dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
2801         return -ENOMEM;
2802 }
2803
2804 /**
2805  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2806  *                                (Descriptors) for all queues
2807  * @adapter: board private structure
2808  *
2809  * Return 0 on success, negative on failure
2810  **/
2811 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2812 {
2813         struct pci_dev *pdev = adapter->pdev;
2814         int i, err = 0;
2815
2816         for (i = 0; i < adapter->num_rx_queues; i++) {
2817                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2818                 if (err) {
2819                         dev_err(&pdev->dev,
2820                                 "Allocation for Rx Queue %u failed\n", i);
2821                         for (i--; i >= 0; i--)
2822                                 igb_free_rx_resources(adapter->rx_ring[i]);
2823                         break;
2824                 }
2825         }
2826
2827         return err;
2828 }
2829
2830 /**
2831  * igb_setup_mrqc - configure the multiple receive queue control registers
2832  * @adapter: Board private structure
2833  **/
2834 static void igb_setup_mrqc(struct igb_adapter *adapter)
2835 {
2836         struct e1000_hw *hw = &adapter->hw;
2837         u32 mrqc, rxcsum;
2838         u32 j, num_rx_queues, shift = 0;
2839         static const u32 rsskey[10] = { 0xDA565A6D, 0xC20E5B25, 0x3D256741,
2840                                         0xB08FA343, 0xCB2BCAD0, 0xB4307BAE,
2841                                         0xA32DCB77, 0x0CF23080, 0x3BB7426A,
2842                                         0xFA01ACBE };
2843
2844         /* Fill out hash function seeds */
2845         for (j = 0; j < 10; j++)
2846                 wr32(E1000_RSSRK(j), rsskey[j]);
2847
2848         num_rx_queues = adapter->rss_queues;
2849
2850         switch (hw->mac.type) {
2851         case e1000_82575:
2852                 shift = 6;
2853                 break;
2854         case e1000_82576:
2855                 /* 82576 supports 2 RSS queues for SR-IOV */
2856                 if (adapter->vfs_allocated_count) {
2857                         shift = 3;
2858                         num_rx_queues = 2;
2859                 }
2860                 break;
2861         default:
2862                 break;
2863         }
2864
2865         /*
2866          * Populate the indirection table 4 entries at a time.  To do this
2867          * we are generating the results for n and n+2 and then interleaving
2868          * those with the results with n+1 and n+3.
2869          */
2870         for (j = 0; j < 32; j++) {
2871                 /* first pass generates n and n+2 */
2872                 u32 base = ((j * 0x00040004) + 0x00020000) * num_rx_queues;
2873                 u32 reta = (base & 0x07800780) >> (7 - shift);
2874
2875                 /* second pass generates n+1 and n+3 */
2876                 base += 0x00010001 * num_rx_queues;
2877                 reta |= (base & 0x07800780) << (1 + shift);
2878
2879                 wr32(E1000_RETA(j), reta);
2880         }
2881
2882         /*
2883          * Disable raw packet checksumming so that RSS hash is placed in
2884          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2885          * offloads as they are enabled by default
2886          */
2887         rxcsum = rd32(E1000_RXCSUM);
2888         rxcsum |= E1000_RXCSUM_PCSD;
2889
2890         if (adapter->hw.mac.type >= e1000_82576)
2891                 /* Enable Receive Checksum Offload for SCTP */
2892                 rxcsum |= E1000_RXCSUM_CRCOFL;
2893
2894         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2895         wr32(E1000_RXCSUM, rxcsum);
2896         /*
2897          * Generate RSS hash based on TCP port numbers and/or
2898          * IPv4/v6 src and dst addresses since UDP cannot be
2899          * hashed reliably due to IP fragmentation
2900          */
2901
2902         mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2903                E1000_MRQC_RSS_FIELD_IPV4_TCP |
2904                E1000_MRQC_RSS_FIELD_IPV6 |
2905                E1000_MRQC_RSS_FIELD_IPV6_TCP |
2906                E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2907
2908         /* If VMDq is enabled then we set the appropriate mode for that, else
2909          * we default to RSS so that an RSS hash is calculated per packet even
2910          * if we are only using one queue */
2911         if (adapter->vfs_allocated_count) {
2912                 if (hw->mac.type > e1000_82575) {
2913                         /* Set the default pool for the PF's first queue */
2914                         u32 vtctl = rd32(E1000_VT_CTL);
2915                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2916                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2917                         vtctl |= adapter->vfs_allocated_count <<
2918                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2919                         wr32(E1000_VT_CTL, vtctl);
2920                 }
2921                 if (adapter->rss_queues > 1)
2922                         mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2923                 else
2924                         mrqc |= E1000_MRQC_ENABLE_VMDQ;
2925         } else {
2926                 if (hw->mac.type != e1000_i211)
2927                         mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
2928         }
2929         igb_vmm_control(adapter);
2930
2931         wr32(E1000_MRQC, mrqc);
2932 }
2933
2934 /**
2935  * igb_setup_rctl - configure the receive control registers
2936  * @adapter: Board private structure
2937  **/
2938 void igb_setup_rctl(struct igb_adapter *adapter)
2939 {
2940         struct e1000_hw *hw = &adapter->hw;
2941         u32 rctl;
2942
2943         rctl = rd32(E1000_RCTL);
2944
2945         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2946         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2947
2948         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2949                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2950
2951         /*
2952          * enable stripping of CRC. It's unlikely this will break BMC
2953          * redirection as it did with e1000. Newer features require
2954          * that the HW strips the CRC.
2955          */
2956         rctl |= E1000_RCTL_SECRC;
2957
2958         /* disable store bad packets and clear size bits. */
2959         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2960
2961         /* enable LPE to prevent packets larger than max_frame_size */
2962         rctl |= E1000_RCTL_LPE;
2963
2964         /* disable queue 0 to prevent tail write w/o re-config */
2965         wr32(E1000_RXDCTL(0), 0);
2966
2967         /* Attention!!!  For SR-IOV PF driver operations you must enable
2968          * queue drop for all VF and PF queues to prevent head of line blocking
2969          * if an un-trusted VF does not provide descriptors to hardware.
2970          */
2971         if (adapter->vfs_allocated_count) {
2972                 /* set all queue drop enable bits */
2973                 wr32(E1000_QDE, ALL_QUEUES);
2974         }
2975
2976         /* This is useful for sniffing bad packets. */
2977         if (adapter->netdev->features & NETIF_F_RXALL) {
2978                 /* UPE and MPE will be handled by normal PROMISC logic
2979                  * in e1000e_set_rx_mode */
2980                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
2981                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
2982                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
2983
2984                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
2985                           E1000_RCTL_DPF | /* Allow filtered pause */
2986                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
2987                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
2988                  * and that breaks VLANs.
2989                  */
2990         }
2991
2992         wr32(E1000_RCTL, rctl);
2993 }
2994
2995 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2996                                    int vfn)
2997 {
2998         struct e1000_hw *hw = &adapter->hw;
2999         u32 vmolr;
3000
3001         /* if it isn't the PF check to see if VFs are enabled and
3002          * increase the size to support vlan tags */
3003         if (vfn < adapter->vfs_allocated_count &&
3004             adapter->vf_data[vfn].vlans_enabled)
3005                 size += VLAN_TAG_SIZE;
3006
3007         vmolr = rd32(E1000_VMOLR(vfn));
3008         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3009         vmolr |= size | E1000_VMOLR_LPE;
3010         wr32(E1000_VMOLR(vfn), vmolr);
3011
3012         return 0;
3013 }
3014
3015 /**
3016  * igb_rlpml_set - set maximum receive packet size
3017  * @adapter: board private structure
3018  *
3019  * Configure maximum receivable packet size.
3020  **/
3021 static void igb_rlpml_set(struct igb_adapter *adapter)
3022 {
3023         u32 max_frame_size = adapter->max_frame_size;
3024         struct e1000_hw *hw = &adapter->hw;
3025         u16 pf_id = adapter->vfs_allocated_count;
3026
3027         if (pf_id) {
3028                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3029                 /*
3030                  * If we're in VMDQ or SR-IOV mode, then set global RLPML
3031                  * to our max jumbo frame size, in case we need to enable
3032                  * jumbo frames on one of the rings later.
3033                  * This will not pass over-length frames into the default
3034                  * queue because it's gated by the VMOLR.RLPML.
3035                  */
3036                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3037         }
3038
3039         wr32(E1000_RLPML, max_frame_size);
3040 }
3041
3042 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3043                                  int vfn, bool aupe)
3044 {
3045         struct e1000_hw *hw = &adapter->hw;
3046         u32 vmolr;
3047
3048         /*
3049          * This register exists only on 82576 and newer so if we are older then
3050          * we should exit and do nothing
3051          */
3052         if (hw->mac.type < e1000_82576)
3053                 return;
3054
3055         vmolr = rd32(E1000_VMOLR(vfn));
3056         vmolr |= E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
3057         if (aupe)
3058                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
3059         else
3060                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3061
3062         /* clear all bits that might not be set */
3063         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3064
3065         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3066                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3067         /*
3068          * for VMDq only allow the VFs and pool 0 to accept broadcast and
3069          * multicast packets
3070          */
3071         if (vfn <= adapter->vfs_allocated_count)
3072                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
3073
3074         wr32(E1000_VMOLR(vfn), vmolr);
3075 }
3076
3077 /**
3078  * igb_configure_rx_ring - Configure a receive ring after Reset
3079  * @adapter: board private structure
3080  * @ring: receive ring to be configured
3081  *
3082  * Configure the Rx unit of the MAC after a reset.
3083  **/
3084 void igb_configure_rx_ring(struct igb_adapter *adapter,
3085                            struct igb_ring *ring)
3086 {
3087         struct e1000_hw *hw = &adapter->hw;
3088         u64 rdba = ring->dma;
3089         int reg_idx = ring->reg_idx;
3090         u32 srrctl = 0, rxdctl = 0;
3091
3092         /* disable the queue */
3093         wr32(E1000_RXDCTL(reg_idx), 0);
3094
3095         /* Set DMA base address registers */
3096         wr32(E1000_RDBAL(reg_idx),
3097              rdba & 0x00000000ffffffffULL);
3098         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3099         wr32(E1000_RDLEN(reg_idx),
3100                        ring->count * sizeof(union e1000_adv_rx_desc));
3101
3102         /* initialize head and tail */
3103         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3104         wr32(E1000_RDH(reg_idx), 0);
3105         writel(0, ring->tail);
3106
3107         /* set descriptor configuration */
3108         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3109 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3110         srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3111 #else
3112         srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3113 #endif
3114         srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3115 #ifdef CONFIG_IGB_PTP
3116         if (hw->mac.type >= e1000_82580)
3117                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3118 #endif /* CONFIG_IGB_PTP */
3119         /* Only set Drop Enable if we are supporting multiple queues */
3120         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3121                 srrctl |= E1000_SRRCTL_DROP_EN;
3122
3123         wr32(E1000_SRRCTL(reg_idx), srrctl);
3124
3125         /* set filtering for VMDQ pools */
3126         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3127
3128         rxdctl |= IGB_RX_PTHRESH;
3129         rxdctl |= IGB_RX_HTHRESH << 8;
3130         rxdctl |= IGB_RX_WTHRESH << 16;
3131
3132         /* enable receive descriptor fetching */
3133         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3134         wr32(E1000_RXDCTL(reg_idx), rxdctl);
3135 }
3136
3137 /**
3138  * igb_configure_rx - Configure receive Unit after Reset
3139  * @adapter: board private structure
3140  *
3141  * Configure the Rx unit of the MAC after a reset.
3142  **/
3143 static void igb_configure_rx(struct igb_adapter *adapter)
3144 {
3145         int i;
3146
3147         /* set UTA to appropriate mode */
3148         igb_set_uta(adapter);
3149
3150         /* set the correct pool for the PF default MAC address in entry 0 */
3151         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3152                          adapter->vfs_allocated_count);
3153
3154         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3155          * the Base and Length of the Rx Descriptor Ring */
3156         for (i = 0; i < adapter->num_rx_queues; i++)
3157                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3158 }
3159
3160 /**
3161  * igb_free_tx_resources - Free Tx Resources per Queue
3162  * @tx_ring: Tx descriptor ring for a specific queue
3163  *
3164  * Free all transmit software resources
3165  **/
3166 void igb_free_tx_resources(struct igb_ring *tx_ring)
3167 {
3168         igb_clean_tx_ring(tx_ring);
3169
3170         vfree(tx_ring->tx_buffer_info);
3171         tx_ring->tx_buffer_info = NULL;
3172
3173         /* if not set, then don't free */
3174         if (!tx_ring->desc)
3175                 return;
3176
3177         dma_free_coherent(tx_ring->dev, tx_ring->size,
3178                           tx_ring->desc, tx_ring->dma);
3179
3180         tx_ring->desc = NULL;
3181 }
3182
3183 /**
3184  * igb_free_all_tx_resources - Free Tx Resources for All Queues
3185  * @adapter: board private structure
3186  *
3187  * Free all transmit software resources
3188  **/
3189 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3190 {
3191         int i;
3192
3193         for (i = 0; i < adapter->num_tx_queues; i++)
3194                 igb_free_tx_resources(adapter->tx_ring[i]);
3195 }
3196
3197 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3198                                     struct igb_tx_buffer *tx_buffer)
3199 {
3200         if (tx_buffer->skb) {
3201                 dev_kfree_skb_any(tx_buffer->skb);
3202                 if (dma_unmap_len(tx_buffer, len))
3203                         dma_unmap_single(ring->dev,
3204                                          dma_unmap_addr(tx_buffer, dma),
3205                                          dma_unmap_len(tx_buffer, len),
3206                                          DMA_TO_DEVICE);
3207         } else if (dma_unmap_len(tx_buffer, len)) {
3208                 dma_unmap_page(ring->dev,
3209                                dma_unmap_addr(tx_buffer, dma),
3210                                dma_unmap_len(tx_buffer, len),
3211                                DMA_TO_DEVICE);
3212         }
3213         tx_buffer->next_to_watch = NULL;
3214         tx_buffer->skb = NULL;
3215         dma_unmap_len_set(tx_buffer, len, 0);
3216         /* buffer_info must be completely set up in the transmit path */
3217 }
3218
3219 /**
3220  * igb_clean_tx_ring - Free Tx Buffers
3221  * @tx_ring: ring to be cleaned
3222  **/
3223 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3224 {
3225         struct igb_tx_buffer *buffer_info;
3226         unsigned long size;
3227         u16 i;
3228
3229         if (!tx_ring->tx_buffer_info)
3230                 return;
3231         /* Free all the Tx ring sk_buffs */
3232
3233         for (i = 0; i < tx_ring->count; i++) {
3234                 buffer_info = &tx_ring->tx_buffer_info[i];
3235                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3236         }
3237
3238         netdev_tx_reset_queue(txring_txq(tx_ring));
3239
3240         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3241         memset(tx_ring->tx_buffer_info, 0, size);
3242
3243         /* Zero out the descriptor ring */
3244         memset(tx_ring->desc, 0, tx_ring->size);
3245
3246         tx_ring->next_to_use = 0;
3247         tx_ring->next_to_clean = 0;
3248 }
3249
3250 /**
3251  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3252  * @adapter: board private structure
3253  **/
3254 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3255 {
3256         int i;
3257
3258         for (i = 0; i < adapter->num_tx_queues; i++)
3259                 igb_clean_tx_ring(adapter->tx_ring[i]);
3260 }
3261
3262 /**
3263  * igb_free_rx_resources - Free Rx Resources
3264  * @rx_ring: ring to clean the resources from
3265  *
3266  * Free all receive software resources
3267  **/
3268 void igb_free_rx_resources(struct igb_ring *rx_ring)
3269 {
3270         igb_clean_rx_ring(rx_ring);
3271
3272         vfree(rx_ring->rx_buffer_info);
3273         rx_ring->rx_buffer_info = NULL;
3274
3275         /* if not set, then don't free */
3276         if (!rx_ring->desc)
3277                 return;
3278
3279         dma_free_coherent(rx_ring->dev, rx_ring->size,
3280                           rx_ring->desc, rx_ring->dma);
3281
3282         rx_ring->desc = NULL;
3283 }
3284
3285 /**
3286  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3287  * @adapter: board private structure
3288  *
3289  * Free all receive software resources
3290  **/
3291 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3292 {
3293         int i;
3294
3295         for (i = 0; i < adapter->num_rx_queues; i++)
3296                 igb_free_rx_resources(adapter->rx_ring[i]);
3297 }
3298
3299 /**
3300  * igb_clean_rx_ring - Free Rx Buffers per Queue
3301  * @rx_ring: ring to free buffers from
3302  **/
3303 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3304 {
3305         unsigned long size;
3306         u16 i;
3307
3308         if (!rx_ring->rx_buffer_info)
3309                 return;
3310
3311         /* Free all the Rx ring sk_buffs */
3312         for (i = 0; i < rx_ring->count; i++) {
3313                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3314                 if (buffer_info->dma) {
3315                         dma_unmap_single(rx_ring->dev,
3316                                          buffer_info->dma,
3317                                          IGB_RX_HDR_LEN,
3318                                          DMA_FROM_DEVICE);
3319                         buffer_info->dma = 0;
3320                 }
3321
3322                 if (buffer_info->skb) {
3323                         dev_kfree_skb(buffer_info->skb);
3324                         buffer_info->skb = NULL;
3325                 }
3326                 if (buffer_info->page_dma) {
3327                         dma_unmap_page(rx_ring->dev,
3328                                        buffer_info->page_dma,
3329                                        PAGE_SIZE / 2,
3330                                        DMA_FROM_DEVICE);
3331                         buffer_info->page_dma = 0;
3332                 }
3333                 if (buffer_info->page) {
3334                         put_page(buffer_info->page);
3335                         buffer_info->page = NULL;
3336                         buffer_info->page_offset = 0;
3337                 }
3338         }
3339
3340         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3341         memset(rx_ring->rx_buffer_info, 0, size);
3342
3343         /* Zero out the descriptor ring */
3344         memset(rx_ring->desc, 0, rx_ring->size);
3345
3346         rx_ring->next_to_clean = 0;
3347         rx_ring->next_to_use = 0;
3348 }
3349
3350 /**
3351  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3352  * @adapter: board private structure
3353  **/
3354 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3355 {
3356         int i;
3357
3358         for (i = 0; i < adapter->num_rx_queues; i++)
3359                 igb_clean_rx_ring(adapter->rx_ring[i]);
3360 }
3361
3362 /**
3363  * igb_set_mac - Change the Ethernet Address of the NIC
3364  * @netdev: network interface device structure
3365  * @p: pointer to an address structure
3366  *
3367  * Returns 0 on success, negative on failure
3368  **/
3369 static int igb_set_mac(struct net_device *netdev, void *p)
3370 {
3371         struct igb_adapter *adapter = netdev_priv(netdev);
3372         struct e1000_hw *hw = &adapter->hw;
3373         struct sockaddr *addr = p;
3374
3375         if (!is_valid_ether_addr(addr->sa_data))
3376                 return -EADDRNOTAVAIL;
3377
3378         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3379         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3380
3381         /* set the correct pool for the new PF MAC address in entry 0 */
3382         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3383                          adapter->vfs_allocated_count);
3384
3385         return 0;
3386 }
3387
3388 /**
3389  * igb_write_mc_addr_list - write multicast addresses to MTA
3390  * @netdev: network interface device structure
3391  *
3392  * Writes multicast address list to the MTA hash table.
3393  * Returns: -ENOMEM on failure
3394  *                0 on no addresses written
3395  *                X on writing X addresses to MTA
3396  **/
3397 static int igb_write_mc_addr_list(struct net_device *netdev)
3398 {
3399         struct igb_adapter *adapter = netdev_priv(netdev);
3400         struct e1000_hw *hw = &adapter->hw;
3401         struct netdev_hw_addr *ha;
3402         u8  *mta_list;
3403         int i;
3404
3405         if (netdev_mc_empty(netdev)) {
3406                 /* nothing to program, so clear mc list */
3407                 igb_update_mc_addr_list(hw, NULL, 0);
3408                 igb_restore_vf_multicasts(adapter);
3409                 return 0;
3410         }
3411
3412         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3413         if (!mta_list)
3414                 return -ENOMEM;
3415
3416         /* The shared function expects a packed array of only addresses. */
3417         i = 0;
3418         netdev_for_each_mc_addr(ha, netdev)
3419                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3420
3421         igb_update_mc_addr_list(hw, mta_list, i);
3422         kfree(mta_list);
3423
3424         return netdev_mc_count(netdev);
3425 }
3426
3427 /**
3428  * igb_write_uc_addr_list - write unicast addresses to RAR table
3429  * @netdev: network interface device structure
3430  *
3431  * Writes unicast address list to the RAR table.
3432  * Returns: -ENOMEM on failure/insufficient address space
3433  *                0 on no addresses written
3434  *                X on writing X addresses to the RAR table
3435  **/
3436 static int igb_write_uc_addr_list(struct net_device *netdev)
3437 {
3438         struct igb_adapter *adapter = netdev_priv(netdev);
3439         struct e1000_hw *hw = &adapter->hw;
3440         unsigned int vfn = adapter->vfs_allocated_count;
3441         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3442         int count = 0;
3443
3444         /* return ENOMEM indicating insufficient memory for addresses */
3445         if (netdev_uc_count(netdev) > rar_entries)
3446                 return -ENOMEM;
3447
3448         if (!netdev_uc_empty(netdev) && rar_entries) {
3449                 struct netdev_hw_addr *ha;
3450
3451                 netdev_for_each_uc_addr(ha, netdev) {
3452                         if (!rar_entries)
3453                                 break;
3454                         igb_rar_set_qsel(adapter, ha->addr,
3455                                          rar_entries--,
3456                                          vfn);
3457                         count++;
3458                 }
3459         }
3460         /* write the addresses in reverse order to avoid write combining */
3461         for (; rar_entries > 0 ; rar_entries--) {
3462                 wr32(E1000_RAH(rar_entries), 0);
3463                 wr32(E1000_RAL(rar_entries), 0);
3464         }
3465         wrfl();
3466
3467         return count;
3468 }
3469
3470 /**
3471  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3472  * @netdev: network interface device structure
3473  *
3474  * The set_rx_mode entry point is called whenever the unicast or multicast
3475  * address lists or the network interface flags are updated.  This routine is
3476  * responsible for configuring the hardware for proper unicast, multicast,
3477  * promiscuous mode, and all-multi behavior.
3478  **/
3479 static void igb_set_rx_mode(struct net_device *netdev)
3480 {
3481         struct igb_adapter *adapter = netdev_priv(netdev);
3482         struct e1000_hw *hw = &adapter->hw;
3483         unsigned int vfn = adapter->vfs_allocated_count;
3484         u32 rctl, vmolr = 0;
3485         int count;
3486
3487         /* Check for Promiscuous and All Multicast modes */
3488         rctl = rd32(E1000_RCTL);
3489
3490         /* clear the effected bits */
3491         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3492
3493         if (netdev->flags & IFF_PROMISC) {
3494                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3495                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3496         } else {
3497                 if (netdev->flags & IFF_ALLMULTI) {
3498                         rctl |= E1000_RCTL_MPE;
3499                         vmolr |= E1000_VMOLR_MPME;
3500                 } else {
3501                         /*
3502                          * Write addresses to the MTA, if the attempt fails
3503                          * then we should just turn on promiscuous mode so
3504                          * that we can at least receive multicast traffic
3505                          */
3506                         count = igb_write_mc_addr_list(netdev);
3507                         if (count < 0) {
3508                                 rctl |= E1000_RCTL_MPE;
3509                                 vmolr |= E1000_VMOLR_MPME;
3510                         } else if (count) {
3511                                 vmolr |= E1000_VMOLR_ROMPE;
3512                         }
3513                 }
3514                 /*
3515                  * Write addresses to available RAR registers, if there is not
3516                  * sufficient space to store all the addresses then enable
3517                  * unicast promiscuous mode
3518                  */
3519                 count = igb_write_uc_addr_list(netdev);
3520                 if (count < 0) {
3521                         rctl |= E1000_RCTL_UPE;
3522                         vmolr |= E1000_VMOLR_ROPE;
3523                 }
3524                 rctl |= E1000_RCTL_VFE;
3525         }
3526         wr32(E1000_RCTL, rctl);
3527
3528         /*
3529          * In order to support SR-IOV and eventually VMDq it is necessary to set
3530          * the VMOLR to enable the appropriate modes.  Without this workaround
3531          * we will have issues with VLAN tag stripping not being done for frames
3532          * that are only arriving because we are the default pool
3533          */
3534         if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3535                 return;
3536
3537         vmolr |= rd32(E1000_VMOLR(vfn)) &
3538                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3539         wr32(E1000_VMOLR(vfn), vmolr);
3540         igb_restore_vf_multicasts(adapter);
3541 }
3542
3543 static void igb_check_wvbr(struct igb_adapter *adapter)
3544 {
3545         struct e1000_hw *hw = &adapter->hw;
3546         u32 wvbr = 0;
3547
3548         switch (hw->mac.type) {
3549         case e1000_82576:
3550         case e1000_i350:
3551                 if (!(wvbr = rd32(E1000_WVBR)))
3552                         return;
3553                 break;
3554         default:
3555                 break;
3556         }
3557
3558         adapter->wvbr |= wvbr;
3559 }
3560
3561 #define IGB_STAGGERED_QUEUE_OFFSET 8
3562
3563 static void igb_spoof_check(struct igb_adapter *adapter)
3564 {
3565         int j;
3566
3567         if (!adapter->wvbr)
3568                 return;
3569
3570         for(j = 0; j < adapter->vfs_allocated_count; j++) {
3571                 if (adapter->wvbr & (1 << j) ||
3572                     adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3573                         dev_warn(&adapter->pdev->dev,
3574                                 "Spoof event(s) detected on VF %d\n", j);
3575                         adapter->wvbr &=
3576                                 ~((1 << j) |
3577                                   (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3578                 }
3579         }
3580 }
3581
3582 /* Need to wait a few seconds after link up to get diagnostic information from
3583  * the phy */
3584 static void igb_update_phy_info(unsigned long data)
3585 {
3586         struct igb_adapter *adapter = (struct igb_adapter *) data;
3587         igb_get_phy_info(&adapter->hw);
3588 }
3589
3590 /**
3591  * igb_has_link - check shared code for link and determine up/down
3592  * @adapter: pointer to driver private info
3593  **/
3594 bool igb_has_link(struct igb_adapter *adapter)
3595 {
3596         struct e1000_hw *hw = &adapter->hw;
3597         bool link_active = false;
3598         s32 ret_val = 0;
3599
3600         /* get_link_status is set on LSC (link status) interrupt or
3601          * rx sequence error interrupt.  get_link_status will stay
3602          * false until the e1000_check_for_link establishes link
3603          * for copper adapters ONLY
3604          */
3605         switch (hw->phy.media_type) {
3606         case e1000_media_type_copper:
3607                 if (hw->mac.get_link_status) {
3608                         ret_val = hw->mac.ops.check_for_link(hw);
3609                         link_active = !hw->mac.get_link_status;
3610                 } else {
3611                         link_active = true;
3612                 }
3613                 break;
3614         case e1000_media_type_internal_serdes:
3615                 ret_val = hw->mac.ops.check_for_link(hw);
3616                 link_active = hw->mac.serdes_has_link;
3617                 break;
3618         default:
3619         case e1000_media_type_unknown:
3620                 break;
3621         }
3622
3623         return link_active;
3624 }
3625
3626 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3627 {
3628         bool ret = false;
3629         u32 ctrl_ext, thstat;
3630
3631         /* check for thermal sensor event on i350 copper only */
3632         if (hw->mac.type == e1000_i350) {
3633                 thstat = rd32(E1000_THSTAT);
3634                 ctrl_ext = rd32(E1000_CTRL_EXT);
3635
3636                 if ((hw->phy.media_type == e1000_media_type_copper) &&
3637                     !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3638                         ret = !!(thstat & event);
3639                 }
3640         }
3641
3642         return ret;
3643 }
3644
3645 /**
3646  * igb_watchdog - Timer Call-back
3647  * @data: pointer to adapter cast into an unsigned long
3648  **/
3649 static void igb_watchdog(unsigned long data)
3650 {
3651         struct igb_adapter *adapter = (struct igb_adapter *)data;
3652         /* Do the rest outside of interrupt context */
3653         schedule_work(&adapter->watchdog_task);
3654 }
3655
3656 static void igb_watchdog_task(struct work_struct *work)
3657 {
3658         struct igb_adapter *adapter = container_of(work,
3659                                                    struct igb_adapter,
3660                                                    watchdog_task);
3661         struct e1000_hw *hw = &adapter->hw;
3662         struct net_device *netdev = adapter->netdev;
3663         u32 link;
3664         int i;
3665
3666         link = igb_has_link(adapter);
3667         if (link) {
3668                 /* Cancel scheduled suspend requests. */
3669                 pm_runtime_resume(netdev->dev.parent);
3670
3671                 if (!netif_carrier_ok(netdev)) {
3672                         u32 ctrl;
3673                         hw->mac.ops.get_speed_and_duplex(hw,
3674                                                          &adapter->link_speed,
3675                                                          &adapter->link_duplex);
3676
3677                         ctrl = rd32(E1000_CTRL);
3678                         /* Links status message must follow this format */
3679                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3680                                "Duplex, Flow Control: %s\n",
3681                                netdev->name,
3682                                adapter->link_speed,
3683                                adapter->link_duplex == FULL_DUPLEX ?
3684                                "Full" : "Half",
3685                                (ctrl & E1000_CTRL_TFCE) &&
3686                                (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3687                                (ctrl & E1000_CTRL_RFCE) ?  "RX" :
3688                                (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
3689
3690                         /* check for thermal sensor event */
3691                         if (igb_thermal_sensor_event(hw,
3692                             E1000_THSTAT_LINK_THROTTLE)) {
3693                                 netdev_info(netdev, "The network adapter link "
3694                                             "speed was downshifted because it "
3695                                             "overheated\n");
3696                         }
3697
3698                         /* adjust timeout factor according to speed/duplex */
3699                         adapter->tx_timeout_factor = 1;
3700                         switch (adapter->link_speed) {
3701                         case SPEED_10:
3702                                 adapter->tx_timeout_factor = 14;
3703                                 break;
3704                         case SPEED_100:
3705                                 /* maybe add some timeout factor ? */
3706                                 break;
3707                         }
3708
3709                         netif_carrier_on(netdev);
3710
3711                         igb_ping_all_vfs(adapter);
3712                         igb_check_vf_rate_limit(adapter);
3713
3714                         /* link state has changed, schedule phy info update */
3715                         if (!test_bit(__IGB_DOWN, &adapter->state))
3716                                 mod_timer(&adapter->phy_info_timer,
3717                                           round_jiffies(jiffies + 2 * HZ));
3718                 }
3719         } else {
3720                 if (netif_carrier_ok(netdev)) {
3721                         adapter->link_speed = 0;
3722                         adapter->link_duplex = 0;
3723
3724                         /* check for thermal sensor event */
3725                         if (igb_thermal_sensor_event(hw,
3726                             E1000_THSTAT_PWR_DOWN)) {
3727                                 netdev_err(netdev, "The network adapter was "
3728                                            "stopped because it overheated\n");
3729                         }
3730
3731                         /* Links status message must follow this format */
3732                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3733                                netdev->name);
3734                         netif_carrier_off(netdev);
3735
3736                         igb_ping_all_vfs(adapter);
3737
3738                         /* link state has changed, schedule phy info update */
3739                         if (!test_bit(__IGB_DOWN, &adapter->state))
3740                                 mod_timer(&adapter->phy_info_timer,
3741                                           round_jiffies(jiffies + 2 * HZ));
3742
3743                         pm_schedule_suspend(netdev->dev.parent,
3744                                             MSEC_PER_SEC * 5);
3745                 }
3746         }
3747
3748         spin_lock(&adapter->stats64_lock);
3749         igb_update_stats(adapter, &adapter->stats64);
3750         spin_unlock(&adapter->stats64_lock);
3751
3752         for (i = 0; i < adapter->num_tx_queues; i++) {
3753                 struct igb_ring *tx_ring = adapter->tx_ring[i];
3754                 if (!netif_carrier_ok(netdev)) {
3755                         /* We've lost link, so the controller stops DMA,
3756                          * but we've got queued Tx work that's never going
3757                          * to get done, so reset controller to flush Tx.
3758                          * (Do the reset outside of interrupt context). */
3759                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3760                                 adapter->tx_timeout_count++;
3761                                 schedule_work(&adapter->reset_task);
3762                                 /* return immediately since reset is imminent */
3763                                 return;
3764                         }
3765                 }
3766
3767                 /* Force detection of hung controller every watchdog period */
3768                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3769         }
3770
3771         /* Cause software interrupt to ensure rx ring is cleaned */
3772         if (adapter->msix_entries) {
3773                 u32 eics = 0;
3774                 for (i = 0; i < adapter->num_q_vectors; i++)
3775                         eics |= adapter->q_vector[i]->eims_value;
3776                 wr32(E1000_EICS, eics);
3777         } else {
3778                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3779         }
3780
3781         igb_spoof_check(adapter);
3782
3783         /* Reset the timer */
3784         if (!test_bit(__IGB_DOWN, &adapter->state))
3785                 mod_timer(&adapter->watchdog_timer,
3786                           round_jiffies(jiffies + 2 * HZ));
3787 }
3788
3789 enum latency_range {
3790         lowest_latency = 0,
3791         low_latency = 1,
3792         bulk_latency = 2,
3793         latency_invalid = 255
3794 };
3795
3796 /**
3797  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3798  *
3799  *      Stores a new ITR value based on strictly on packet size.  This
3800  *      algorithm is less sophisticated than that used in igb_update_itr,
3801  *      due to the difficulty of synchronizing statistics across multiple
3802  *      receive rings.  The divisors and thresholds used by this function
3803  *      were determined based on theoretical maximum wire speed and testing
3804  *      data, in order to minimize response time while increasing bulk
3805  *      throughput.
3806  *      This functionality is controlled by the InterruptThrottleRate module
3807  *      parameter (see igb_param.c)
3808  *      NOTE:  This function is called only when operating in a multiqueue
3809  *             receive environment.
3810  * @q_vector: pointer to q_vector
3811  **/
3812 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3813 {
3814         int new_val = q_vector->itr_val;
3815         int avg_wire_size = 0;
3816         struct igb_adapter *adapter = q_vector->adapter;
3817         unsigned int packets;
3818
3819         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3820          * ints/sec - ITR timer value of 120 ticks.
3821          */
3822         if (adapter->link_speed != SPEED_1000) {
3823                 new_val = IGB_4K_ITR;
3824                 goto set_itr_val;
3825         }
3826
3827         packets = q_vector->rx.total_packets;
3828         if (packets)
3829                 avg_wire_size = q_vector->rx.total_bytes / packets;
3830
3831         packets = q_vector->tx.total_packets;
3832         if (packets)
3833                 avg_wire_size = max_t(u32, avg_wire_size,
3834                                       q_vector->tx.total_bytes / packets);
3835
3836         /* if avg_wire_size isn't set no work was done */
3837         if (!avg_wire_size)
3838                 goto clear_counts;
3839
3840         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3841         avg_wire_size += 24;
3842
3843         /* Don't starve jumbo frames */
3844         avg_wire_size = min(avg_wire_size, 3000);
3845
3846         /* Give a little boost to mid-size frames */
3847         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3848                 new_val = avg_wire_size / 3;
3849         else
3850                 new_val = avg_wire_size / 2;
3851
3852         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3853         if (new_val < IGB_20K_ITR &&
3854             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3855              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3856                 new_val = IGB_20K_ITR;
3857
3858 set_itr_val:
3859         if (new_val != q_vector->itr_val) {
3860                 q_vector->itr_val = new_val;
3861                 q_vector->set_itr = 1;
3862         }
3863 clear_counts:
3864         q_vector->rx.total_bytes = 0;
3865         q_vector->rx.total_packets = 0;
3866         q_vector->tx.total_bytes = 0;
3867         q_vector->tx.total_packets = 0;
3868 }
3869
3870 /**
3871  * igb_update_itr - update the dynamic ITR value based on statistics
3872  *      Stores a new ITR value based on packets and byte
3873  *      counts during the last interrupt.  The advantage of per interrupt
3874  *      computation is faster updates and more accurate ITR for the current
3875  *      traffic pattern.  Constants in this function were computed
3876  *      based on theoretical maximum wire speed and thresholds were set based
3877  *      on testing data as well as attempting to minimize response time
3878  *      while increasing bulk throughput.
3879  *      this functionality is controlled by the InterruptThrottleRate module
3880  *      parameter (see igb_param.c)
3881  *      NOTE:  These calculations are only valid when operating in a single-
3882  *             queue environment.
3883  * @q_vector: pointer to q_vector
3884  * @ring_container: ring info to update the itr for
3885  **/
3886 static void igb_update_itr(struct igb_q_vector *q_vector,
3887                            struct igb_ring_container *ring_container)
3888 {
3889         unsigned int packets = ring_container->total_packets;
3890         unsigned int bytes = ring_container->total_bytes;
3891         u8 itrval = ring_container->itr;
3892
3893         /* no packets, exit with status unchanged */
3894         if (packets == 0)
3895                 return;
3896
3897         switch (itrval) {
3898         case lowest_latency:
3899                 /* handle TSO and jumbo frames */
3900                 if (bytes/packets > 8000)
3901                         itrval = bulk_latency;
3902                 else if ((packets < 5) && (bytes > 512))
3903                         itrval = low_latency;
3904                 break;
3905         case low_latency:  /* 50 usec aka 20000 ints/s */
3906                 if (bytes > 10000) {
3907                         /* this if handles the TSO accounting */
3908                         if (bytes/packets > 8000) {
3909                                 itrval = bulk_latency;
3910                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3911                                 itrval = bulk_latency;
3912                         } else if ((packets > 35)) {
3913                                 itrval = lowest_latency;
3914                         }
3915                 } else if (bytes/packets > 2000) {
3916                         itrval = bulk_latency;
3917                 } else if (packets <= 2 && bytes < 512) {
3918                         itrval = lowest_latency;
3919                 }
3920                 break;
3921         case bulk_latency: /* 250 usec aka 4000 ints/s */
3922                 if (bytes > 25000) {
3923                         if (packets > 35)
3924                                 itrval = low_latency;
3925                 } else if (bytes < 1500) {
3926                         itrval = low_latency;
3927                 }
3928                 break;
3929         }
3930
3931         /* clear work counters since we have the values we need */
3932         ring_container->total_bytes = 0;
3933         ring_container->total_packets = 0;
3934
3935         /* write updated itr to ring container */
3936         ring_container->itr = itrval;
3937 }
3938
3939 static void igb_set_itr(struct igb_q_vector *q_vector)
3940 {
3941         struct igb_adapter *adapter = q_vector->adapter;
3942         u32 new_itr = q_vector->itr_val;
3943         u8 current_itr = 0;
3944
3945         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3946         if (adapter->link_speed != SPEED_1000) {
3947                 current_itr = 0;
3948                 new_itr = IGB_4K_ITR;
3949                 goto set_itr_now;
3950         }
3951
3952         igb_update_itr(q_vector, &q_vector->tx);
3953         igb_update_itr(q_vector, &q_vector->rx);
3954
3955         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3956
3957         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3958         if (current_itr == lowest_latency &&
3959             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3960              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3961                 current_itr = low_latency;
3962
3963         switch (current_itr) {
3964         /* counts and packets in update_itr are dependent on these numbers */
3965         case lowest_latency:
3966                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
3967                 break;
3968         case low_latency:
3969                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
3970                 break;
3971         case bulk_latency:
3972                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
3973                 break;
3974         default:
3975                 break;
3976         }
3977
3978 set_itr_now:
3979         if (new_itr != q_vector->itr_val) {
3980                 /* this attempts to bias the interrupt rate towards Bulk
3981                  * by adding intermediate steps when interrupt rate is
3982                  * increasing */
3983                 new_itr = new_itr > q_vector->itr_val ?
3984                              max((new_itr * q_vector->itr_val) /
3985                                  (new_itr + (q_vector->itr_val >> 2)),
3986                                  new_itr) :
3987                              new_itr;
3988                 /* Don't write the value here; it resets the adapter's
3989                  * internal timer, and causes us to delay far longer than
3990                  * we should between interrupts.  Instead, we write the ITR
3991                  * value at the beginning of the next interrupt so the timing
3992                  * ends up being correct.
3993                  */
3994                 q_vector->itr_val = new_itr;
3995                 q_vector->set_itr = 1;
3996         }
3997 }
3998
3999 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4000                             u32 type_tucmd, u32 mss_l4len_idx)
4001 {
4002         struct e1000_adv_tx_context_desc *context_desc;
4003         u16 i = tx_ring->next_to_use;
4004
4005         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4006
4007         i++;
4008         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4009
4010         /* set bits to identify this as an advanced context descriptor */
4011         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4012
4013         /* For 82575, context index must be unique per ring. */
4014         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4015                 mss_l4len_idx |= tx_ring->reg_idx << 4;
4016
4017         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
4018         context_desc->seqnum_seed       = 0;
4019         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
4020         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
4021 }
4022
4023 static int igb_tso(struct igb_ring *tx_ring,
4024                    struct igb_tx_buffer *first,
4025                    u8 *hdr_len)
4026 {
4027         struct sk_buff *skb = first->skb;
4028         u32 vlan_macip_lens, type_tucmd;
4029         u32 mss_l4len_idx, l4len;
4030
4031         if (!skb_is_gso(skb))
4032                 return 0;
4033
4034         if (skb_header_cloned(skb)) {
4035                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4036                 if (err)
4037                         return err;
4038         }
4039
4040         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4041         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4042
4043         if (first->protocol == __constant_htons(ETH_P_IP)) {
4044                 struct iphdr *iph = ip_hdr(skb);
4045                 iph->tot_len = 0;
4046                 iph->check = 0;
4047                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4048                                                          iph->daddr, 0,
4049                                                          IPPROTO_TCP,
4050                                                          0);
4051                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4052                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4053                                    IGB_TX_FLAGS_CSUM |
4054                                    IGB_TX_FLAGS_IPV4;
4055         } else if (skb_is_gso_v6(skb)) {
4056                 ipv6_hdr(skb)->payload_len = 0;
4057                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4058                                                        &ipv6_hdr(skb)->daddr,
4059                                                        0, IPPROTO_TCP, 0);
4060                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4061                                    IGB_TX_FLAGS_CSUM;
4062         }
4063
4064         /* compute header lengths */
4065         l4len = tcp_hdrlen(skb);
4066         *hdr_len = skb_transport_offset(skb) + l4len;
4067
4068         /* update gso size and bytecount with header size */
4069         first->gso_segs = skb_shinfo(skb)->gso_segs;
4070         first->bytecount += (first->gso_segs - 1) * *hdr_len;
4071
4072         /* MSS L4LEN IDX */
4073         mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4074         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4075
4076         /* VLAN MACLEN IPLEN */
4077         vlan_macip_lens = skb_network_header_len(skb);
4078         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4079         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4080
4081         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4082
4083         return 1;
4084 }
4085
4086 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4087 {
4088         struct sk_buff *skb = first->skb;
4089         u32 vlan_macip_lens = 0;
4090         u32 mss_l4len_idx = 0;
4091         u32 type_tucmd = 0;
4092
4093         if (skb->ip_summed != CHECKSUM_PARTIAL) {
4094                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4095                         return;
4096         } else {
4097                 u8 l4_hdr = 0;
4098                 switch (first->protocol) {
4099                 case __constant_htons(ETH_P_IP):
4100                         vlan_macip_lens |= skb_network_header_len(skb);
4101                         type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4102                         l4_hdr = ip_hdr(skb)->protocol;
4103                         break;
4104                 case __constant_htons(ETH_P_IPV6):
4105                         vlan_macip_lens |= skb_network_header_len(skb);
4106                         l4_hdr = ipv6_hdr(skb)->nexthdr;
4107                         break;
4108                 default:
4109                         if (unlikely(net_ratelimit())) {
4110                                 dev_warn(tx_ring->dev,
4111                                  "partial checksum but proto=%x!\n",
4112                                  first->protocol);
4113                         }
4114                         break;
4115                 }
4116
4117                 switch (l4_hdr) {
4118                 case IPPROTO_TCP:
4119                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4120                         mss_l4len_idx = tcp_hdrlen(skb) <<
4121                                         E1000_ADVTXD_L4LEN_SHIFT;
4122                         break;
4123                 case IPPROTO_SCTP:
4124                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4125                         mss_l4len_idx = sizeof(struct sctphdr) <<
4126                                         E1000_ADVTXD_L4LEN_SHIFT;
4127                         break;
4128                 case IPPROTO_UDP:
4129                         mss_l4len_idx = sizeof(struct udphdr) <<
4130                                         E1000_ADVTXD_L4LEN_SHIFT;
4131                         break;
4132                 default:
4133                         if (unlikely(net_ratelimit())) {
4134                                 dev_warn(tx_ring->dev,
4135                                  "partial checksum but l4 proto=%x!\n",
4136                                  l4_hdr);
4137                         }
4138                         break;
4139                 }
4140
4141                 /* update TX checksum flag */
4142                 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4143         }
4144
4145         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4146         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4147
4148         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4149 }
4150
4151 static __le32 igb_tx_cmd_type(u32 tx_flags)
4152 {
4153         /* set type for advanced descriptor with frame checksum insertion */
4154         __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4155                                       E1000_ADVTXD_DCMD_IFCS |
4156                                       E1000_ADVTXD_DCMD_DEXT);
4157
4158         /* set HW vlan bit if vlan is present */
4159         if (tx_flags & IGB_TX_FLAGS_VLAN)
4160                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4161
4162 #ifdef CONFIG_IGB_PTP
4163         /* set timestamp bit if present */
4164         if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP))
4165                 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4166 #endif /* CONFIG_IGB_PTP */
4167
4168         /* set segmentation bits for TSO */
4169         if (tx_flags & IGB_TX_FLAGS_TSO)
4170                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4171
4172         return cmd_type;
4173 }
4174
4175 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4176                                  union e1000_adv_tx_desc *tx_desc,
4177                                  u32 tx_flags, unsigned int paylen)
4178 {
4179         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4180
4181         /* 82575 requires a unique index per ring if any offload is enabled */
4182         if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4183             test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4184                 olinfo_status |= tx_ring->reg_idx << 4;
4185
4186         /* insert L4 checksum */
4187         if (tx_flags & IGB_TX_FLAGS_CSUM) {
4188                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4189
4190                 /* insert IPv4 checksum */
4191                 if (tx_flags & IGB_TX_FLAGS_IPV4)
4192                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4193         }
4194
4195         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4196 }
4197
4198 /*
4199  * The largest size we can write to the descriptor is 65535.  In order to
4200  * maintain a power of two alignment we have to limit ourselves to 32K.
4201  */
4202 #define IGB_MAX_TXD_PWR 15
4203 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
4204
4205 static void igb_tx_map(struct igb_ring *tx_ring,
4206                        struct igb_tx_buffer *first,
4207                        const u8 hdr_len)
4208 {
4209         struct sk_buff *skb = first->skb;
4210         struct igb_tx_buffer *tx_buffer;
4211         union e1000_adv_tx_desc *tx_desc;
4212         dma_addr_t dma;
4213         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4214         unsigned int data_len = skb->data_len;
4215         unsigned int size = skb_headlen(skb);
4216         unsigned int paylen = skb->len - hdr_len;
4217         __le32 cmd_type;
4218         u32 tx_flags = first->tx_flags;
4219         u16 i = tx_ring->next_to_use;
4220
4221         tx_desc = IGB_TX_DESC(tx_ring, i);
4222
4223         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4224         cmd_type = igb_tx_cmd_type(tx_flags);
4225
4226         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4227         if (dma_mapping_error(tx_ring->dev, dma))
4228                 goto dma_error;
4229
4230         /* record length, and DMA address */
4231         dma_unmap_len_set(first, len, size);
4232         dma_unmap_addr_set(first, dma, dma);
4233         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4234
4235         for (;;) {
4236                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4237                         tx_desc->read.cmd_type_len =
4238                                 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4239
4240                         i++;
4241                         tx_desc++;
4242                         if (i == tx_ring->count) {
4243                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
4244                                 i = 0;
4245                         }
4246
4247                         dma += IGB_MAX_DATA_PER_TXD;
4248                         size -= IGB_MAX_DATA_PER_TXD;
4249
4250                         tx_desc->read.olinfo_status = 0;
4251                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4252                 }
4253
4254                 if (likely(!data_len))
4255                         break;
4256
4257                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4258
4259                 i++;
4260                 tx_desc++;
4261                 if (i == tx_ring->count) {
4262                         tx_desc = IGB_TX_DESC(tx_ring, 0);
4263                         i = 0;
4264                 }
4265
4266                 size = skb_frag_size(frag);
4267                 data_len -= size;
4268
4269                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4270                                    size, DMA_TO_DEVICE);
4271                 if (dma_mapping_error(tx_ring->dev, dma))
4272                         goto dma_error;
4273
4274                 tx_buffer = &tx_ring->tx_buffer_info[i];
4275                 dma_unmap_len_set(tx_buffer, len, size);
4276                 dma_unmap_addr_set(tx_buffer, dma, dma);
4277
4278                 tx_desc->read.olinfo_status = 0;
4279                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4280
4281                 frag++;
4282         }
4283
4284         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4285
4286         /* write last descriptor with RS and EOP bits */
4287         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4288         if (unlikely(skb->no_fcs))
4289                 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
4290         tx_desc->read.cmd_type_len = cmd_type;
4291
4292         /* set the timestamp */
4293         first->time_stamp = jiffies;
4294
4295         /*
4296          * Force memory writes to complete before letting h/w know there
4297          * are new descriptors to fetch.  (Only applicable for weak-ordered
4298          * memory model archs, such as IA-64).
4299          *
4300          * We also need this memory barrier to make certain all of the
4301          * status bits have been updated before next_to_watch is written.
4302          */
4303         wmb();
4304
4305         /* set next_to_watch value indicating a packet is present */
4306         first->next_to_watch = tx_desc;
4307
4308         i++;
4309         if (i == tx_ring->count)
4310                 i = 0;
4311
4312         tx_ring->next_to_use = i;
4313
4314         writel(i, tx_ring->tail);
4315
4316         /* we need this if more than one processor can write to our tail
4317          * at a time, it syncronizes IO on IA64/Altix systems */
4318         mmiowb();
4319
4320         return;
4321
4322 dma_error:
4323         dev_err(tx_ring->dev, "TX DMA map failed\n");
4324
4325         /* clear dma mappings for failed tx_buffer_info map */
4326         for (;;) {
4327                 tx_buffer = &tx_ring->tx_buffer_info[i];
4328                 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
4329                 if (tx_buffer == first)
4330                         break;
4331                 if (i == 0)
4332                         i = tx_ring->count;
4333                 i--;
4334         }
4335
4336         tx_ring->next_to_use = i;
4337 }
4338
4339 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4340 {
4341         struct net_device *netdev = tx_ring->netdev;
4342
4343         netif_stop_subqueue(netdev, tx_ring->queue_index);
4344
4345         /* Herbert's original patch had:
4346          *  smp_mb__after_netif_stop_queue();
4347          * but since that doesn't exist yet, just open code it. */
4348         smp_mb();
4349
4350         /* We need to check again in a case another CPU has just
4351          * made room available. */
4352         if (igb_desc_unused(tx_ring) < size)
4353                 return -EBUSY;
4354
4355         /* A reprieve! */
4356         netif_wake_subqueue(netdev, tx_ring->queue_index);
4357
4358         u64_stats_update_begin(&tx_ring->tx_syncp2);
4359         tx_ring->tx_stats.restart_queue2++;
4360         u64_stats_update_end(&tx_ring->tx_syncp2);
4361
4362         return 0;
4363 }
4364
4365 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4366 {
4367         if (igb_desc_unused(tx_ring) >= size)
4368                 return 0;
4369         return __igb_maybe_stop_tx(tx_ring, size);
4370 }
4371
4372 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4373                                 struct igb_ring *tx_ring)
4374 {
4375 #ifdef CONFIG_IGB_PTP
4376         struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4377 #endif /* CONFIG_IGB_PTP */
4378         struct igb_tx_buffer *first;
4379         int tso;
4380         u32 tx_flags = 0;
4381         __be16 protocol = vlan_get_protocol(skb);
4382         u8 hdr_len = 0;
4383
4384         /* need: 1 descriptor per page,
4385          *       + 2 desc gap to keep tail from touching head,
4386          *       + 1 desc for skb->data,
4387          *       + 1 desc for context descriptor,
4388          * otherwise try next time */
4389         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4390                 /* this is a hard error */
4391                 return NETDEV_TX_BUSY;
4392         }
4393
4394         /* record the location of the first descriptor for this packet */
4395         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4396         first->skb = skb;
4397         first->bytecount = skb->len;
4398         first->gso_segs = 1;
4399
4400 #ifdef CONFIG_IGB_PTP
4401         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4402                      !(adapter->ptp_tx_skb))) {
4403                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4404                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4405
4406                 adapter->ptp_tx_skb = skb_get(skb);
4407                 if (adapter->hw.mac.type == e1000_82576)
4408                         schedule_work(&adapter->ptp_tx_work);
4409         }
4410 #endif /* CONFIG_IGB_PTP */
4411
4412         if (vlan_tx_tag_present(skb)) {
4413                 tx_flags |= IGB_TX_FLAGS_VLAN;
4414                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4415         }
4416
4417         /* record initial flags and protocol */
4418         first->tx_flags = tx_flags;
4419         first->protocol = protocol;
4420
4421         tso = igb_tso(tx_ring, first, &hdr_len);
4422         if (tso < 0)
4423                 goto out_drop;
4424         else if (!tso)
4425                 igb_tx_csum(tx_ring, first);
4426
4427         igb_tx_map(tx_ring, first, hdr_len);
4428
4429         /* Make sure there is space in the ring for the next send. */
4430         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4431
4432         return NETDEV_TX_OK;
4433
4434 out_drop:
4435         igb_unmap_and_free_tx_resource(tx_ring, first);
4436
4437         return NETDEV_TX_OK;
4438 }
4439
4440 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4441                                                     struct sk_buff *skb)
4442 {
4443         unsigned int r_idx = skb->queue_mapping;
4444
4445         if (r_idx >= adapter->num_tx_queues)
4446                 r_idx = r_idx % adapter->num_tx_queues;
4447
4448         return adapter->tx_ring[r_idx];
4449 }
4450
4451 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4452                                   struct net_device *netdev)
4453 {
4454         struct igb_adapter *adapter = netdev_priv(netdev);
4455
4456         if (test_bit(__IGB_DOWN, &adapter->state)) {
4457                 dev_kfree_skb_any(skb);
4458                 return NETDEV_TX_OK;
4459         }
4460
4461         if (skb->len <= 0) {
4462                 dev_kfree_skb_any(skb);
4463                 return NETDEV_TX_OK;
4464         }
4465
4466         /*
4467          * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4468          * in order to meet this minimum size requirement.
4469          */
4470         if (skb->len < 17) {
4471                 if (skb_padto(skb, 17))
4472                         return NETDEV_TX_OK;
4473                 skb->len = 17;
4474         }
4475
4476         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4477 }
4478
4479 /**
4480  * igb_tx_timeout - Respond to a Tx Hang
4481  * @netdev: network interface device structure
4482  **/
4483 static void igb_tx_timeout(struct net_device *netdev)
4484 {
4485         struct igb_adapter *adapter = netdev_priv(netdev);
4486         struct e1000_hw *hw = &adapter->hw;
4487
4488         /* Do the reset outside of interrupt context */
4489         adapter->tx_timeout_count++;
4490
4491         if (hw->mac.type >= e1000_82580)
4492                 hw->dev_spec._82575.global_device_reset = true;
4493
4494         schedule_work(&adapter->reset_task);
4495         wr32(E1000_EICS,
4496              (adapter->eims_enable_mask & ~adapter->eims_other));
4497 }
4498
4499 static void igb_reset_task(struct work_struct *work)
4500 {
4501         struct igb_adapter *adapter;
4502         adapter = container_of(work, struct igb_adapter, reset_task);
4503
4504         igb_dump(adapter);
4505         netdev_err(adapter->netdev, "Reset adapter\n");
4506         igb_reinit_locked(adapter);
4507 }
4508
4509 /**
4510  * igb_get_stats64 - Get System Network Statistics
4511  * @netdev: network interface device structure
4512  * @stats: rtnl_link_stats64 pointer
4513  *
4514  **/
4515 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4516                                                  struct rtnl_link_stats64 *stats)
4517 {
4518         struct igb_adapter *adapter = netdev_priv(netdev);
4519
4520         spin_lock(&adapter->stats64_lock);
4521         igb_update_stats(adapter, &adapter->stats64);
4522         memcpy(stats, &adapter->stats64, sizeof(*stats));
4523         spin_unlock(&adapter->stats64_lock);
4524
4525         return stats;
4526 }
4527
4528 /**
4529  * igb_change_mtu - Change the Maximum Transfer Unit
4530  * @netdev: network interface device structure
4531  * @new_mtu: new value for maximum frame size
4532  *
4533  * Returns 0 on success, negative on failure
4534  **/
4535 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4536 {
4537         struct igb_adapter *adapter = netdev_priv(netdev);
4538         struct pci_dev *pdev = adapter->pdev;
4539         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4540
4541         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4542                 dev_err(&pdev->dev, "Invalid MTU setting\n");
4543                 return -EINVAL;
4544         }
4545
4546 #define MAX_STD_JUMBO_FRAME_SIZE 9238
4547         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4548                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4549                 return -EINVAL;
4550         }
4551
4552         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4553                 msleep(1);
4554
4555         /* igb_down has a dependency on max_frame_size */
4556         adapter->max_frame_size = max_frame;
4557
4558         if (netif_running(netdev))
4559                 igb_down(adapter);
4560
4561         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4562                  netdev->mtu, new_mtu);
4563         netdev->mtu = new_mtu;
4564
4565         if (netif_running(netdev))
4566                 igb_up(adapter);
4567         else
4568                 igb_reset(adapter);
4569
4570         clear_bit(__IGB_RESETTING, &adapter->state);
4571
4572         return 0;
4573 }
4574
4575 /**
4576  * igb_update_stats - Update the board statistics counters
4577  * @adapter: board private structure
4578  **/
4579
4580 void igb_update_stats(struct igb_adapter *adapter,
4581                       struct rtnl_link_stats64 *net_stats)
4582 {
4583         struct e1000_hw *hw = &adapter->hw;
4584         struct pci_dev *pdev = adapter->pdev;
4585         u32 reg, mpc;
4586         u16 phy_tmp;
4587         int i;
4588         u64 bytes, packets;
4589         unsigned int start;
4590         u64 _bytes, _packets;
4591
4592 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4593
4594         /*
4595          * Prevent stats update while adapter is being reset, or if the pci
4596          * connection is down.
4597          */
4598         if (adapter->link_speed == 0)
4599                 return;
4600         if (pci_channel_offline(pdev))
4601                 return;
4602
4603         bytes = 0;
4604         packets = 0;
4605         for (i = 0; i < adapter->num_rx_queues; i++) {
4606                 u32 rqdpc = rd32(E1000_RQDPC(i));
4607                 struct igb_ring *ring = adapter->rx_ring[i];
4608
4609                 if (rqdpc) {
4610                         ring->rx_stats.drops += rqdpc;
4611                         net_stats->rx_fifo_errors += rqdpc;
4612                 }
4613
4614                 do {
4615                         start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4616                         _bytes = ring->rx_stats.bytes;
4617                         _packets = ring->rx_stats.packets;
4618                 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4619                 bytes += _bytes;
4620                 packets += _packets;
4621         }
4622
4623         net_stats->rx_bytes = bytes;
4624         net_stats->rx_packets = packets;
4625
4626         bytes = 0;
4627         packets = 0;
4628         for (i = 0; i < adapter->num_tx_queues; i++) {
4629                 struct igb_ring *ring = adapter->tx_ring[i];
4630                 do {
4631                         start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4632                         _bytes = ring->tx_stats.bytes;
4633                         _packets = ring->tx_stats.packets;
4634                 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4635                 bytes += _bytes;
4636                 packets += _packets;
4637         }
4638         net_stats->tx_bytes = bytes;
4639         net_stats->tx_packets = packets;
4640
4641         /* read stats registers */
4642         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4643         adapter->stats.gprc += rd32(E1000_GPRC);
4644         adapter->stats.gorc += rd32(E1000_GORCL);
4645         rd32(E1000_GORCH); /* clear GORCL */
4646         adapter->stats.bprc += rd32(E1000_BPRC);
4647         adapter->stats.mprc += rd32(E1000_MPRC);
4648         adapter->stats.roc += rd32(E1000_ROC);
4649
4650         adapter->stats.prc64 += rd32(E1000_PRC64);
4651         adapter->stats.prc127 += rd32(E1000_PRC127);
4652         adapter->stats.prc255 += rd32(E1000_PRC255);
4653         adapter->stats.prc511 += rd32(E1000_PRC511);
4654         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4655         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4656         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4657         adapter->stats.sec += rd32(E1000_SEC);
4658
4659         mpc = rd32(E1000_MPC);
4660         adapter->stats.mpc += mpc;
4661         net_stats->rx_fifo_errors += mpc;
4662         adapter->stats.scc += rd32(E1000_SCC);
4663         adapter->stats.ecol += rd32(E1000_ECOL);
4664         adapter->stats.mcc += rd32(E1000_MCC);
4665         adapter->stats.latecol += rd32(E1000_LATECOL);
4666         adapter->stats.dc += rd32(E1000_DC);
4667         adapter->stats.rlec += rd32(E1000_RLEC);
4668         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4669         adapter->stats.xontxc += rd32(E1000_XONTXC);
4670         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4671         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4672         adapter->stats.fcruc += rd32(E1000_FCRUC);
4673         adapter->stats.gptc += rd32(E1000_GPTC);
4674         adapter->stats.gotc += rd32(E1000_GOTCL);
4675         rd32(E1000_GOTCH); /* clear GOTCL */
4676         adapter->stats.rnbc += rd32(E1000_RNBC);
4677         adapter->stats.ruc += rd32(E1000_RUC);
4678         adapter->stats.rfc += rd32(E1000_RFC);
4679         adapter->stats.rjc += rd32(E1000_RJC);
4680         adapter->stats.tor += rd32(E1000_TORH);
4681         adapter->stats.tot += rd32(E1000_TOTH);
4682         adapter->stats.tpr += rd32(E1000_TPR);
4683
4684         adapter->stats.ptc64 += rd32(E1000_PTC64);
4685         adapter->stats.ptc127 += rd32(E1000_PTC127);
4686         adapter->stats.ptc255 += rd32(E1000_PTC255);
4687         adapter->stats.ptc511 += rd32(E1000_PTC511);
4688         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4689         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4690
4691         adapter->stats.mptc += rd32(E1000_MPTC);
4692         adapter->stats.bptc += rd32(E1000_BPTC);
4693
4694         adapter->stats.tpt += rd32(E1000_TPT);
4695         adapter->stats.colc += rd32(E1000_COLC);
4696
4697         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4698         /* read internal phy specific stats */
4699         reg = rd32(E1000_CTRL_EXT);
4700         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4701                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4702
4703                 /* this stat has invalid values on i210/i211 */
4704                 if ((hw->mac.type != e1000_i210) &&
4705                     (hw->mac.type != e1000_i211))
4706                         adapter->stats.tncrs += rd32(E1000_TNCRS);
4707         }
4708
4709         adapter->stats.tsctc += rd32(E1000_TSCTC);
4710         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4711
4712         adapter->stats.iac += rd32(E1000_IAC);
4713         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4714         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4715         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4716         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4717         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4718         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4719         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4720         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4721
4722         /* Fill out the OS statistics structure */
4723         net_stats->multicast = adapter->stats.mprc;
4724         net_stats->collisions = adapter->stats.colc;
4725
4726         /* Rx Errors */
4727
4728         /* RLEC on some newer hardware can be incorrect so build
4729          * our own version based on RUC and ROC */
4730         net_stats->rx_errors = adapter->stats.rxerrc +
4731                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4732                 adapter->stats.ruc + adapter->stats.roc +
4733                 adapter->stats.cexterr;
4734         net_stats->rx_length_errors = adapter->stats.ruc +
4735                                       adapter->stats.roc;
4736         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4737         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4738         net_stats->rx_missed_errors = adapter->stats.mpc;
4739
4740         /* Tx Errors */
4741         net_stats->tx_errors = adapter->stats.ecol +
4742                                adapter->stats.latecol;
4743         net_stats->tx_aborted_errors = adapter->stats.ecol;
4744         net_stats->tx_window_errors = adapter->stats.latecol;
4745         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4746
4747         /* Tx Dropped needs to be maintained elsewhere */
4748
4749         /* Phy Stats */
4750         if (hw->phy.media_type == e1000_media_type_copper) {
4751                 if ((adapter->link_speed == SPEED_1000) &&
4752                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4753                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4754                         adapter->phy_stats.idle_errors += phy_tmp;
4755                 }
4756         }
4757
4758         /* Management Stats */
4759         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4760         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4761         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4762
4763         /* OS2BMC Stats */
4764         reg = rd32(E1000_MANC);
4765         if (reg & E1000_MANC_EN_BMC2OS) {
4766                 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4767                 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4768                 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4769                 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4770         }
4771 }
4772
4773 static irqreturn_t igb_msix_other(int irq, void *data)
4774 {
4775         struct igb_adapter *adapter = data;
4776         struct e1000_hw *hw = &adapter->hw;
4777         u32 icr = rd32(E1000_ICR);
4778         /* reading ICR causes bit 31 of EICR to be cleared */
4779
4780         if (icr & E1000_ICR_DRSTA)
4781                 schedule_work(&adapter->reset_task);
4782
4783         if (icr & E1000_ICR_DOUTSYNC) {
4784                 /* HW is reporting DMA is out of sync */
4785                 adapter->stats.doosync++;
4786                 /* The DMA Out of Sync is also indication of a spoof event
4787                  * in IOV mode. Check the Wrong VM Behavior register to
4788                  * see if it is really a spoof event. */
4789                 igb_check_wvbr(adapter);
4790         }
4791
4792         /* Check for a mailbox event */
4793         if (icr & E1000_ICR_VMMB)
4794                 igb_msg_task(adapter);
4795
4796         if (icr & E1000_ICR_LSC) {
4797                 hw->mac.get_link_status = 1;
4798                 /* guard against interrupt when we're going down */
4799                 if (!test_bit(__IGB_DOWN, &adapter->state))
4800                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4801         }
4802
4803 #ifdef CONFIG_IGB_PTP
4804         if (icr & E1000_ICR_TS) {
4805                 u32 tsicr = rd32(E1000_TSICR);
4806
4807                 if (tsicr & E1000_TSICR_TXTS) {
4808                         /* acknowledge the interrupt */
4809                         wr32(E1000_TSICR, E1000_TSICR_TXTS);
4810                         /* retrieve hardware timestamp */
4811                         schedule_work(&adapter->ptp_tx_work);
4812                 }
4813         }
4814 #endif /* CONFIG_IGB_PTP */
4815
4816         wr32(E1000_EIMS, adapter->eims_other);
4817
4818         return IRQ_HANDLED;
4819 }
4820
4821 static void igb_write_itr(struct igb_q_vector *q_vector)
4822 {
4823         struct igb_adapter *adapter = q_vector->adapter;
4824         u32 itr_val = q_vector->itr_val & 0x7FFC;
4825
4826         if (!q_vector->set_itr)
4827                 return;
4828
4829         if (!itr_val)
4830                 itr_val = 0x4;
4831
4832         if (adapter->hw.mac.type == e1000_82575)
4833                 itr_val |= itr_val << 16;
4834         else
4835                 itr_val |= E1000_EITR_CNT_IGNR;
4836
4837         writel(itr_val, q_vector->itr_register);
4838         q_vector->set_itr = 0;
4839 }
4840
4841 static irqreturn_t igb_msix_ring(int irq, void *data)
4842 {
4843         struct igb_q_vector *q_vector = data;
4844
4845         /* Write the ITR value calculated from the previous interrupt. */
4846         igb_write_itr(q_vector);
4847
4848         napi_schedule(&q_vector->napi);
4849
4850         return IRQ_HANDLED;
4851 }
4852
4853 #ifdef CONFIG_IGB_DCA
4854 static void igb_update_dca(struct igb_q_vector *q_vector)
4855 {
4856         struct igb_adapter *adapter = q_vector->adapter;
4857         struct e1000_hw *hw = &adapter->hw;
4858         int cpu = get_cpu();
4859
4860         if (q_vector->cpu == cpu)
4861                 goto out_no_update;
4862
4863         if (q_vector->tx.ring) {
4864                 int q = q_vector->tx.ring->reg_idx;
4865                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4866                 if (hw->mac.type == e1000_82575) {
4867                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4868                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4869                 } else {
4870                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4871                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4872                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4873                 }
4874                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4875                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4876         }
4877         if (q_vector->rx.ring) {
4878                 int q = q_vector->rx.ring->reg_idx;
4879                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4880                 if (hw->mac.type == e1000_82575) {
4881                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4882                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4883                 } else {
4884                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4885                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4886                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4887                 }
4888                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4889                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4890                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4891                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4892         }
4893         q_vector->cpu = cpu;
4894 out_no_update:
4895         put_cpu();
4896 }
4897
4898 static void igb_setup_dca(struct igb_adapter *adapter)
4899 {
4900         struct e1000_hw *hw = &adapter->hw;
4901         int i;
4902
4903         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4904                 return;
4905
4906         /* Always use CB2 mode, difference is masked in the CB driver. */
4907         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4908
4909         for (i = 0; i < adapter->num_q_vectors; i++) {
4910                 adapter->q_vector[i]->cpu = -1;
4911                 igb_update_dca(adapter->q_vector[i]);
4912         }
4913 }
4914
4915 static int __igb_notify_dca(struct device *dev, void *data)
4916 {
4917         struct net_device *netdev = dev_get_drvdata(dev);
4918         struct igb_adapter *adapter = netdev_priv(netdev);
4919         struct pci_dev *pdev = adapter->pdev;
4920         struct e1000_hw *hw = &adapter->hw;
4921         unsigned long event = *(unsigned long *)data;
4922
4923         switch (event) {
4924         case DCA_PROVIDER_ADD:
4925                 /* if already enabled, don't do it again */
4926                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4927                         break;
4928                 if (dca_add_requester(dev) == 0) {
4929                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4930                         dev_info(&pdev->dev, "DCA enabled\n");
4931                         igb_setup_dca(adapter);
4932                         break;
4933                 }
4934                 /* Fall Through since DCA is disabled. */
4935         case DCA_PROVIDER_REMOVE:
4936                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4937                         /* without this a class_device is left
4938                          * hanging around in the sysfs model */
4939                         dca_remove_requester(dev);
4940                         dev_info(&pdev->dev, "DCA disabled\n");
4941                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4942                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4943                 }
4944                 break;
4945         }
4946
4947         return 0;
4948 }
4949
4950 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4951                           void *p)
4952 {
4953         int ret_val;
4954
4955         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4956                                          __igb_notify_dca);
4957
4958         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4959 }
4960 #endif /* CONFIG_IGB_DCA */
4961
4962 #ifdef CONFIG_PCI_IOV
4963 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4964 {
4965         unsigned char mac_addr[ETH_ALEN];
4966
4967         eth_random_addr(mac_addr);
4968         igb_set_vf_mac(adapter, vf, mac_addr);
4969
4970         return 0;
4971 }
4972
4973 static bool igb_vfs_are_assigned(struct igb_adapter *adapter)
4974 {
4975         struct pci_dev *pdev = adapter->pdev;
4976         struct pci_dev *vfdev;
4977         int dev_id;
4978
4979         switch (adapter->hw.mac.type) {
4980         case e1000_82576:
4981                 dev_id = IGB_82576_VF_DEV_ID;
4982                 break;
4983         case e1000_i350:
4984                 dev_id = IGB_I350_VF_DEV_ID;
4985                 break;
4986         default:
4987                 return false;
4988         }
4989
4990         /* loop through all the VFs to see if we own any that are assigned */
4991         vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, dev_id, NULL);
4992         while (vfdev) {
4993                 /* if we don't own it we don't care */
4994                 if (vfdev->is_virtfn && vfdev->physfn == pdev) {
4995                         /* if it is assigned we cannot release it */
4996                         if (vfdev->dev_flags & PCI_DEV_FLAGS_ASSIGNED)
4997                                 return true;
4998                 }
4999
5000                 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, dev_id, vfdev);
5001         }
5002
5003         return false;
5004 }
5005
5006 #endif
5007 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5008 {
5009         struct e1000_hw *hw = &adapter->hw;
5010         u32 ping;
5011         int i;
5012
5013         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5014                 ping = E1000_PF_CONTROL_MSG;
5015                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5016                         ping |= E1000_VT_MSGTYPE_CTS;
5017                 igb_write_mbx(hw, &ping, 1, i);
5018         }
5019 }
5020
5021 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5022 {
5023         struct e1000_hw *hw = &adapter->hw;
5024         u32 vmolr = rd32(E1000_VMOLR(vf));
5025         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5026
5027         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5028                             IGB_VF_FLAG_MULTI_PROMISC);
5029         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5030
5031         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5032                 vmolr |= E1000_VMOLR_MPME;
5033                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5034                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5035         } else {
5036                 /*
5037                  * if we have hashes and we are clearing a multicast promisc
5038                  * flag we need to write the hashes to the MTA as this step
5039                  * was previously skipped
5040                  */
5041                 if (vf_data->num_vf_mc_hashes > 30) {
5042                         vmolr |= E1000_VMOLR_MPME;
5043                 } else if (vf_data->num_vf_mc_hashes) {
5044                         int j;
5045                         vmolr |= E1000_VMOLR_ROMPE;
5046                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5047                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5048                 }
5049         }
5050
5051         wr32(E1000_VMOLR(vf), vmolr);
5052
5053         /* there are flags left unprocessed, likely not supported */
5054         if (*msgbuf & E1000_VT_MSGINFO_MASK)
5055                 return -EINVAL;
5056
5057         return 0;
5058
5059 }
5060
5061 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5062                                   u32 *msgbuf, u32 vf)
5063 {
5064         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5065         u16 *hash_list = (u16 *)&msgbuf[1];
5066         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5067         int i;
5068
5069         /* salt away the number of multicast addresses assigned
5070          * to this VF for later use to restore when the PF multi cast
5071          * list changes
5072          */
5073         vf_data->num_vf_mc_hashes = n;
5074
5075         /* only up to 30 hash values supported */
5076         if (n > 30)
5077                 n = 30;
5078
5079         /* store the hashes for later use */
5080         for (i = 0; i < n; i++)
5081                 vf_data->vf_mc_hashes[i] = hash_list[i];
5082
5083         /* Flush and reset the mta with the new values */
5084         igb_set_rx_mode(adapter->netdev);
5085
5086         return 0;
5087 }
5088
5089 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5090 {
5091         struct e1000_hw *hw = &adapter->hw;
5092         struct vf_data_storage *vf_data;
5093         int i, j;
5094
5095         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5096                 u32 vmolr = rd32(E1000_VMOLR(i));
5097                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5098
5099                 vf_data = &adapter->vf_data[i];
5100
5101                 if ((vf_data->num_vf_mc_hashes > 30) ||
5102                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5103                         vmolr |= E1000_VMOLR_MPME;
5104                 } else if (vf_data->num_vf_mc_hashes) {
5105                         vmolr |= E1000_VMOLR_ROMPE;
5106                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5107                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5108                 }
5109                 wr32(E1000_VMOLR(i), vmolr);
5110         }
5111 }
5112
5113 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5114 {
5115         struct e1000_hw *hw = &adapter->hw;
5116         u32 pool_mask, reg, vid;
5117         int i;
5118
5119         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5120
5121         /* Find the vlan filter for this id */
5122         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5123                 reg = rd32(E1000_VLVF(i));
5124
5125                 /* remove the vf from the pool */
5126                 reg &= ~pool_mask;
5127
5128                 /* if pool is empty then remove entry from vfta */
5129                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5130                     (reg & E1000_VLVF_VLANID_ENABLE)) {
5131                         reg = 0;
5132                         vid = reg & E1000_VLVF_VLANID_MASK;
5133                         igb_vfta_set(hw, vid, false);
5134                 }
5135
5136                 wr32(E1000_VLVF(i), reg);
5137         }
5138
5139         adapter->vf_data[vf].vlans_enabled = 0;
5140 }
5141
5142 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5143 {
5144         struct e1000_hw *hw = &adapter->hw;
5145         u32 reg, i;
5146
5147         /* The vlvf table only exists on 82576 hardware and newer */
5148         if (hw->mac.type < e1000_82576)
5149                 return -1;
5150
5151         /* we only need to do this if VMDq is enabled */
5152         if (!adapter->vfs_allocated_count)
5153                 return -1;
5154
5155         /* Find the vlan filter for this id */
5156         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5157                 reg = rd32(E1000_VLVF(i));
5158                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5159                     vid == (reg & E1000_VLVF_VLANID_MASK))
5160                         break;
5161         }
5162
5163         if (add) {
5164                 if (i == E1000_VLVF_ARRAY_SIZE) {
5165                         /* Did not find a matching VLAN ID entry that was
5166                          * enabled.  Search for a free filter entry, i.e.
5167                          * one without the enable bit set
5168                          */
5169                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5170                                 reg = rd32(E1000_VLVF(i));
5171                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5172                                         break;
5173                         }
5174                 }
5175                 if (i < E1000_VLVF_ARRAY_SIZE) {
5176                         /* Found an enabled/available entry */
5177                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5178
5179                         /* if !enabled we need to set this up in vfta */
5180                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5181                                 /* add VID to filter table */
5182                                 igb_vfta_set(hw, vid, true);
5183                                 reg |= E1000_VLVF_VLANID_ENABLE;
5184                         }
5185                         reg &= ~E1000_VLVF_VLANID_MASK;
5186                         reg |= vid;
5187                         wr32(E1000_VLVF(i), reg);
5188
5189                         /* do not modify RLPML for PF devices */
5190                         if (vf >= adapter->vfs_allocated_count)
5191                                 return 0;
5192
5193                         if (!adapter->vf_data[vf].vlans_enabled) {
5194                                 u32 size;
5195                                 reg = rd32(E1000_VMOLR(vf));
5196                                 size = reg & E1000_VMOLR_RLPML_MASK;
5197                                 size += 4;
5198                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5199                                 reg |= size;
5200                                 wr32(E1000_VMOLR(vf), reg);
5201                         }
5202
5203                         adapter->vf_data[vf].vlans_enabled++;
5204                 }
5205         } else {
5206                 if (i < E1000_VLVF_ARRAY_SIZE) {
5207                         /* remove vf from the pool */
5208                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5209                         /* if pool is empty then remove entry from vfta */
5210                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5211                                 reg = 0;
5212                                 igb_vfta_set(hw, vid, false);
5213                         }
5214                         wr32(E1000_VLVF(i), reg);
5215
5216                         /* do not modify RLPML for PF devices */
5217                         if (vf >= adapter->vfs_allocated_count)
5218                                 return 0;
5219
5220                         adapter->vf_data[vf].vlans_enabled--;
5221                         if (!adapter->vf_data[vf].vlans_enabled) {
5222                                 u32 size;
5223                                 reg = rd32(E1000_VMOLR(vf));
5224                                 size = reg & E1000_VMOLR_RLPML_MASK;
5225                                 size -= 4;
5226                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5227                                 reg |= size;
5228                                 wr32(E1000_VMOLR(vf), reg);
5229                         }
5230                 }
5231         }
5232         return 0;
5233 }
5234
5235 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5236 {
5237         struct e1000_hw *hw = &adapter->hw;
5238
5239         if (vid)
5240                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5241         else
5242                 wr32(E1000_VMVIR(vf), 0);
5243 }
5244
5245 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5246                                int vf, u16 vlan, u8 qos)
5247 {
5248         int err = 0;
5249         struct igb_adapter *adapter = netdev_priv(netdev);
5250
5251         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5252                 return -EINVAL;
5253         if (vlan || qos) {
5254                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5255                 if (err)
5256                         goto out;
5257                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5258                 igb_set_vmolr(adapter, vf, !vlan);
5259                 adapter->vf_data[vf].pf_vlan = vlan;
5260                 adapter->vf_data[vf].pf_qos = qos;
5261                 dev_info(&adapter->pdev->dev,
5262                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5263                 if (test_bit(__IGB_DOWN, &adapter->state)) {
5264                         dev_warn(&adapter->pdev->dev,
5265                                  "The VF VLAN has been set,"
5266                                  " but the PF device is not up.\n");
5267                         dev_warn(&adapter->pdev->dev,
5268                                  "Bring the PF device up before"
5269                                  " attempting to use the VF device.\n");
5270                 }
5271         } else {
5272                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5273                                    false, vf);
5274                 igb_set_vmvir(adapter, vlan, vf);
5275                 igb_set_vmolr(adapter, vf, true);
5276                 adapter->vf_data[vf].pf_vlan = 0;
5277                 adapter->vf_data[vf].pf_qos = 0;
5278        }
5279 out:
5280        return err;
5281 }
5282
5283 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5284 {
5285         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5286         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5287
5288         return igb_vlvf_set(adapter, vid, add, vf);
5289 }
5290
5291 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5292 {
5293         /* clear flags - except flag that indicates PF has set the MAC */
5294         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5295         adapter->vf_data[vf].last_nack = jiffies;
5296
5297         /* reset offloads to defaults */
5298         igb_set_vmolr(adapter, vf, true);
5299
5300         /* reset vlans for device */
5301         igb_clear_vf_vfta(adapter, vf);
5302         if (adapter->vf_data[vf].pf_vlan)
5303                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5304                                     adapter->vf_data[vf].pf_vlan,
5305                                     adapter->vf_data[vf].pf_qos);
5306         else
5307                 igb_clear_vf_vfta(adapter, vf);
5308
5309         /* reset multicast table array for vf */
5310         adapter->vf_data[vf].num_vf_mc_hashes = 0;
5311
5312         /* Flush and reset the mta with the new values */
5313         igb_set_rx_mode(adapter->netdev);
5314 }
5315
5316 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5317 {
5318         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5319
5320         /* generate a new mac address as we were hotplug removed/added */
5321         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5322                 eth_random_addr(vf_mac);
5323
5324         /* process remaining reset events */
5325         igb_vf_reset(adapter, vf);
5326 }
5327
5328 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5329 {
5330         struct e1000_hw *hw = &adapter->hw;
5331         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5332         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5333         u32 reg, msgbuf[3];
5334         u8 *addr = (u8 *)(&msgbuf[1]);
5335
5336         /* process all the same items cleared in a function level reset */
5337         igb_vf_reset(adapter, vf);
5338
5339         /* set vf mac address */
5340         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5341
5342         /* enable transmit and receive for vf */
5343         reg = rd32(E1000_VFTE);
5344         wr32(E1000_VFTE, reg | (1 << vf));
5345         reg = rd32(E1000_VFRE);
5346         wr32(E1000_VFRE, reg | (1 << vf));
5347
5348         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5349
5350         /* reply to reset with ack and vf mac address */
5351         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5352         memcpy(addr, vf_mac, 6);
5353         igb_write_mbx(hw, msgbuf, 3, vf);
5354 }
5355
5356 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5357 {
5358         /*
5359          * The VF MAC Address is stored in a packed array of bytes
5360          * starting at the second 32 bit word of the msg array
5361          */
5362         unsigned char *addr = (char *)&msg[1];
5363         int err = -1;
5364
5365         if (is_valid_ether_addr(addr))
5366                 err = igb_set_vf_mac(adapter, vf, addr);
5367
5368         return err;
5369 }
5370
5371 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5372 {
5373         struct e1000_hw *hw = &adapter->hw;
5374         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5375         u32 msg = E1000_VT_MSGTYPE_NACK;
5376
5377         /* if device isn't clear to send it shouldn't be reading either */
5378         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5379             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5380                 igb_write_mbx(hw, &msg, 1, vf);
5381                 vf_data->last_nack = jiffies;
5382         }
5383 }
5384
5385 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5386 {
5387         struct pci_dev *pdev = adapter->pdev;
5388         u32 msgbuf[E1000_VFMAILBOX_SIZE];
5389         struct e1000_hw *hw = &adapter->hw;
5390         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5391         s32 retval;
5392
5393         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5394
5395         if (retval) {
5396                 /* if receive failed revoke VF CTS stats and restart init */
5397                 dev_err(&pdev->dev, "Error receiving message from VF\n");
5398                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5399                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5400                         return;
5401                 goto out;
5402         }
5403
5404         /* this is a message we already processed, do nothing */
5405         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5406                 return;
5407
5408         /*
5409          * until the vf completes a reset it should not be
5410          * allowed to start any configuration.
5411          */
5412
5413         if (msgbuf[0] == E1000_VF_RESET) {
5414                 igb_vf_reset_msg(adapter, vf);
5415                 return;
5416         }
5417
5418         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5419                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5420                         return;
5421                 retval = -1;
5422                 goto out;
5423         }
5424
5425         switch ((msgbuf[0] & 0xFFFF)) {
5426         case E1000_VF_SET_MAC_ADDR:
5427                 retval = -EINVAL;
5428                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5429                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5430                 else
5431                         dev_warn(&pdev->dev,
5432                                  "VF %d attempted to override administratively "
5433                                  "set MAC address\nReload the VF driver to "
5434                                  "resume operations\n", vf);
5435                 break;
5436         case E1000_VF_SET_PROMISC:
5437                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5438                 break;
5439         case E1000_VF_SET_MULTICAST:
5440                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5441                 break;
5442         case E1000_VF_SET_LPE:
5443                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5444                 break;
5445         case E1000_VF_SET_VLAN:
5446                 retval = -1;
5447                 if (vf_data->pf_vlan)
5448                         dev_warn(&pdev->dev,
5449                                  "VF %d attempted to override administratively "
5450                                  "set VLAN tag\nReload the VF driver to "
5451                                  "resume operations\n", vf);
5452                 else
5453                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5454                 break;
5455         default:
5456                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5457                 retval = -1;
5458                 break;
5459         }
5460
5461         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5462 out:
5463         /* notify the VF of the results of what it sent us */
5464         if (retval)
5465                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5466         else
5467                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5468
5469         igb_write_mbx(hw, msgbuf, 1, vf);
5470 }
5471
5472 static void igb_msg_task(struct igb_adapter *adapter)
5473 {
5474         struct e1000_hw *hw = &adapter->hw;
5475         u32 vf;
5476
5477         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5478                 /* process any reset requests */
5479                 if (!igb_check_for_rst(hw, vf))
5480                         igb_vf_reset_event(adapter, vf);
5481
5482                 /* process any messages pending */
5483                 if (!igb_check_for_msg(hw, vf))
5484                         igb_rcv_msg_from_vf(adapter, vf);
5485
5486                 /* process any acks */
5487                 if (!igb_check_for_ack(hw, vf))
5488                         igb_rcv_ack_from_vf(adapter, vf);
5489         }
5490 }
5491
5492 /**
5493  *  igb_set_uta - Set unicast filter table address
5494  *  @adapter: board private structure
5495  *
5496  *  The unicast table address is a register array of 32-bit registers.
5497  *  The table is meant to be used in a way similar to how the MTA is used
5498  *  however due to certain limitations in the hardware it is necessary to
5499  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5500  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
5501  **/
5502 static void igb_set_uta(struct igb_adapter *adapter)
5503 {
5504         struct e1000_hw *hw = &adapter->hw;
5505         int i;
5506
5507         /* The UTA table only exists on 82576 hardware and newer */
5508         if (hw->mac.type < e1000_82576)
5509                 return;
5510
5511         /* we only need to do this if VMDq is enabled */
5512         if (!adapter->vfs_allocated_count)
5513                 return;
5514
5515         for (i = 0; i < hw->mac.uta_reg_count; i++)
5516                 array_wr32(E1000_UTA, i, ~0);
5517 }
5518
5519 /**
5520  * igb_intr_msi - Interrupt Handler
5521  * @irq: interrupt number
5522  * @data: pointer to a network interface device structure
5523  **/
5524 static irqreturn_t igb_intr_msi(int irq, void *data)
5525 {
5526         struct igb_adapter *adapter = data;
5527         struct igb_q_vector *q_vector = adapter->q_vector[0];
5528         struct e1000_hw *hw = &adapter->hw;
5529         /* read ICR disables interrupts using IAM */
5530         u32 icr = rd32(E1000_ICR);
5531
5532         igb_write_itr(q_vector);
5533
5534         if (icr & E1000_ICR_DRSTA)
5535                 schedule_work(&adapter->reset_task);
5536
5537         if (icr & E1000_ICR_DOUTSYNC) {
5538                 /* HW is reporting DMA is out of sync */
5539                 adapter->stats.doosync++;
5540         }
5541
5542         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5543                 hw->mac.get_link_status = 1;
5544                 if (!test_bit(__IGB_DOWN, &adapter->state))
5545                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5546         }
5547
5548 #ifdef CONFIG_IGB_PTP
5549         if (icr & E1000_ICR_TS) {
5550                 u32 tsicr = rd32(E1000_TSICR);
5551
5552                 if (tsicr & E1000_TSICR_TXTS) {
5553                         /* acknowledge the interrupt */
5554                         wr32(E1000_TSICR, E1000_TSICR_TXTS);
5555                         /* retrieve hardware timestamp */
5556                         schedule_work(&adapter->ptp_tx_work);
5557                 }
5558         }
5559 #endif /* CONFIG_IGB_PTP */
5560
5561         napi_schedule(&q_vector->napi);
5562
5563         return IRQ_HANDLED;
5564 }
5565
5566 /**
5567  * igb_intr - Legacy Interrupt Handler
5568  * @irq: interrupt number
5569  * @data: pointer to a network interface device structure
5570  **/
5571 static irqreturn_t igb_intr(int irq, void *data)
5572 {
5573         struct igb_adapter *adapter = data;
5574         struct igb_q_vector *q_vector = adapter->q_vector[0];
5575         struct e1000_hw *hw = &adapter->hw;
5576         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5577          * need for the IMC write */
5578         u32 icr = rd32(E1000_ICR);
5579
5580         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5581          * not set, then the adapter didn't send an interrupt */
5582         if (!(icr & E1000_ICR_INT_ASSERTED))
5583                 return IRQ_NONE;
5584
5585         igb_write_itr(q_vector);
5586
5587         if (icr & E1000_ICR_DRSTA)
5588                 schedule_work(&adapter->reset_task);
5589
5590         if (icr & E1000_ICR_DOUTSYNC) {
5591                 /* HW is reporting DMA is out of sync */
5592                 adapter->stats.doosync++;
5593         }
5594
5595         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5596                 hw->mac.get_link_status = 1;
5597                 /* guard against interrupt when we're going down */
5598                 if (!test_bit(__IGB_DOWN, &adapter->state))
5599                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5600         }
5601
5602 #ifdef CONFIG_IGB_PTP
5603         if (icr & E1000_ICR_TS) {
5604                 u32 tsicr = rd32(E1000_TSICR);
5605
5606                 if (tsicr & E1000_TSICR_TXTS) {
5607                         /* acknowledge the interrupt */
5608                         wr32(E1000_TSICR, E1000_TSICR_TXTS);
5609                         /* retrieve hardware timestamp */
5610                         schedule_work(&adapter->ptp_tx_work);
5611                 }
5612         }
5613 #endif /* CONFIG_IGB_PTP */
5614
5615         napi_schedule(&q_vector->napi);
5616
5617         return IRQ_HANDLED;
5618 }
5619
5620 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5621 {
5622         struct igb_adapter *adapter = q_vector->adapter;
5623         struct e1000_hw *hw = &adapter->hw;
5624
5625         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5626             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5627                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5628                         igb_set_itr(q_vector);
5629                 else
5630                         igb_update_ring_itr(q_vector);
5631         }
5632
5633         if (!test_bit(__IGB_DOWN, &adapter->state)) {
5634                 if (adapter->msix_entries)
5635                         wr32(E1000_EIMS, q_vector->eims_value);
5636                 else
5637                         igb_irq_enable(adapter);
5638         }
5639 }
5640
5641 /**
5642  * igb_poll - NAPI Rx polling callback
5643  * @napi: napi polling structure
5644  * @budget: count of how many packets we should handle
5645  **/
5646 static int igb_poll(struct napi_struct *napi, int budget)
5647 {
5648         struct igb_q_vector *q_vector = container_of(napi,
5649                                                      struct igb_q_vector,
5650                                                      napi);
5651         bool clean_complete = true;
5652
5653 #ifdef CONFIG_IGB_DCA
5654         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5655                 igb_update_dca(q_vector);
5656 #endif
5657         if (q_vector->tx.ring)
5658                 clean_complete = igb_clean_tx_irq(q_vector);
5659
5660         if (q_vector->rx.ring)
5661                 clean_complete &= igb_clean_rx_irq(q_vector, budget);
5662
5663         /* If all work not completed, return budget and keep polling */
5664         if (!clean_complete)
5665                 return budget;
5666
5667         /* If not enough Rx work done, exit the polling mode */
5668         napi_complete(napi);
5669         igb_ring_irq_enable(q_vector);
5670
5671         return 0;
5672 }
5673
5674 /**
5675  * igb_clean_tx_irq - Reclaim resources after transmit completes
5676  * @q_vector: pointer to q_vector containing needed info
5677  *
5678  * returns true if ring is completely cleaned
5679  **/
5680 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5681 {
5682         struct igb_adapter *adapter = q_vector->adapter;
5683         struct igb_ring *tx_ring = q_vector->tx.ring;
5684         struct igb_tx_buffer *tx_buffer;
5685         union e1000_adv_tx_desc *tx_desc;
5686         unsigned int total_bytes = 0, total_packets = 0;
5687         unsigned int budget = q_vector->tx.work_limit;
5688         unsigned int i = tx_ring->next_to_clean;
5689
5690         if (test_bit(__IGB_DOWN, &adapter->state))
5691                 return true;
5692
5693         tx_buffer = &tx_ring->tx_buffer_info[i];
5694         tx_desc = IGB_TX_DESC(tx_ring, i);
5695         i -= tx_ring->count;
5696
5697         do {
5698                 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
5699
5700                 /* if next_to_watch is not set then there is no work pending */
5701                 if (!eop_desc)
5702                         break;
5703
5704                 /* prevent any other reads prior to eop_desc */
5705                 rmb();
5706
5707                 /* if DD is not set pending work has not been completed */
5708                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5709                         break;
5710
5711                 /* clear next_to_watch to prevent false hangs */
5712                 tx_buffer->next_to_watch = NULL;
5713
5714                 /* update the statistics for this packet */
5715                 total_bytes += tx_buffer->bytecount;
5716                 total_packets += tx_buffer->gso_segs;
5717
5718                 /* free the skb */
5719                 dev_kfree_skb_any(tx_buffer->skb);
5720
5721                 /* unmap skb header data */
5722                 dma_unmap_single(tx_ring->dev,
5723                                  dma_unmap_addr(tx_buffer, dma),
5724                                  dma_unmap_len(tx_buffer, len),
5725                                  DMA_TO_DEVICE);
5726
5727                 /* clear tx_buffer data */
5728                 tx_buffer->skb = NULL;
5729                 dma_unmap_len_set(tx_buffer, len, 0);
5730
5731                 /* clear last DMA location and unmap remaining buffers */
5732                 while (tx_desc != eop_desc) {
5733                         tx_buffer++;
5734                         tx_desc++;
5735                         i++;
5736                         if (unlikely(!i)) {
5737                                 i -= tx_ring->count;
5738                                 tx_buffer = tx_ring->tx_buffer_info;
5739                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
5740                         }
5741
5742                         /* unmap any remaining paged data */
5743                         if (dma_unmap_len(tx_buffer, len)) {
5744                                 dma_unmap_page(tx_ring->dev,
5745                                                dma_unmap_addr(tx_buffer, dma),
5746                                                dma_unmap_len(tx_buffer, len),
5747                                                DMA_TO_DEVICE);
5748                                 dma_unmap_len_set(tx_buffer, len, 0);
5749                         }
5750                 }
5751
5752                 /* move us one more past the eop_desc for start of next pkt */
5753                 tx_buffer++;
5754                 tx_desc++;
5755                 i++;
5756                 if (unlikely(!i)) {
5757                         i -= tx_ring->count;
5758                         tx_buffer = tx_ring->tx_buffer_info;
5759                         tx_desc = IGB_TX_DESC(tx_ring, 0);
5760                 }
5761
5762                 /* issue prefetch for next Tx descriptor */
5763                 prefetch(tx_desc);
5764
5765                 /* update budget accounting */
5766                 budget--;
5767         } while (likely(budget));
5768
5769         netdev_tx_completed_queue(txring_txq(tx_ring),
5770                                   total_packets, total_bytes);
5771         i += tx_ring->count;
5772         tx_ring->next_to_clean = i;
5773         u64_stats_update_begin(&tx_ring->tx_syncp);
5774         tx_ring->tx_stats.bytes += total_bytes;
5775         tx_ring->tx_stats.packets += total_packets;
5776         u64_stats_update_end(&tx_ring->tx_syncp);
5777         q_vector->tx.total_bytes += total_bytes;
5778         q_vector->tx.total_packets += total_packets;
5779
5780         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
5781                 struct e1000_hw *hw = &adapter->hw;
5782
5783                 /* Detect a transmit hang in hardware, this serializes the
5784                  * check with the clearing of time_stamp and movement of i */
5785                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5786                 if (tx_buffer->next_to_watch &&
5787                     time_after(jiffies, tx_buffer->time_stamp +
5788                                (adapter->tx_timeout_factor * HZ)) &&
5789                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5790
5791                         /* detected Tx unit hang */
5792                         dev_err(tx_ring->dev,
5793                                 "Detected Tx Unit Hang\n"
5794                                 "  Tx Queue             <%d>\n"
5795                                 "  TDH                  <%x>\n"
5796                                 "  TDT                  <%x>\n"
5797                                 "  next_to_use          <%x>\n"
5798                                 "  next_to_clean        <%x>\n"
5799                                 "buffer_info[next_to_clean]\n"
5800                                 "  time_stamp           <%lx>\n"
5801                                 "  next_to_watch        <%p>\n"
5802                                 "  jiffies              <%lx>\n"
5803                                 "  desc.status          <%x>\n",
5804                                 tx_ring->queue_index,
5805                                 rd32(E1000_TDH(tx_ring->reg_idx)),
5806                                 readl(tx_ring->tail),
5807                                 tx_ring->next_to_use,
5808                                 tx_ring->next_to_clean,
5809                                 tx_buffer->time_stamp,
5810                                 tx_buffer->next_to_watch,
5811                                 jiffies,
5812                                 tx_buffer->next_to_watch->wb.status);
5813                         netif_stop_subqueue(tx_ring->netdev,
5814                                             tx_ring->queue_index);
5815
5816                         /* we are about to reset, no point in enabling stuff */
5817                         return true;
5818                 }
5819         }
5820
5821         if (unlikely(total_packets &&
5822                      netif_carrier_ok(tx_ring->netdev) &&
5823                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5824                 /* Make sure that anybody stopping the queue after this
5825                  * sees the new next_to_clean.
5826                  */
5827                 smp_mb();
5828                 if (__netif_subqueue_stopped(tx_ring->netdev,
5829                                              tx_ring->queue_index) &&
5830                     !(test_bit(__IGB_DOWN, &adapter->state))) {
5831                         netif_wake_subqueue(tx_ring->netdev,
5832                                             tx_ring->queue_index);
5833
5834                         u64_stats_update_begin(&tx_ring->tx_syncp);
5835                         tx_ring->tx_stats.restart_queue++;
5836                         u64_stats_update_end(&tx_ring->tx_syncp);
5837                 }
5838         }
5839
5840         return !!budget;
5841 }
5842
5843 static inline void igb_rx_checksum(struct igb_ring *ring,
5844                                    union e1000_adv_rx_desc *rx_desc,
5845                                    struct sk_buff *skb)
5846 {
5847         skb_checksum_none_assert(skb);
5848
5849         /* Ignore Checksum bit is set */
5850         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
5851                 return;
5852
5853         /* Rx checksum disabled via ethtool */
5854         if (!(ring->netdev->features & NETIF_F_RXCSUM))
5855                 return;
5856
5857         /* TCP/UDP checksum error bit is set */
5858         if (igb_test_staterr(rx_desc,
5859                              E1000_RXDEXT_STATERR_TCPE |
5860                              E1000_RXDEXT_STATERR_IPE)) {
5861                 /*
5862                  * work around errata with sctp packets where the TCPE aka
5863                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5864                  * packets, (aka let the stack check the crc32c)
5865                  */
5866                 if (!((skb->len == 60) &&
5867                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
5868                         u64_stats_update_begin(&ring->rx_syncp);
5869                         ring->rx_stats.csum_err++;
5870                         u64_stats_update_end(&ring->rx_syncp);
5871                 }
5872                 /* let the stack verify checksum errors */
5873                 return;
5874         }
5875         /* It must be a TCP or UDP packet with a valid checksum */
5876         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5877                                       E1000_RXD_STAT_UDPCS))
5878                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5879
5880         dev_dbg(ring->dev, "cksum success: bits %08X\n",
5881                 le32_to_cpu(rx_desc->wb.upper.status_error));
5882 }
5883
5884 static inline void igb_rx_hash(struct igb_ring *ring,
5885                                union e1000_adv_rx_desc *rx_desc,
5886                                struct sk_buff *skb)
5887 {
5888         if (ring->netdev->features & NETIF_F_RXHASH)
5889                 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5890 }
5891
5892 static void igb_rx_vlan(struct igb_ring *ring,
5893                         union e1000_adv_rx_desc *rx_desc,
5894                         struct sk_buff *skb)
5895 {
5896         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5897                 u16 vid;
5898                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5899                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5900                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5901                 else
5902                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5903
5904                 __vlan_hwaccel_put_tag(skb, vid);
5905         }
5906 }
5907
5908 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
5909 {
5910         /* HW will not DMA in data larger than the given buffer, even if it
5911          * parses the (NFS, of course) header to be larger.  In that case, it
5912          * fills the header buffer and spills the rest into the page.
5913          */
5914         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5915                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5916         if (hlen > IGB_RX_HDR_LEN)
5917                 hlen = IGB_RX_HDR_LEN;
5918         return hlen;
5919 }
5920
5921 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
5922 {
5923         struct igb_ring *rx_ring = q_vector->rx.ring;
5924         union e1000_adv_rx_desc *rx_desc;
5925         const int current_node = numa_node_id();
5926         unsigned int total_bytes = 0, total_packets = 0;
5927         u16 cleaned_count = igb_desc_unused(rx_ring);
5928         u16 i = rx_ring->next_to_clean;
5929
5930         rx_desc = IGB_RX_DESC(rx_ring, i);
5931
5932         while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
5933                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5934                 struct sk_buff *skb = buffer_info->skb;
5935                 union e1000_adv_rx_desc *next_rxd;
5936
5937                 buffer_info->skb = NULL;
5938                 prefetch(skb->data);
5939
5940                 i++;
5941                 if (i == rx_ring->count)
5942                         i = 0;
5943
5944                 next_rxd = IGB_RX_DESC(rx_ring, i);
5945                 prefetch(next_rxd);
5946
5947                 /*
5948                  * This memory barrier is needed to keep us from reading
5949                  * any other fields out of the rx_desc until we know the
5950                  * RXD_STAT_DD bit is set
5951                  */
5952                 rmb();
5953
5954                 if (!skb_is_nonlinear(skb)) {
5955                         __skb_put(skb, igb_get_hlen(rx_desc));
5956                         dma_unmap_single(rx_ring->dev, buffer_info->dma,
5957                                          IGB_RX_HDR_LEN,
5958                                          DMA_FROM_DEVICE);
5959                         buffer_info->dma = 0;
5960                 }
5961
5962                 if (rx_desc->wb.upper.length) {
5963                         u16 length = le16_to_cpu(rx_desc->wb.upper.length);
5964
5965                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5966                                                 buffer_info->page,
5967                                                 buffer_info->page_offset,
5968                                                 length);
5969
5970                         skb->len += length;
5971                         skb->data_len += length;
5972                         skb->truesize += PAGE_SIZE / 2;
5973
5974                         if ((page_count(buffer_info->page) != 1) ||
5975                             (page_to_nid(buffer_info->page) != current_node))
5976                                 buffer_info->page = NULL;
5977                         else
5978                                 get_page(buffer_info->page);
5979
5980                         dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
5981                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
5982                         buffer_info->page_dma = 0;
5983                 }
5984
5985                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
5986                         struct igb_rx_buffer *next_buffer;
5987                         next_buffer = &rx_ring->rx_buffer_info[i];
5988                         buffer_info->skb = next_buffer->skb;
5989                         buffer_info->dma = next_buffer->dma;
5990                         next_buffer->skb = skb;
5991                         next_buffer->dma = 0;
5992                         goto next_desc;
5993                 }
5994
5995                 if (unlikely((igb_test_staterr(rx_desc,
5996                                                E1000_RXDEXT_ERR_FRAME_ERR_MASK))
5997                              && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
5998                         dev_kfree_skb_any(skb);
5999                         goto next_desc;
6000                 }
6001
6002 #ifdef CONFIG_IGB_PTP
6003                 igb_ptp_rx_hwtstamp(q_vector, rx_desc, skb);
6004 #endif /* CONFIG_IGB_PTP */
6005                 igb_rx_hash(rx_ring, rx_desc, skb);
6006                 igb_rx_checksum(rx_ring, rx_desc, skb);
6007                 igb_rx_vlan(rx_ring, rx_desc, skb);
6008
6009                 total_bytes += skb->len;
6010                 total_packets++;
6011
6012                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6013
6014                 napi_gro_receive(&q_vector->napi, skb);
6015
6016                 budget--;
6017 next_desc:
6018                 if (!budget)
6019                         break;
6020
6021                 cleaned_count++;
6022                 /* return some buffers to hardware, one at a time is too slow */
6023                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
6024                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
6025                         cleaned_count = 0;
6026                 }
6027
6028                 /* use prefetched values */
6029                 rx_desc = next_rxd;
6030         }
6031
6032         rx_ring->next_to_clean = i;
6033         u64_stats_update_begin(&rx_ring->rx_syncp);
6034         rx_ring->rx_stats.packets += total_packets;
6035         rx_ring->rx_stats.bytes += total_bytes;
6036         u64_stats_update_end(&rx_ring->rx_syncp);
6037         q_vector->rx.total_packets += total_packets;
6038         q_vector->rx.total_bytes += total_bytes;
6039
6040         if (cleaned_count)
6041                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6042
6043         return !!budget;
6044 }
6045
6046 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
6047                                  struct igb_rx_buffer *bi)
6048 {
6049         struct sk_buff *skb = bi->skb;
6050         dma_addr_t dma = bi->dma;
6051
6052         if (dma)
6053                 return true;
6054
6055         if (likely(!skb)) {
6056                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6057                                                 IGB_RX_HDR_LEN);
6058                 bi->skb = skb;
6059                 if (!skb) {
6060                         rx_ring->rx_stats.alloc_failed++;
6061                         return false;
6062                 }
6063
6064                 /* initialize skb for ring */
6065                 skb_record_rx_queue(skb, rx_ring->queue_index);
6066         }
6067
6068         dma = dma_map_single(rx_ring->dev, skb->data,
6069                              IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6070
6071         if (dma_mapping_error(rx_ring->dev, dma)) {
6072                 rx_ring->rx_stats.alloc_failed++;
6073                 return false;
6074         }
6075
6076         bi->dma = dma;
6077         return true;
6078 }
6079
6080 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6081                                   struct igb_rx_buffer *bi)
6082 {
6083         struct page *page = bi->page;
6084         dma_addr_t page_dma = bi->page_dma;
6085         unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6086
6087         if (page_dma)
6088                 return true;
6089
6090         if (!page) {
6091                 page = __skb_alloc_page(GFP_ATOMIC, bi->skb);
6092                 bi->page = page;
6093                 if (unlikely(!page)) {
6094                         rx_ring->rx_stats.alloc_failed++;
6095                         return false;
6096                 }
6097         }
6098
6099         page_dma = dma_map_page(rx_ring->dev, page,
6100                                 page_offset, PAGE_SIZE / 2,
6101                                 DMA_FROM_DEVICE);
6102
6103         if (dma_mapping_error(rx_ring->dev, page_dma)) {
6104                 rx_ring->rx_stats.alloc_failed++;
6105                 return false;
6106         }
6107
6108         bi->page_dma = page_dma;
6109         bi->page_offset = page_offset;
6110         return true;
6111 }
6112
6113 /**
6114  * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6115  * @adapter: address of board private structure
6116  **/
6117 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6118 {
6119         union e1000_adv_rx_desc *rx_desc;
6120         struct igb_rx_buffer *bi;
6121         u16 i = rx_ring->next_to_use;
6122
6123         rx_desc = IGB_RX_DESC(rx_ring, i);
6124         bi = &rx_ring->rx_buffer_info[i];
6125         i -= rx_ring->count;
6126
6127         while (cleaned_count--) {
6128                 if (!igb_alloc_mapped_skb(rx_ring, bi))
6129                         break;
6130
6131                 /* Refresh the desc even if buffer_addrs didn't change
6132                  * because each write-back erases this info. */
6133                 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
6134
6135                 if (!igb_alloc_mapped_page(rx_ring, bi))
6136                         break;
6137
6138                 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
6139
6140                 rx_desc++;
6141                 bi++;
6142                 i++;
6143                 if (unlikely(!i)) {
6144                         rx_desc = IGB_RX_DESC(rx_ring, 0);
6145                         bi = rx_ring->rx_buffer_info;
6146                         i -= rx_ring->count;
6147                 }
6148
6149                 /* clear the hdr_addr for the next_to_use descriptor */
6150                 rx_desc->read.hdr_addr = 0;
6151         }
6152
6153         i += rx_ring->count;
6154
6155         if (rx_ring->next_to_use != i) {
6156                 rx_ring->next_to_use = i;
6157
6158                 /* Force memory writes to complete before letting h/w
6159                  * know there are new descriptors to fetch.  (Only
6160                  * applicable for weak-ordered memory model archs,
6161                  * such as IA-64). */
6162                 wmb();
6163                 writel(i, rx_ring->tail);
6164         }
6165 }
6166
6167 /**
6168  * igb_mii_ioctl -
6169  * @netdev:
6170  * @ifreq:
6171  * @cmd:
6172  **/
6173 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6174 {
6175         struct igb_adapter *adapter = netdev_priv(netdev);
6176         struct mii_ioctl_data *data = if_mii(ifr);
6177
6178         if (adapter->hw.phy.media_type != e1000_media_type_copper)
6179                 return -EOPNOTSUPP;
6180
6181         switch (cmd) {
6182         case SIOCGMIIPHY:
6183                 data->phy_id = adapter->hw.phy.addr;
6184                 break;
6185         case SIOCGMIIREG:
6186                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6187                                      &data->val_out))
6188                         return -EIO;
6189                 break;
6190         case SIOCSMIIREG:
6191         default:
6192                 return -EOPNOTSUPP;
6193         }
6194         return 0;
6195 }
6196
6197 /**
6198  * igb_ioctl -
6199  * @netdev:
6200  * @ifreq:
6201  * @cmd:
6202  **/
6203 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6204 {
6205         switch (cmd) {
6206         case SIOCGMIIPHY:
6207         case SIOCGMIIREG:
6208         case SIOCSMIIREG:
6209                 return igb_mii_ioctl(netdev, ifr, cmd);
6210 #ifdef CONFIG_IGB_PTP
6211         case SIOCSHWTSTAMP:
6212                 return igb_ptp_hwtstamp_ioctl(netdev, ifr, cmd);
6213 #endif /* CONFIG_IGB_PTP */
6214         default:
6215                 return -EOPNOTSUPP;
6216         }
6217 }
6218
6219 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6220 {
6221         struct igb_adapter *adapter = hw->back;
6222
6223         if (pcie_capability_read_word(adapter->pdev, reg, value))
6224                 return -E1000_ERR_CONFIG;
6225
6226         return 0;
6227 }
6228
6229 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6230 {
6231         struct igb_adapter *adapter = hw->back;
6232
6233         if (pcie_capability_write_word(adapter->pdev, reg, *value))
6234                 return -E1000_ERR_CONFIG;
6235
6236         return 0;
6237 }
6238
6239 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6240 {
6241         struct igb_adapter *adapter = netdev_priv(netdev);
6242         struct e1000_hw *hw = &adapter->hw;
6243         u32 ctrl, rctl;
6244         bool enable = !!(features & NETIF_F_HW_VLAN_RX);
6245
6246         if (enable) {
6247                 /* enable VLAN tag insert/strip */
6248                 ctrl = rd32(E1000_CTRL);
6249                 ctrl |= E1000_CTRL_VME;
6250                 wr32(E1000_CTRL, ctrl);
6251
6252                 /* Disable CFI check */
6253                 rctl = rd32(E1000_RCTL);
6254                 rctl &= ~E1000_RCTL_CFIEN;
6255                 wr32(E1000_RCTL, rctl);
6256         } else {
6257                 /* disable VLAN tag insert/strip */
6258                 ctrl = rd32(E1000_CTRL);
6259                 ctrl &= ~E1000_CTRL_VME;
6260                 wr32(E1000_CTRL, ctrl);
6261         }
6262
6263         igb_rlpml_set(adapter);
6264 }
6265
6266 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6267 {
6268         struct igb_adapter *adapter = netdev_priv(netdev);
6269         struct e1000_hw *hw = &adapter->hw;
6270         int pf_id = adapter->vfs_allocated_count;
6271
6272         /* attempt to add filter to vlvf array */
6273         igb_vlvf_set(adapter, vid, true, pf_id);
6274
6275         /* add the filter since PF can receive vlans w/o entry in vlvf */
6276         igb_vfta_set(hw, vid, true);
6277
6278         set_bit(vid, adapter->active_vlans);
6279
6280         return 0;
6281 }
6282
6283 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6284 {
6285         struct igb_adapter *adapter = netdev_priv(netdev);
6286         struct e1000_hw *hw = &adapter->hw;
6287         int pf_id = adapter->vfs_allocated_count;
6288         s32 err;
6289
6290         /* remove vlan from VLVF table array */
6291         err = igb_vlvf_set(adapter, vid, false, pf_id);
6292
6293         /* if vid was not present in VLVF just remove it from table */
6294         if (err)
6295                 igb_vfta_set(hw, vid, false);
6296
6297         clear_bit(vid, adapter->active_vlans);
6298
6299         return 0;
6300 }
6301
6302 static void igb_restore_vlan(struct igb_adapter *adapter)
6303 {
6304         u16 vid;
6305
6306         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6307
6308         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6309                 igb_vlan_rx_add_vid(adapter->netdev, vid);
6310 }
6311
6312 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6313 {
6314         struct pci_dev *pdev = adapter->pdev;
6315         struct e1000_mac_info *mac = &adapter->hw.mac;
6316
6317         mac->autoneg = 0;
6318
6319         /* Make sure dplx is at most 1 bit and lsb of speed is not set
6320          * for the switch() below to work */
6321         if ((spd & 1) || (dplx & ~1))
6322                 goto err_inval;
6323
6324         /* Fiber NIC's only allow 1000 Gbps Full duplex */
6325         if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6326             spd != SPEED_1000 &&
6327             dplx != DUPLEX_FULL)
6328                 goto err_inval;
6329
6330         switch (spd + dplx) {
6331         case SPEED_10 + DUPLEX_HALF:
6332                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6333                 break;
6334         case SPEED_10 + DUPLEX_FULL:
6335                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6336                 break;
6337         case SPEED_100 + DUPLEX_HALF:
6338                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6339                 break;
6340         case SPEED_100 + DUPLEX_FULL:
6341                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6342                 break;
6343         case SPEED_1000 + DUPLEX_FULL:
6344                 mac->autoneg = 1;
6345                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6346                 break;
6347         case SPEED_1000 + DUPLEX_HALF: /* not supported */
6348         default:
6349                 goto err_inval;
6350         }
6351
6352         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
6353         adapter->hw.phy.mdix = AUTO_ALL_MODES;
6354
6355         return 0;
6356
6357 err_inval:
6358         dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6359         return -EINVAL;
6360 }
6361
6362 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6363                           bool runtime)
6364 {
6365         struct net_device *netdev = pci_get_drvdata(pdev);
6366         struct igb_adapter *adapter = netdev_priv(netdev);
6367         struct e1000_hw *hw = &adapter->hw;
6368         u32 ctrl, rctl, status;
6369         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
6370 #ifdef CONFIG_PM
6371         int retval = 0;
6372 #endif
6373
6374         netif_device_detach(netdev);
6375
6376         if (netif_running(netdev))
6377                 __igb_close(netdev, true);
6378
6379         igb_clear_interrupt_scheme(adapter);
6380
6381 #ifdef CONFIG_PM
6382         retval = pci_save_state(pdev);
6383         if (retval)
6384                 return retval;
6385 #endif
6386
6387         status = rd32(E1000_STATUS);
6388         if (status & E1000_STATUS_LU)
6389                 wufc &= ~E1000_WUFC_LNKC;
6390
6391         if (wufc) {
6392                 igb_setup_rctl(adapter);
6393                 igb_set_rx_mode(netdev);
6394
6395                 /* turn on all-multi mode if wake on multicast is enabled */
6396                 if (wufc & E1000_WUFC_MC) {
6397                         rctl = rd32(E1000_RCTL);
6398                         rctl |= E1000_RCTL_MPE;
6399                         wr32(E1000_RCTL, rctl);
6400                 }
6401
6402                 ctrl = rd32(E1000_CTRL);
6403                 /* advertise wake from D3Cold */
6404                 #define E1000_CTRL_ADVD3WUC 0x00100000
6405                 /* phy power management enable */
6406                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6407                 ctrl |= E1000_CTRL_ADVD3WUC;
6408                 wr32(E1000_CTRL, ctrl);
6409
6410                 /* Allow time for pending master requests to run */
6411                 igb_disable_pcie_master(hw);
6412
6413                 wr32(E1000_WUC, E1000_WUC_PME_EN);
6414                 wr32(E1000_WUFC, wufc);
6415         } else {
6416                 wr32(E1000_WUC, 0);
6417                 wr32(E1000_WUFC, 0);
6418         }
6419
6420         *enable_wake = wufc || adapter->en_mng_pt;
6421         if (!*enable_wake)
6422                 igb_power_down_link(adapter);
6423         else
6424                 igb_power_up_link(adapter);
6425
6426         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6427          * would have already happened in close and is redundant. */
6428         igb_release_hw_control(adapter);
6429
6430         pci_disable_device(pdev);
6431
6432         return 0;
6433 }
6434
6435 #ifdef CONFIG_PM
6436 #ifdef CONFIG_PM_SLEEP
6437 static int igb_suspend(struct device *dev)
6438 {
6439         int retval;
6440         bool wake;
6441         struct pci_dev *pdev = to_pci_dev(dev);
6442
6443         retval = __igb_shutdown(pdev, &wake, 0);
6444         if (retval)
6445                 return retval;
6446
6447         if (wake) {
6448                 pci_prepare_to_sleep(pdev);
6449         } else {
6450                 pci_wake_from_d3(pdev, false);
6451                 pci_set_power_state(pdev, PCI_D3hot);
6452         }
6453
6454         return 0;
6455 }
6456 #endif /* CONFIG_PM_SLEEP */
6457
6458 static int igb_resume(struct device *dev)
6459 {
6460         struct pci_dev *pdev = to_pci_dev(dev);
6461         struct net_device *netdev = pci_get_drvdata(pdev);
6462         struct igb_adapter *adapter = netdev_priv(netdev);
6463         struct e1000_hw *hw = &adapter->hw;
6464         u32 err;
6465
6466         pci_set_power_state(pdev, PCI_D0);
6467         pci_restore_state(pdev);
6468         pci_save_state(pdev);
6469
6470         err = pci_enable_device_mem(pdev);
6471         if (err) {
6472                 dev_err(&pdev->dev,
6473                         "igb: Cannot enable PCI device from suspend\n");
6474                 return err;
6475         }
6476         pci_set_master(pdev);
6477
6478         pci_enable_wake(pdev, PCI_D3hot, 0);
6479         pci_enable_wake(pdev, PCI_D3cold, 0);
6480
6481         if (igb_init_interrupt_scheme(adapter)) {
6482                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6483                 return -ENOMEM;
6484         }
6485
6486         igb_reset(adapter);
6487
6488         /* let the f/w know that the h/w is now under the control of the
6489          * driver. */
6490         igb_get_hw_control(adapter);
6491
6492         wr32(E1000_WUS, ~0);
6493
6494         if (netdev->flags & IFF_UP) {
6495                 err = __igb_open(netdev, true);
6496                 if (err)
6497                         return err;
6498         }
6499
6500         netif_device_attach(netdev);
6501         return 0;
6502 }
6503
6504 #ifdef CONFIG_PM_RUNTIME
6505 static int igb_runtime_idle(struct device *dev)
6506 {
6507         struct pci_dev *pdev = to_pci_dev(dev);
6508         struct net_device *netdev = pci_get_drvdata(pdev);
6509         struct igb_adapter *adapter = netdev_priv(netdev);
6510
6511         if (!igb_has_link(adapter))
6512                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6513
6514         return -EBUSY;
6515 }
6516
6517 static int igb_runtime_suspend(struct device *dev)
6518 {
6519         struct pci_dev *pdev = to_pci_dev(dev);
6520         int retval;
6521         bool wake;
6522
6523         retval = __igb_shutdown(pdev, &wake, 1);
6524         if (retval)
6525                 return retval;
6526
6527         if (wake) {
6528                 pci_prepare_to_sleep(pdev);
6529         } else {
6530                 pci_wake_from_d3(pdev, false);
6531                 pci_set_power_state(pdev, PCI_D3hot);
6532         }
6533
6534         return 0;
6535 }
6536
6537 static int igb_runtime_resume(struct device *dev)
6538 {
6539         return igb_resume(dev);
6540 }
6541 #endif /* CONFIG_PM_RUNTIME */
6542 #endif
6543
6544 static void igb_shutdown(struct pci_dev *pdev)
6545 {
6546         bool wake;
6547
6548         __igb_shutdown(pdev, &wake, 0);
6549
6550         if (system_state == SYSTEM_POWER_OFF) {
6551                 pci_wake_from_d3(pdev, wake);
6552                 pci_set_power_state(pdev, PCI_D3hot);
6553         }
6554 }
6555
6556 #ifdef CONFIG_NET_POLL_CONTROLLER
6557 /*
6558  * Polling 'interrupt' - used by things like netconsole to send skbs
6559  * without having to re-enable interrupts. It's not called while
6560  * the interrupt routine is executing.
6561  */
6562 static void igb_netpoll(struct net_device *netdev)
6563 {
6564         struct igb_adapter *adapter = netdev_priv(netdev);
6565         struct e1000_hw *hw = &adapter->hw;
6566         struct igb_q_vector *q_vector;
6567         int i;
6568
6569         for (i = 0; i < adapter->num_q_vectors; i++) {
6570                 q_vector = adapter->q_vector[i];
6571                 if (adapter->msix_entries)
6572                         wr32(E1000_EIMC, q_vector->eims_value);
6573                 else
6574                         igb_irq_disable(adapter);
6575                 napi_schedule(&q_vector->napi);
6576         }
6577 }
6578 #endif /* CONFIG_NET_POLL_CONTROLLER */
6579
6580 /**
6581  * igb_io_error_detected - called when PCI error is detected
6582  * @pdev: Pointer to PCI device
6583  * @state: The current pci connection state
6584  *
6585  * This function is called after a PCI bus error affecting
6586  * this device has been detected.
6587  */
6588 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6589                                               pci_channel_state_t state)
6590 {
6591         struct net_device *netdev = pci_get_drvdata(pdev);
6592         struct igb_adapter *adapter = netdev_priv(netdev);
6593
6594         netif_device_detach(netdev);
6595
6596         if (state == pci_channel_io_perm_failure)
6597                 return PCI_ERS_RESULT_DISCONNECT;
6598
6599         if (netif_running(netdev))
6600                 igb_down(adapter);
6601         pci_disable_device(pdev);
6602
6603         /* Request a slot slot reset. */
6604         return PCI_ERS_RESULT_NEED_RESET;
6605 }
6606
6607 /**
6608  * igb_io_slot_reset - called after the pci bus has been reset.
6609  * @pdev: Pointer to PCI device
6610  *
6611  * Restart the card from scratch, as if from a cold-boot. Implementation
6612  * resembles the first-half of the igb_resume routine.
6613  */
6614 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6615 {
6616         struct net_device *netdev = pci_get_drvdata(pdev);
6617         struct igb_adapter *adapter = netdev_priv(netdev);
6618         struct e1000_hw *hw = &adapter->hw;
6619         pci_ers_result_t result;
6620         int err;
6621
6622         if (pci_enable_device_mem(pdev)) {
6623                 dev_err(&pdev->dev,
6624                         "Cannot re-enable PCI device after reset.\n");
6625                 result = PCI_ERS_RESULT_DISCONNECT;
6626         } else {
6627                 pci_set_master(pdev);
6628                 pci_restore_state(pdev);
6629                 pci_save_state(pdev);
6630
6631                 pci_enable_wake(pdev, PCI_D3hot, 0);
6632                 pci_enable_wake(pdev, PCI_D3cold, 0);
6633
6634                 igb_reset(adapter);
6635                 wr32(E1000_WUS, ~0);
6636                 result = PCI_ERS_RESULT_RECOVERED;
6637         }
6638
6639         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6640         if (err) {
6641                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6642                         "failed 0x%0x\n", err);
6643                 /* non-fatal, continue */
6644         }
6645
6646         return result;
6647 }
6648
6649 /**
6650  * igb_io_resume - called when traffic can start flowing again.
6651  * @pdev: Pointer to PCI device
6652  *
6653  * This callback is called when the error recovery driver tells us that
6654  * its OK to resume normal operation. Implementation resembles the
6655  * second-half of the igb_resume routine.
6656  */
6657 static void igb_io_resume(struct pci_dev *pdev)
6658 {
6659         struct net_device *netdev = pci_get_drvdata(pdev);
6660         struct igb_adapter *adapter = netdev_priv(netdev);
6661
6662         if (netif_running(netdev)) {
6663                 if (igb_up(adapter)) {
6664                         dev_err(&pdev->dev, "igb_up failed after reset\n");
6665                         return;
6666                 }
6667         }
6668
6669         netif_device_attach(netdev);
6670
6671         /* let the f/w know that the h/w is now under the control of the
6672          * driver. */
6673         igb_get_hw_control(adapter);
6674 }
6675
6676 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6677                              u8 qsel)
6678 {
6679         u32 rar_low, rar_high;
6680         struct e1000_hw *hw = &adapter->hw;
6681
6682         /* HW expects these in little endian so we reverse the byte order
6683          * from network order (big endian) to little endian
6684          */
6685         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6686                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6687         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6688
6689         /* Indicate to hardware the Address is Valid. */
6690         rar_high |= E1000_RAH_AV;
6691
6692         if (hw->mac.type == e1000_82575)
6693                 rar_high |= E1000_RAH_POOL_1 * qsel;
6694         else
6695                 rar_high |= E1000_RAH_POOL_1 << qsel;
6696
6697         wr32(E1000_RAL(index), rar_low);
6698         wrfl();
6699         wr32(E1000_RAH(index), rar_high);
6700         wrfl();
6701 }
6702
6703 static int igb_set_vf_mac(struct igb_adapter *adapter,
6704                           int vf, unsigned char *mac_addr)
6705 {
6706         struct e1000_hw *hw = &adapter->hw;
6707         /* VF MAC addresses start at end of receive addresses and moves
6708          * torwards the first, as a result a collision should not be possible */
6709         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6710
6711         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6712
6713         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6714
6715         return 0;
6716 }
6717
6718 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6719 {
6720         struct igb_adapter *adapter = netdev_priv(netdev);
6721         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6722                 return -EINVAL;
6723         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6724         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6725         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6726                                       " change effective.");
6727         if (test_bit(__IGB_DOWN, &adapter->state)) {
6728                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6729                          " but the PF device is not up.\n");
6730                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6731                          " attempting to use the VF device.\n");
6732         }
6733         return igb_set_vf_mac(adapter, vf, mac);
6734 }
6735
6736 static int igb_link_mbps(int internal_link_speed)
6737 {
6738         switch (internal_link_speed) {
6739         case SPEED_100:
6740                 return 100;
6741         case SPEED_1000:
6742                 return 1000;
6743         default:
6744                 return 0;
6745         }
6746 }
6747
6748 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6749                                   int link_speed)
6750 {
6751         int rf_dec, rf_int;
6752         u32 bcnrc_val;
6753
6754         if (tx_rate != 0) {
6755                 /* Calculate the rate factor values to set */
6756                 rf_int = link_speed / tx_rate;
6757                 rf_dec = (link_speed - (rf_int * tx_rate));
6758                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6759
6760                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6761                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6762                                E1000_RTTBCNRC_RF_INT_MASK);
6763                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6764         } else {
6765                 bcnrc_val = 0;
6766         }
6767
6768         wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6769         /*
6770          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
6771          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
6772          */
6773         wr32(E1000_RTTBCNRM, 0x14);
6774         wr32(E1000_RTTBCNRC, bcnrc_val);
6775 }
6776
6777 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6778 {
6779         int actual_link_speed, i;
6780         bool reset_rate = false;
6781
6782         /* VF TX rate limit was not set or not supported */
6783         if ((adapter->vf_rate_link_speed == 0) ||
6784             (adapter->hw.mac.type != e1000_82576))
6785                 return;
6786
6787         actual_link_speed = igb_link_mbps(adapter->link_speed);
6788         if (actual_link_speed != adapter->vf_rate_link_speed) {
6789                 reset_rate = true;
6790                 adapter->vf_rate_link_speed = 0;
6791                 dev_info(&adapter->pdev->dev,
6792                          "Link speed has been changed. VF Transmit "
6793                          "rate is disabled\n");
6794         }
6795
6796         for (i = 0; i < adapter->vfs_allocated_count; i++) {
6797                 if (reset_rate)
6798                         adapter->vf_data[i].tx_rate = 0;
6799
6800                 igb_set_vf_rate_limit(&adapter->hw, i,
6801                                       adapter->vf_data[i].tx_rate,
6802                                       actual_link_speed);
6803         }
6804 }
6805
6806 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6807 {
6808         struct igb_adapter *adapter = netdev_priv(netdev);
6809         struct e1000_hw *hw = &adapter->hw;
6810         int actual_link_speed;
6811
6812         if (hw->mac.type != e1000_82576)
6813                 return -EOPNOTSUPP;
6814
6815         actual_link_speed = igb_link_mbps(adapter->link_speed);
6816         if ((vf >= adapter->vfs_allocated_count) ||
6817             (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6818             (tx_rate < 0) || (tx_rate > actual_link_speed))
6819                 return -EINVAL;
6820
6821         adapter->vf_rate_link_speed = actual_link_speed;
6822         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6823         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6824
6825         return 0;
6826 }
6827
6828 static int igb_ndo_get_vf_config(struct net_device *netdev,
6829                                  int vf, struct ifla_vf_info *ivi)
6830 {
6831         struct igb_adapter *adapter = netdev_priv(netdev);
6832         if (vf >= adapter->vfs_allocated_count)
6833                 return -EINVAL;
6834         ivi->vf = vf;
6835         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6836         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
6837         ivi->vlan = adapter->vf_data[vf].pf_vlan;
6838         ivi->qos = adapter->vf_data[vf].pf_qos;
6839         return 0;
6840 }
6841
6842 static void igb_vmm_control(struct igb_adapter *adapter)
6843 {
6844         struct e1000_hw *hw = &adapter->hw;
6845         u32 reg;
6846
6847         switch (hw->mac.type) {
6848         case e1000_82575:
6849         case e1000_i210:
6850         case e1000_i211:
6851         default:
6852                 /* replication is not supported for 82575 */
6853                 return;
6854         case e1000_82576:
6855                 /* notify HW that the MAC is adding vlan tags */
6856                 reg = rd32(E1000_DTXCTL);
6857                 reg |= E1000_DTXCTL_VLAN_ADDED;
6858                 wr32(E1000_DTXCTL, reg);
6859         case e1000_82580:
6860                 /* enable replication vlan tag stripping */
6861                 reg = rd32(E1000_RPLOLR);
6862                 reg |= E1000_RPLOLR_STRVLAN;
6863                 wr32(E1000_RPLOLR, reg);
6864         case e1000_i350:
6865                 /* none of the above registers are supported by i350 */
6866                 break;
6867         }
6868
6869         if (adapter->vfs_allocated_count) {
6870                 igb_vmdq_set_loopback_pf(hw, true);
6871                 igb_vmdq_set_replication_pf(hw, true);
6872                 igb_vmdq_set_anti_spoofing_pf(hw, true,
6873                                                 adapter->vfs_allocated_count);
6874         } else {
6875                 igb_vmdq_set_loopback_pf(hw, false);
6876                 igb_vmdq_set_replication_pf(hw, false);
6877         }
6878 }
6879
6880 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
6881 {
6882         struct e1000_hw *hw = &adapter->hw;
6883         u32 dmac_thr;
6884         u16 hwm;
6885
6886         if (hw->mac.type > e1000_82580) {
6887                 if (adapter->flags & IGB_FLAG_DMAC) {
6888                         u32 reg;
6889
6890                         /* force threshold to 0. */
6891                         wr32(E1000_DMCTXTH, 0);
6892
6893                         /*
6894                          * DMA Coalescing high water mark needs to be greater
6895                          * than the Rx threshold. Set hwm to PBA - max frame
6896                          * size in 16B units, capping it at PBA - 6KB.
6897                          */
6898                         hwm = 64 * pba - adapter->max_frame_size / 16;
6899                         if (hwm < 64 * (pba - 6))
6900                                 hwm = 64 * (pba - 6);
6901                         reg = rd32(E1000_FCRTC);
6902                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
6903                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
6904                                 & E1000_FCRTC_RTH_COAL_MASK);
6905                         wr32(E1000_FCRTC, reg);
6906
6907                         /*
6908                          * Set the DMA Coalescing Rx threshold to PBA - 2 * max
6909                          * frame size, capping it at PBA - 10KB.
6910                          */
6911                         dmac_thr = pba - adapter->max_frame_size / 512;
6912                         if (dmac_thr < pba - 10)
6913                                 dmac_thr = pba - 10;
6914                         reg = rd32(E1000_DMACR);
6915                         reg &= ~E1000_DMACR_DMACTHR_MASK;
6916                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
6917                                 & E1000_DMACR_DMACTHR_MASK);
6918
6919                         /* transition to L0x or L1 if available..*/
6920                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
6921
6922                         /* watchdog timer= +-1000 usec in 32usec intervals */
6923                         reg |= (1000 >> 5);
6924
6925                         /* Disable BMC-to-OS Watchdog Enable */
6926                         reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
6927                         wr32(E1000_DMACR, reg);
6928
6929                         /*
6930                          * no lower threshold to disable
6931                          * coalescing(smart fifb)-UTRESH=0
6932                          */
6933                         wr32(E1000_DMCRTRH, 0);
6934
6935                         reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
6936
6937                         wr32(E1000_DMCTLX, reg);
6938
6939                         /*
6940                          * free space in tx packet buffer to wake from
6941                          * DMA coal
6942                          */
6943                         wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
6944                              (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
6945
6946                         /*
6947                          * make low power state decision controlled
6948                          * by DMA coal
6949                          */
6950                         reg = rd32(E1000_PCIEMISC);
6951                         reg &= ~E1000_PCIEMISC_LX_DECISION;
6952                         wr32(E1000_PCIEMISC, reg);
6953                 } /* endif adapter->dmac is not disabled */
6954         } else if (hw->mac.type == e1000_82580) {
6955                 u32 reg = rd32(E1000_PCIEMISC);
6956                 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
6957                 wr32(E1000_DMACR, 0);
6958         }
6959 }
6960
6961 /* igb_main.c */