1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <linux/slab.h>
36 #include <net/checksum.h>
37 #include <net/ip6_checksum.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
41 #include <linux/if_vlan.h>
42 #include <linux/pci.h>
43 #include <linux/pci-aspm.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/if_ether.h>
47 #include <linux/aer.h>
49 #include <linux/dca.h>
53 #define DRV_VERSION "2.1.0-k2"
54 char igb_driver_name[] = "igb";
55 char igb_driver_version[] = DRV_VERSION;
56 static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
60 static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
89 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
91 void igb_reset(struct igb_adapter *);
92 static int igb_setup_all_tx_resources(struct igb_adapter *);
93 static int igb_setup_all_rx_resources(struct igb_adapter *);
94 static void igb_free_all_tx_resources(struct igb_adapter *);
95 static void igb_free_all_rx_resources(struct igb_adapter *);
96 static void igb_setup_mrqc(struct igb_adapter *);
97 void igb_update_stats(struct igb_adapter *);
98 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99 static void __devexit igb_remove(struct pci_dev *pdev);
100 static int igb_sw_init(struct igb_adapter *);
101 static int igb_open(struct net_device *);
102 static int igb_close(struct net_device *);
103 static void igb_configure_tx(struct igb_adapter *);
104 static void igb_configure_rx(struct igb_adapter *);
105 static void igb_clean_all_tx_rings(struct igb_adapter *);
106 static void igb_clean_all_rx_rings(struct igb_adapter *);
107 static void igb_clean_tx_ring(struct igb_ring *);
108 static void igb_clean_rx_ring(struct igb_ring *);
109 static void igb_set_rx_mode(struct net_device *);
110 static void igb_update_phy_info(unsigned long);
111 static void igb_watchdog(unsigned long);
112 static void igb_watchdog_task(struct work_struct *);
113 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
114 static struct net_device_stats *igb_get_stats(struct net_device *);
115 static int igb_change_mtu(struct net_device *, int);
116 static int igb_set_mac(struct net_device *, void *);
117 static void igb_set_uta(struct igb_adapter *adapter);
118 static irqreturn_t igb_intr(int irq, void *);
119 static irqreturn_t igb_intr_msi(int irq, void *);
120 static irqreturn_t igb_msix_other(int irq, void *);
121 static irqreturn_t igb_msix_ring(int irq, void *);
122 #ifdef CONFIG_IGB_DCA
123 static void igb_update_dca(struct igb_q_vector *);
124 static void igb_setup_dca(struct igb_adapter *);
125 #endif /* CONFIG_IGB_DCA */
126 static bool igb_clean_tx_irq(struct igb_q_vector *);
127 static int igb_poll(struct napi_struct *, int);
128 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
129 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130 static void igb_tx_timeout(struct net_device *);
131 static void igb_reset_task(struct work_struct *);
132 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133 static void igb_vlan_rx_add_vid(struct net_device *, u16);
134 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135 static void igb_restore_vlan(struct igb_adapter *);
136 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
137 static void igb_ping_all_vfs(struct igb_adapter *);
138 static void igb_msg_task(struct igb_adapter *);
139 static void igb_vmm_control(struct igb_adapter *);
140 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
141 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
150 static int igb_suspend(struct pci_dev *, pm_message_t);
151 static int igb_resume(struct pci_dev *);
153 static void igb_shutdown(struct pci_dev *);
154 #ifdef CONFIG_IGB_DCA
155 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156 static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
162 #ifdef CONFIG_NET_POLL_CONTROLLER
163 /* for netdump / net console */
164 static void igb_netpoll(struct net_device *);
166 #ifdef CONFIG_PCI_IOV
167 static unsigned int max_vfs = 0;
168 module_param(max_vfs, uint, 0);
169 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171 #endif /* CONFIG_PCI_IOV */
173 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176 static void igb_io_resume(struct pci_dev *);
178 static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
185 static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
189 .remove = __devexit_p(igb_remove),
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
199 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201 MODULE_LICENSE("GPL");
202 MODULE_VERSION(DRV_VERSION);
204 struct igb_reg_info {
209 static const struct igb_reg_info igb_reg_info_tbl[] = {
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
216 /* Interrupt Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
241 /* List Terminator */
246 * igb_regdump - register printout routine
248 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
254 switch (reginfo->ofs) {
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
317 * igb_dump - Print registers, tx-rings and rx-rings
319 static void igb_dump(struct igb_adapter *adapter)
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
334 if (!netif_msg_hw(adapter))
337 /* Print netdevice Info */
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
381 /* Transmit Descriptor Formats
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
409 (u64)buffer_info->dma,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
422 printk(KERN_CONT "\n");
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
432 /* Print RX Rings Summary */
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
443 if (!netif_msg_rx_status(adapter))
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
448 /* Advanced Receive Descriptor (Read) Format
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
457 * Advanced Receive Descriptor (Write-Back) Format
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
498 (u64)buffer_info->dma,
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
509 print_hex_dump(KERN_INFO, "",
513 buffer_info->page_dma +
514 buffer_info->page_offset),
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
524 printk(KERN_CONT "\n");
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
537 static cycle_t igb_read_clock(const struct cyclecounter *tc)
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
561 * igb_get_hw_dev - return device
562 * used by hardware layer to print debugging information
564 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
566 struct igb_adapter *adapter = hw->back;
567 return adapter->netdev;
571 * igb_init_module - Driver Registration Routine
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
576 static int __init igb_init_module(void)
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
582 printk(KERN_INFO "%s\n", igb_copyright);
584 #ifdef CONFIG_IGB_DCA
585 dca_register_notify(&dca_notifier);
587 ret = pci_register_driver(&igb_driver);
591 module_init(igb_init_module);
594 * igb_exit_module - Driver Exit Cleanup Routine
596 * igb_exit_module is called just before the driver is removed
599 static void __exit igb_exit_module(void)
601 #ifdef CONFIG_IGB_DCA
602 dca_unregister_notify(&dca_notifier);
604 pci_unregister_driver(&igb_driver);
607 module_exit(igb_exit_module);
609 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
617 static void igb_cache_ring_register(struct igb_adapter *adapter)
620 u32 rbase_offset = adapter->vfs_allocated_count;
622 switch (adapter->hw.mac.type) {
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
629 if (adapter->vfs_allocated_count) {
630 for (; i < adapter->rss_queues; i++)
631 adapter->rx_ring[i]->reg_idx = rbase_offset +
633 for (; j < adapter->rss_queues; j++)
634 adapter->tx_ring[j]->reg_idx = rbase_offset +
641 for (; i < adapter->num_rx_queues; i++)
642 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
643 for (; j < adapter->num_tx_queues; j++)
644 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
649 static void igb_free_queues(struct igb_adapter *adapter)
653 for (i = 0; i < adapter->num_tx_queues; i++) {
654 kfree(adapter->tx_ring[i]);
655 adapter->tx_ring[i] = NULL;
657 for (i = 0; i < adapter->num_rx_queues; i++) {
658 kfree(adapter->rx_ring[i]);
659 adapter->rx_ring[i] = NULL;
661 adapter->num_rx_queues = 0;
662 adapter->num_tx_queues = 0;
666 * igb_alloc_queues - Allocate memory for all rings
667 * @adapter: board private structure to initialize
669 * We allocate one ring per queue at run-time since we don't know the
670 * number of queues at compile-time.
672 static int igb_alloc_queues(struct igb_adapter *adapter)
674 struct igb_ring *ring;
677 for (i = 0; i < adapter->num_tx_queues; i++) {
678 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
681 ring->count = adapter->tx_ring_count;
682 ring->queue_index = i;
683 ring->dev = &adapter->pdev->dev;
684 ring->netdev = adapter->netdev;
685 /* For 82575, context index must be unique per ring. */
686 if (adapter->hw.mac.type == e1000_82575)
687 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
688 adapter->tx_ring[i] = ring;
691 for (i = 0; i < adapter->num_rx_queues; i++) {
692 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
695 ring->count = adapter->rx_ring_count;
696 ring->queue_index = i;
697 ring->dev = &adapter->pdev->dev;
698 ring->netdev = adapter->netdev;
699 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
700 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
701 /* set flag indicating ring supports SCTP checksum offload */
702 if (adapter->hw.mac.type >= e1000_82576)
703 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
704 adapter->rx_ring[i] = ring;
707 igb_cache_ring_register(adapter);
712 igb_free_queues(adapter);
717 #define IGB_N0_QUEUE -1
718 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
721 struct igb_adapter *adapter = q_vector->adapter;
722 struct e1000_hw *hw = &adapter->hw;
724 int rx_queue = IGB_N0_QUEUE;
725 int tx_queue = IGB_N0_QUEUE;
727 if (q_vector->rx_ring)
728 rx_queue = q_vector->rx_ring->reg_idx;
729 if (q_vector->tx_ring)
730 tx_queue = q_vector->tx_ring->reg_idx;
732 switch (hw->mac.type) {
734 /* The 82575 assigns vectors using a bitmask, which matches the
735 bitmask for the EICR/EIMS/EIMC registers. To assign one
736 or more queues to a vector, we write the appropriate bits
737 into the MSIXBM register for that vector. */
738 if (rx_queue > IGB_N0_QUEUE)
739 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
740 if (tx_queue > IGB_N0_QUEUE)
741 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
742 if (!adapter->msix_entries && msix_vector == 0)
743 msixbm |= E1000_EIMS_OTHER;
744 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
745 q_vector->eims_value = msixbm;
748 /* 82576 uses a table-based method for assigning vectors.
749 Each queue has a single entry in the table to which we write
750 a vector number along with a "valid" bit. Sadly, the layout
751 of the table is somewhat counterintuitive. */
752 if (rx_queue > IGB_N0_QUEUE) {
753 index = (rx_queue & 0x7);
754 ivar = array_rd32(E1000_IVAR0, index);
756 /* vector goes into low byte of register */
757 ivar = ivar & 0xFFFFFF00;
758 ivar |= msix_vector | E1000_IVAR_VALID;
760 /* vector goes into third byte of register */
761 ivar = ivar & 0xFF00FFFF;
762 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
764 array_wr32(E1000_IVAR0, index, ivar);
766 if (tx_queue > IGB_N0_QUEUE) {
767 index = (tx_queue & 0x7);
768 ivar = array_rd32(E1000_IVAR0, index);
770 /* vector goes into second byte of register */
771 ivar = ivar & 0xFFFF00FF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
774 /* vector goes into high byte of register */
775 ivar = ivar & 0x00FFFFFF;
776 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
778 array_wr32(E1000_IVAR0, index, ivar);
780 q_vector->eims_value = 1 << msix_vector;
784 /* 82580 uses the same table-based approach as 82576 but has fewer
785 entries as a result we carry over for queues greater than 4. */
786 if (rx_queue > IGB_N0_QUEUE) {
787 index = (rx_queue >> 1);
788 ivar = array_rd32(E1000_IVAR0, index);
789 if (rx_queue & 0x1) {
790 /* vector goes into third byte of register */
791 ivar = ivar & 0xFF00FFFF;
792 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
794 /* vector goes into low byte of register */
795 ivar = ivar & 0xFFFFFF00;
796 ivar |= msix_vector | E1000_IVAR_VALID;
798 array_wr32(E1000_IVAR0, index, ivar);
800 if (tx_queue > IGB_N0_QUEUE) {
801 index = (tx_queue >> 1);
802 ivar = array_rd32(E1000_IVAR0, index);
803 if (tx_queue & 0x1) {
804 /* vector goes into high byte of register */
805 ivar = ivar & 0x00FFFFFF;
806 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
808 /* vector goes into second byte of register */
809 ivar = ivar & 0xFFFF00FF;
810 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
812 array_wr32(E1000_IVAR0, index, ivar);
814 q_vector->eims_value = 1 << msix_vector;
821 /* add q_vector eims value to global eims_enable_mask */
822 adapter->eims_enable_mask |= q_vector->eims_value;
824 /* configure q_vector to set itr on first interrupt */
825 q_vector->set_itr = 1;
829 * igb_configure_msix - Configure MSI-X hardware
831 * igb_configure_msix sets up the hardware to properly
832 * generate MSI-X interrupts.
834 static void igb_configure_msix(struct igb_adapter *adapter)
838 struct e1000_hw *hw = &adapter->hw;
840 adapter->eims_enable_mask = 0;
842 /* set vector for other causes, i.e. link changes */
843 switch (hw->mac.type) {
845 tmp = rd32(E1000_CTRL_EXT);
846 /* enable MSI-X PBA support*/
847 tmp |= E1000_CTRL_EXT_PBA_CLR;
849 /* Auto-Mask interrupts upon ICR read. */
850 tmp |= E1000_CTRL_EXT_EIAME;
851 tmp |= E1000_CTRL_EXT_IRCA;
853 wr32(E1000_CTRL_EXT, tmp);
855 /* enable msix_other interrupt */
856 array_wr32(E1000_MSIXBM(0), vector++,
858 adapter->eims_other = E1000_EIMS_OTHER;
865 /* Turn on MSI-X capability first, or our settings
866 * won't stick. And it will take days to debug. */
867 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
868 E1000_GPIE_PBA | E1000_GPIE_EIAME |
871 /* enable msix_other interrupt */
872 adapter->eims_other = 1 << vector;
873 tmp = (vector++ | E1000_IVAR_VALID) << 8;
875 wr32(E1000_IVAR_MISC, tmp);
878 /* do nothing, since nothing else supports MSI-X */
880 } /* switch (hw->mac.type) */
882 adapter->eims_enable_mask |= adapter->eims_other;
884 for (i = 0; i < adapter->num_q_vectors; i++)
885 igb_assign_vector(adapter->q_vector[i], vector++);
891 * igb_request_msix - Initialize MSI-X interrupts
893 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
896 static int igb_request_msix(struct igb_adapter *adapter)
898 struct net_device *netdev = adapter->netdev;
899 struct e1000_hw *hw = &adapter->hw;
900 int i, err = 0, vector = 0;
902 err = request_irq(adapter->msix_entries[vector].vector,
903 igb_msix_other, 0, netdev->name, adapter);
908 for (i = 0; i < adapter->num_q_vectors; i++) {
909 struct igb_q_vector *q_vector = adapter->q_vector[i];
911 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
913 if (q_vector->rx_ring && q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
915 q_vector->rx_ring->queue_index);
916 else if (q_vector->tx_ring)
917 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
918 q_vector->tx_ring->queue_index);
919 else if (q_vector->rx_ring)
920 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
921 q_vector->rx_ring->queue_index);
923 sprintf(q_vector->name, "%s-unused", netdev->name);
925 err = request_irq(adapter->msix_entries[vector].vector,
926 igb_msix_ring, 0, q_vector->name,
933 igb_configure_msix(adapter);
939 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
941 if (adapter->msix_entries) {
942 pci_disable_msix(adapter->pdev);
943 kfree(adapter->msix_entries);
944 adapter->msix_entries = NULL;
945 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
946 pci_disable_msi(adapter->pdev);
951 * igb_free_q_vectors - Free memory allocated for interrupt vectors
952 * @adapter: board private structure to initialize
954 * This function frees the memory allocated to the q_vectors. In addition if
955 * NAPI is enabled it will delete any references to the NAPI struct prior
956 * to freeing the q_vector.
958 static void igb_free_q_vectors(struct igb_adapter *adapter)
962 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
963 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
964 adapter->q_vector[v_idx] = NULL;
967 netif_napi_del(&q_vector->napi);
970 adapter->num_q_vectors = 0;
974 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
976 * This function resets the device so that it has 0 rx queues, tx queues, and
977 * MSI-X interrupts allocated.
979 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
981 igb_free_queues(adapter);
982 igb_free_q_vectors(adapter);
983 igb_reset_interrupt_capability(adapter);
987 * igb_set_interrupt_capability - set MSI or MSI-X if supported
989 * Attempt to configure interrupts using the best available
990 * capabilities of the hardware and kernel.
992 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
997 /* Number of supported queues. */
998 adapter->num_rx_queues = adapter->rss_queues;
999 adapter->num_tx_queues = adapter->rss_queues;
1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1004 /* if tx handler is separate add 1 for every tx queue */
1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1011 /* add 1 vector for link status interrupts */
1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1015 if (!adapter->msix_entries)
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1027 igb_reset_interrupt_capability(adapter);
1029 /* If we can't do MSI-X, try MSI */
1031 #ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 adapter->vfs_allocated_count = 0;
1047 adapter->rss_queues = 1;
1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1049 adapter->num_rx_queues = 1;
1050 adapter->num_tx_queues = 1;
1051 adapter->num_q_vectors = 1;
1052 if (!pci_enable_msi(adapter->pdev))
1053 adapter->flags |= IGB_FLAG_HAS_MSI;
1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1066 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1076 q_vector->adapter = adapter;
1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1085 igb_free_q_vectors(adapter);
1089 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1095 q_vector->rx_ring->q_vector = q_vector;
1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
1101 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1107 q_vector->tx_ring->q_vector = q_vector;
1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1116 * This function maps the recently allocated queues to vectors.
1118 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1148 * This function initializes the interrupts and allocates all of the queues.
1150 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1152 struct pci_dev *pdev = adapter->pdev;
1155 igb_set_interrupt_capability(adapter);
1157 err = igb_alloc_q_vectors(adapter);
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1163 err = igb_alloc_queues(adapter);
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1169 err = igb_map_ring_to_vector(adapter);
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1178 igb_free_queues(adapter);
1180 igb_free_q_vectors(adapter);
1181 err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1187 * igb_request_irq - initialize interrupts
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1192 static int igb_request_irq(struct igb_adapter *adapter)
1194 struct net_device *netdev = adapter->netdev;
1195 struct pci_dev *pdev = adapter->pdev;
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
1202 /* fall back to MSI */
1203 igb_clear_interrupt_scheme(adapter);
1204 if (!pci_enable_msi(adapter->pdev))
1205 adapter->flags |= IGB_FLAG_HAS_MSI;
1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
1208 adapter->num_tx_queues = 1;
1209 adapter->num_rx_queues = 1;
1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1214 "Unable to allocate memory for vectors\n");
1217 err = igb_alloc_queues(adapter);
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
1227 igb_assign_vector(adapter->q_vector[0], 0);
1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1232 netdev->name, adapter);
1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1242 netdev->name, adapter);
1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 static void igb_free_irq(struct igb_adapter *adapter)
1254 if (adapter->msix_entries) {
1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1265 free_irq(adapter->pdev->irq, adapter);
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1273 static void igb_irq_disable(struct igb_adapter *adapter)
1275 struct e1000_hw *hw = &adapter->hw;
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1282 if (adapter->msix_entries) {
1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_IMC, ~0);
1293 synchronize_irq(adapter->pdev->irq);
1297 * igb_irq_enable - Enable default interrupt generation settings
1298 * @adapter: board private structure
1300 static void igb_irq_enable(struct igb_adapter *adapter)
1302 struct e1000_hw *hw = &adapter->hw;
1304 if (adapter->msix_entries) {
1305 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1306 u32 regval = rd32(E1000_EIAC);
1307 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1308 regval = rd32(E1000_EIAM);
1309 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1310 wr32(E1000_EIMS, adapter->eims_enable_mask);
1311 if (adapter->vfs_allocated_count) {
1312 wr32(E1000_MBVFIMR, 0xFF);
1313 ims |= E1000_IMS_VMMB;
1315 if (adapter->hw.mac.type == e1000_82580)
1316 ims |= E1000_IMS_DRSTA;
1318 wr32(E1000_IMS, ims);
1320 wr32(E1000_IMS, IMS_ENABLE_MASK |
1322 wr32(E1000_IAM, IMS_ENABLE_MASK |
1327 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1329 struct e1000_hw *hw = &adapter->hw;
1330 u16 vid = adapter->hw.mng_cookie.vlan_id;
1331 u16 old_vid = adapter->mng_vlan_id;
1333 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1334 /* add VID to filter table */
1335 igb_vfta_set(hw, vid, true);
1336 adapter->mng_vlan_id = vid;
1338 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1341 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1343 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1344 /* remove VID from filter table */
1345 igb_vfta_set(hw, old_vid, false);
1350 * igb_release_hw_control - release control of the h/w to f/w
1351 * @adapter: address of board private structure
1353 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1354 * For ASF and Pass Through versions of f/w this means that the
1355 * driver is no longer loaded.
1358 static void igb_release_hw_control(struct igb_adapter *adapter)
1360 struct e1000_hw *hw = &adapter->hw;
1363 /* Let firmware take over control of h/w */
1364 ctrl_ext = rd32(E1000_CTRL_EXT);
1365 wr32(E1000_CTRL_EXT,
1366 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1370 * igb_get_hw_control - get control of the h/w from f/w
1371 * @adapter: address of board private structure
1373 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1374 * For ASF and Pass Through versions of f/w this means that
1375 * the driver is loaded.
1378 static void igb_get_hw_control(struct igb_adapter *adapter)
1380 struct e1000_hw *hw = &adapter->hw;
1383 /* Let firmware know the driver has taken over */
1384 ctrl_ext = rd32(E1000_CTRL_EXT);
1385 wr32(E1000_CTRL_EXT,
1386 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1390 * igb_configure - configure the hardware for RX and TX
1391 * @adapter: private board structure
1393 static void igb_configure(struct igb_adapter *adapter)
1395 struct net_device *netdev = adapter->netdev;
1398 igb_get_hw_control(adapter);
1399 igb_set_rx_mode(netdev);
1401 igb_restore_vlan(adapter);
1403 igb_setup_tctl(adapter);
1404 igb_setup_mrqc(adapter);
1405 igb_setup_rctl(adapter);
1407 igb_configure_tx(adapter);
1408 igb_configure_rx(adapter);
1410 igb_rx_fifo_flush_82575(&adapter->hw);
1412 /* call igb_desc_unused which always leaves
1413 * at least 1 descriptor unused to make sure
1414 * next_to_use != next_to_clean */
1415 for (i = 0; i < adapter->num_rx_queues; i++) {
1416 struct igb_ring *ring = adapter->rx_ring[i];
1417 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1422 * igb_power_up_link - Power up the phy/serdes link
1423 * @adapter: address of board private structure
1425 void igb_power_up_link(struct igb_adapter *adapter)
1427 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1428 igb_power_up_phy_copper(&adapter->hw);
1430 igb_power_up_serdes_link_82575(&adapter->hw);
1434 * igb_power_down_link - Power down the phy/serdes link
1435 * @adapter: address of board private structure
1437 static void igb_power_down_link(struct igb_adapter *adapter)
1439 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1440 igb_power_down_phy_copper_82575(&adapter->hw);
1442 igb_shutdown_serdes_link_82575(&adapter->hw);
1446 * igb_up - Open the interface and prepare it to handle traffic
1447 * @adapter: board private structure
1449 int igb_up(struct igb_adapter *adapter)
1451 struct e1000_hw *hw = &adapter->hw;
1454 /* hardware has been reset, we need to reload some things */
1455 igb_configure(adapter);
1457 clear_bit(__IGB_DOWN, &adapter->state);
1459 for (i = 0; i < adapter->num_q_vectors; i++) {
1460 struct igb_q_vector *q_vector = adapter->q_vector[i];
1461 napi_enable(&q_vector->napi);
1463 if (adapter->msix_entries)
1464 igb_configure_msix(adapter);
1466 igb_assign_vector(adapter->q_vector[0], 0);
1468 /* Clear any pending interrupts. */
1470 igb_irq_enable(adapter);
1472 /* notify VFs that reset has been completed */
1473 if (adapter->vfs_allocated_count) {
1474 u32 reg_data = rd32(E1000_CTRL_EXT);
1475 reg_data |= E1000_CTRL_EXT_PFRSTD;
1476 wr32(E1000_CTRL_EXT, reg_data);
1479 netif_tx_start_all_queues(adapter->netdev);
1481 /* start the watchdog. */
1482 hw->mac.get_link_status = 1;
1483 schedule_work(&adapter->watchdog_task);
1488 void igb_down(struct igb_adapter *adapter)
1490 struct net_device *netdev = adapter->netdev;
1491 struct e1000_hw *hw = &adapter->hw;
1495 /* signal that we're down so the interrupt handler does not
1496 * reschedule our watchdog timer */
1497 set_bit(__IGB_DOWN, &adapter->state);
1499 /* disable receives in the hardware */
1500 rctl = rd32(E1000_RCTL);
1501 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1502 /* flush and sleep below */
1504 netif_tx_stop_all_queues(netdev);
1506 /* disable transmits in the hardware */
1507 tctl = rd32(E1000_TCTL);
1508 tctl &= ~E1000_TCTL_EN;
1509 wr32(E1000_TCTL, tctl);
1510 /* flush both disables and wait for them to finish */
1514 for (i = 0; i < adapter->num_q_vectors; i++) {
1515 struct igb_q_vector *q_vector = adapter->q_vector[i];
1516 napi_disable(&q_vector->napi);
1519 igb_irq_disable(adapter);
1521 del_timer_sync(&adapter->watchdog_timer);
1522 del_timer_sync(&adapter->phy_info_timer);
1524 netif_carrier_off(netdev);
1526 /* record the stats before reset*/
1527 igb_update_stats(adapter);
1529 adapter->link_speed = 0;
1530 adapter->link_duplex = 0;
1532 if (!pci_channel_offline(adapter->pdev))
1534 igb_clean_all_tx_rings(adapter);
1535 igb_clean_all_rx_rings(adapter);
1536 #ifdef CONFIG_IGB_DCA
1538 /* since we reset the hardware DCA settings were cleared */
1539 igb_setup_dca(adapter);
1543 void igb_reinit_locked(struct igb_adapter *adapter)
1545 WARN_ON(in_interrupt());
1546 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1550 clear_bit(__IGB_RESETTING, &adapter->state);
1553 void igb_reset(struct igb_adapter *adapter)
1555 struct pci_dev *pdev = adapter->pdev;
1556 struct e1000_hw *hw = &adapter->hw;
1557 struct e1000_mac_info *mac = &hw->mac;
1558 struct e1000_fc_info *fc = &hw->fc;
1559 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1562 /* Repartition Pba for greater than 9k mtu
1563 * To take effect CTRL.RST is required.
1565 switch (mac->type) {
1568 pba = rd32(E1000_RXPBS);
1569 pba = igb_rxpbs_adjust_82580(pba);
1572 pba = rd32(E1000_RXPBS);
1573 pba &= E1000_RXPBS_SIZE_MASK_82576;
1577 pba = E1000_PBA_34K;
1581 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1582 (mac->type < e1000_82576)) {
1583 /* adjust PBA for jumbo frames */
1584 wr32(E1000_PBA, pba);
1586 /* To maintain wire speed transmits, the Tx FIFO should be
1587 * large enough to accommodate two full transmit packets,
1588 * rounded up to the next 1KB and expressed in KB. Likewise,
1589 * the Rx FIFO should be large enough to accommodate at least
1590 * one full receive packet and is similarly rounded up and
1591 * expressed in KB. */
1592 pba = rd32(E1000_PBA);
1593 /* upper 16 bits has Tx packet buffer allocation size in KB */
1594 tx_space = pba >> 16;
1595 /* lower 16 bits has Rx packet buffer allocation size in KB */
1597 /* the tx fifo also stores 16 bytes of information about the tx
1598 * but don't include ethernet FCS because hardware appends it */
1599 min_tx_space = (adapter->max_frame_size +
1600 sizeof(union e1000_adv_tx_desc) -
1602 min_tx_space = ALIGN(min_tx_space, 1024);
1603 min_tx_space >>= 10;
1604 /* software strips receive CRC, so leave room for it */
1605 min_rx_space = adapter->max_frame_size;
1606 min_rx_space = ALIGN(min_rx_space, 1024);
1607 min_rx_space >>= 10;
1609 /* If current Tx allocation is less than the min Tx FIFO size,
1610 * and the min Tx FIFO size is less than the current Rx FIFO
1611 * allocation, take space away from current Rx allocation */
1612 if (tx_space < min_tx_space &&
1613 ((min_tx_space - tx_space) < pba)) {
1614 pba = pba - (min_tx_space - tx_space);
1616 /* if short on rx space, rx wins and must trump tx
1618 if (pba < min_rx_space)
1621 wr32(E1000_PBA, pba);
1624 /* flow control settings */
1625 /* The high water mark must be low enough to fit one full frame
1626 * (or the size used for early receive) above it in the Rx FIFO.
1627 * Set it to the lower of:
1628 * - 90% of the Rx FIFO size, or
1629 * - the full Rx FIFO size minus one full frame */
1630 hwm = min(((pba << 10) * 9 / 10),
1631 ((pba << 10) - 2 * adapter->max_frame_size));
1633 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1634 fc->low_water = fc->high_water - 16;
1635 fc->pause_time = 0xFFFF;
1637 fc->current_mode = fc->requested_mode;
1639 /* disable receive for all VFs and wait one second */
1640 if (adapter->vfs_allocated_count) {
1642 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1643 adapter->vf_data[i].flags = 0;
1645 /* ping all the active vfs to let them know we are going down */
1646 igb_ping_all_vfs(adapter);
1648 /* disable transmits and receives */
1649 wr32(E1000_VFRE, 0);
1650 wr32(E1000_VFTE, 0);
1653 /* Allow time for pending master requests to run */
1654 hw->mac.ops.reset_hw(hw);
1657 if (hw->mac.ops.init_hw(hw))
1658 dev_err(&pdev->dev, "Hardware Error\n");
1660 if (hw->mac.type == e1000_82580) {
1661 u32 reg = rd32(E1000_PCIEMISC);
1662 wr32(E1000_PCIEMISC,
1663 reg & ~E1000_PCIEMISC_LX_DECISION);
1665 if (!netif_running(adapter->netdev))
1666 igb_power_down_link(adapter);
1668 igb_update_mng_vlan(adapter);
1670 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1671 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1673 igb_get_phy_info(hw);
1676 static const struct net_device_ops igb_netdev_ops = {
1677 .ndo_open = igb_open,
1678 .ndo_stop = igb_close,
1679 .ndo_start_xmit = igb_xmit_frame_adv,
1680 .ndo_get_stats = igb_get_stats,
1681 .ndo_set_rx_mode = igb_set_rx_mode,
1682 .ndo_set_multicast_list = igb_set_rx_mode,
1683 .ndo_set_mac_address = igb_set_mac,
1684 .ndo_change_mtu = igb_change_mtu,
1685 .ndo_do_ioctl = igb_ioctl,
1686 .ndo_tx_timeout = igb_tx_timeout,
1687 .ndo_validate_addr = eth_validate_addr,
1688 .ndo_vlan_rx_register = igb_vlan_rx_register,
1689 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1690 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1691 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1692 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1693 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1694 .ndo_get_vf_config = igb_ndo_get_vf_config,
1695 #ifdef CONFIG_NET_POLL_CONTROLLER
1696 .ndo_poll_controller = igb_netpoll,
1701 * igb_probe - Device Initialization Routine
1702 * @pdev: PCI device information struct
1703 * @ent: entry in igb_pci_tbl
1705 * Returns 0 on success, negative on failure
1707 * igb_probe initializes an adapter identified by a pci_dev structure.
1708 * The OS initialization, configuring of the adapter private structure,
1709 * and a hardware reset occur.
1711 static int __devinit igb_probe(struct pci_dev *pdev,
1712 const struct pci_device_id *ent)
1714 struct net_device *netdev;
1715 struct igb_adapter *adapter;
1716 struct e1000_hw *hw;
1717 u16 eeprom_data = 0;
1718 static int global_quad_port_a; /* global quad port a indication */
1719 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1720 unsigned long mmio_start, mmio_len;
1721 int err, pci_using_dac;
1722 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1725 /* Catch broken hardware that put the wrong VF device ID in
1726 * the PCIe SR-IOV capability.
1728 if (pdev->is_virtfn) {
1729 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1730 pci_name(pdev), pdev->vendor, pdev->device);
1734 err = pci_enable_device_mem(pdev);
1739 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1741 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1745 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1747 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1749 dev_err(&pdev->dev, "No usable DMA "
1750 "configuration, aborting\n");
1756 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1762 pci_enable_pcie_error_reporting(pdev);
1764 pci_set_master(pdev);
1765 pci_save_state(pdev);
1768 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1769 IGB_ABS_MAX_TX_QUEUES);
1771 goto err_alloc_etherdev;
1773 SET_NETDEV_DEV(netdev, &pdev->dev);
1775 pci_set_drvdata(pdev, netdev);
1776 adapter = netdev_priv(netdev);
1777 adapter->netdev = netdev;
1778 adapter->pdev = pdev;
1781 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1783 mmio_start = pci_resource_start(pdev, 0);
1784 mmio_len = pci_resource_len(pdev, 0);
1787 hw->hw_addr = ioremap(mmio_start, mmio_len);
1791 netdev->netdev_ops = &igb_netdev_ops;
1792 igb_set_ethtool_ops(netdev);
1793 netdev->watchdog_timeo = 5 * HZ;
1795 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1797 netdev->mem_start = mmio_start;
1798 netdev->mem_end = mmio_start + mmio_len;
1800 /* PCI config space info */
1801 hw->vendor_id = pdev->vendor;
1802 hw->device_id = pdev->device;
1803 hw->revision_id = pdev->revision;
1804 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1805 hw->subsystem_device_id = pdev->subsystem_device;
1807 /* Copy the default MAC, PHY and NVM function pointers */
1808 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1809 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1810 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1811 /* Initialize skew-specific constants */
1812 err = ei->get_invariants(hw);
1816 /* setup the private structure */
1817 err = igb_sw_init(adapter);
1821 igb_get_bus_info_pcie(hw);
1823 hw->phy.autoneg_wait_to_complete = false;
1825 /* Copper options */
1826 if (hw->phy.media_type == e1000_media_type_copper) {
1827 hw->phy.mdix = AUTO_ALL_MODES;
1828 hw->phy.disable_polarity_correction = false;
1829 hw->phy.ms_type = e1000_ms_hw_default;
1832 if (igb_check_reset_block(hw))
1833 dev_info(&pdev->dev,
1834 "PHY reset is blocked due to SOL/IDER session.\n");
1836 netdev->features = NETIF_F_SG |
1838 NETIF_F_HW_VLAN_TX |
1839 NETIF_F_HW_VLAN_RX |
1840 NETIF_F_HW_VLAN_FILTER;
1842 netdev->features |= NETIF_F_IPV6_CSUM;
1843 netdev->features |= NETIF_F_TSO;
1844 netdev->features |= NETIF_F_TSO6;
1845 netdev->features |= NETIF_F_GRO;
1847 netdev->vlan_features |= NETIF_F_TSO;
1848 netdev->vlan_features |= NETIF_F_TSO6;
1849 netdev->vlan_features |= NETIF_F_IP_CSUM;
1850 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1851 netdev->vlan_features |= NETIF_F_SG;
1854 netdev->features |= NETIF_F_HIGHDMA;
1856 if (hw->mac.type >= e1000_82576)
1857 netdev->features |= NETIF_F_SCTP_CSUM;
1859 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1861 /* before reading the NVM, reset the controller to put the device in a
1862 * known good starting state */
1863 hw->mac.ops.reset_hw(hw);
1865 /* make sure the NVM is good */
1866 if (igb_validate_nvm_checksum(hw) < 0) {
1867 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1872 /* copy the MAC address out of the NVM */
1873 if (hw->mac.ops.read_mac_addr(hw))
1874 dev_err(&pdev->dev, "NVM Read Error\n");
1876 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1877 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1879 if (!is_valid_ether_addr(netdev->perm_addr)) {
1880 dev_err(&pdev->dev, "Invalid MAC Address\n");
1885 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1886 (unsigned long) adapter);
1887 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1888 (unsigned long) adapter);
1890 INIT_WORK(&adapter->reset_task, igb_reset_task);
1891 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1893 /* Initialize link properties that are user-changeable */
1894 adapter->fc_autoneg = true;
1895 hw->mac.autoneg = true;
1896 hw->phy.autoneg_advertised = 0x2f;
1898 hw->fc.requested_mode = e1000_fc_default;
1899 hw->fc.current_mode = e1000_fc_default;
1901 igb_validate_mdi_setting(hw);
1903 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1904 * enable the ACPI Magic Packet filter
1907 if (hw->bus.func == 0)
1908 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1909 else if (hw->mac.type == e1000_82580)
1910 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1911 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1913 else if (hw->bus.func == 1)
1914 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1916 if (eeprom_data & eeprom_apme_mask)
1917 adapter->eeprom_wol |= E1000_WUFC_MAG;
1919 /* now that we have the eeprom settings, apply the special cases where
1920 * the eeprom may be wrong or the board simply won't support wake on
1921 * lan on a particular port */
1922 switch (pdev->device) {
1923 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1924 adapter->eeprom_wol = 0;
1926 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1927 case E1000_DEV_ID_82576_FIBER:
1928 case E1000_DEV_ID_82576_SERDES:
1929 /* Wake events only supported on port A for dual fiber
1930 * regardless of eeprom setting */
1931 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1932 adapter->eeprom_wol = 0;
1934 case E1000_DEV_ID_82576_QUAD_COPPER:
1935 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1936 /* if quad port adapter, disable WoL on all but port A */
1937 if (global_quad_port_a != 0)
1938 adapter->eeprom_wol = 0;
1940 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1941 /* Reset for multiple quad port adapters */
1942 if (++global_quad_port_a == 4)
1943 global_quad_port_a = 0;
1947 /* initialize the wol settings based on the eeprom settings */
1948 adapter->wol = adapter->eeprom_wol;
1949 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1951 /* reset the hardware with the new settings */
1954 /* let the f/w know that the h/w is now under the control of the
1956 igb_get_hw_control(adapter);
1958 strcpy(netdev->name, "eth%d");
1959 err = register_netdev(netdev);
1963 /* carrier off reporting is important to ethtool even BEFORE open */
1964 netif_carrier_off(netdev);
1966 #ifdef CONFIG_IGB_DCA
1967 if (dca_add_requester(&pdev->dev) == 0) {
1968 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1969 dev_info(&pdev->dev, "DCA enabled\n");
1970 igb_setup_dca(adapter);
1974 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1975 /* print bus type/speed/width info */
1976 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1978 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1979 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1981 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1982 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1983 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1987 igb_read_part_num(hw, &part_num);
1988 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1989 (part_num >> 8), (part_num & 0xff));
1991 dev_info(&pdev->dev,
1992 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1993 adapter->msix_entries ? "MSI-X" :
1994 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1995 adapter->num_rx_queues, adapter->num_tx_queues);
2000 igb_release_hw_control(adapter);
2002 if (!igb_check_reset_block(hw))
2005 if (hw->flash_address)
2006 iounmap(hw->flash_address);
2008 igb_clear_interrupt_scheme(adapter);
2009 iounmap(hw->hw_addr);
2011 free_netdev(netdev);
2013 pci_release_selected_regions(pdev,
2014 pci_select_bars(pdev, IORESOURCE_MEM));
2017 pci_disable_device(pdev);
2022 * igb_remove - Device Removal Routine
2023 * @pdev: PCI device information struct
2025 * igb_remove is called by the PCI subsystem to alert the driver
2026 * that it should release a PCI device. The could be caused by a
2027 * Hot-Plug event, or because the driver is going to be removed from
2030 static void __devexit igb_remove(struct pci_dev *pdev)
2032 struct net_device *netdev = pci_get_drvdata(pdev);
2033 struct igb_adapter *adapter = netdev_priv(netdev);
2034 struct e1000_hw *hw = &adapter->hw;
2036 /* flush_scheduled work may reschedule our watchdog task, so
2037 * explicitly disable watchdog tasks from being rescheduled */
2038 set_bit(__IGB_DOWN, &adapter->state);
2039 del_timer_sync(&adapter->watchdog_timer);
2040 del_timer_sync(&adapter->phy_info_timer);
2042 flush_scheduled_work();
2044 #ifdef CONFIG_IGB_DCA
2045 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2046 dev_info(&pdev->dev, "DCA disabled\n");
2047 dca_remove_requester(&pdev->dev);
2048 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2049 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2053 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2054 * would have already happened in close and is redundant. */
2055 igb_release_hw_control(adapter);
2057 unregister_netdev(netdev);
2059 igb_clear_interrupt_scheme(adapter);
2061 #ifdef CONFIG_PCI_IOV
2062 /* reclaim resources allocated to VFs */
2063 if (adapter->vf_data) {
2064 /* disable iov and allow time for transactions to clear */
2065 pci_disable_sriov(pdev);
2068 kfree(adapter->vf_data);
2069 adapter->vf_data = NULL;
2070 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2072 dev_info(&pdev->dev, "IOV Disabled\n");
2076 iounmap(hw->hw_addr);
2077 if (hw->flash_address)
2078 iounmap(hw->flash_address);
2079 pci_release_selected_regions(pdev,
2080 pci_select_bars(pdev, IORESOURCE_MEM));
2082 free_netdev(netdev);
2084 pci_disable_pcie_error_reporting(pdev);
2086 pci_disable_device(pdev);
2090 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2091 * @adapter: board private structure to initialize
2093 * This function initializes the vf specific data storage and then attempts to
2094 * allocate the VFs. The reason for ordering it this way is because it is much
2095 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2096 * the memory for the VFs.
2098 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2100 #ifdef CONFIG_PCI_IOV
2101 struct pci_dev *pdev = adapter->pdev;
2103 if (adapter->vfs_allocated_count > 7)
2104 adapter->vfs_allocated_count = 7;
2106 if (adapter->vfs_allocated_count) {
2107 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2108 sizeof(struct vf_data_storage),
2110 /* if allocation failed then we do not support SR-IOV */
2111 if (!adapter->vf_data) {
2112 adapter->vfs_allocated_count = 0;
2113 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2118 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2119 kfree(adapter->vf_data);
2120 adapter->vf_data = NULL;
2121 #endif /* CONFIG_PCI_IOV */
2122 adapter->vfs_allocated_count = 0;
2123 #ifdef CONFIG_PCI_IOV
2125 unsigned char mac_addr[ETH_ALEN];
2127 dev_info(&pdev->dev, "%d vfs allocated\n",
2128 adapter->vfs_allocated_count);
2129 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2130 random_ether_addr(mac_addr);
2131 igb_set_vf_mac(adapter, i, mac_addr);
2134 #endif /* CONFIG_PCI_IOV */
2139 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2140 * @adapter: board private structure to initialize
2142 * igb_init_hw_timer initializes the function pointer and values for the hw
2143 * timer found in hardware.
2145 static void igb_init_hw_timer(struct igb_adapter *adapter)
2147 struct e1000_hw *hw = &adapter->hw;
2149 switch (hw->mac.type) {
2152 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2153 adapter->cycles.read = igb_read_clock;
2154 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2155 adapter->cycles.mult = 1;
2157 * The 82580 timesync updates the system timer every 8ns by 8ns
2158 * and the value cannot be shifted. Instead we need to shift
2159 * the registers to generate a 64bit timer value. As a result
2160 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2161 * 24 in order to generate a larger value for synchronization.
2163 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2164 /* disable system timer temporarily by setting bit 31 */
2165 wr32(E1000_TSAUXC, 0x80000000);
2168 /* Set registers so that rollover occurs soon to test this. */
2169 wr32(E1000_SYSTIMR, 0x00000000);
2170 wr32(E1000_SYSTIML, 0x80000000);
2171 wr32(E1000_SYSTIMH, 0x000000FF);
2174 /* enable system timer by clearing bit 31 */
2175 wr32(E1000_TSAUXC, 0x0);
2178 timecounter_init(&adapter->clock,
2180 ktime_to_ns(ktime_get_real()));
2182 * Synchronize our NIC clock against system wall clock. NIC
2183 * time stamp reading requires ~3us per sample, each sample
2184 * was pretty stable even under load => only require 10
2185 * samples for each offset comparison.
2187 memset(&adapter->compare, 0, sizeof(adapter->compare));
2188 adapter->compare.source = &adapter->clock;
2189 adapter->compare.target = ktime_get_real;
2190 adapter->compare.num_samples = 10;
2191 timecompare_update(&adapter->compare, 0);
2195 * Initialize hardware timer: we keep it running just in case
2196 * that some program needs it later on.
2198 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2199 adapter->cycles.read = igb_read_clock;
2200 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2201 adapter->cycles.mult = 1;
2203 * Scale the NIC clock cycle by a large factor so that
2204 * relatively small clock corrections can be added or
2205 * substracted at each clock tick. The drawbacks of a large
2206 * factor are a) that the clock register overflows more quickly
2207 * (not such a big deal) and b) that the increment per tick has
2208 * to fit into 24 bits. As a result we need to use a shift of
2209 * 19 so we can fit a value of 16 into the TIMINCA register.
2211 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2213 (1 << E1000_TIMINCA_16NS_SHIFT) |
2214 (16 << IGB_82576_TSYNC_SHIFT));
2216 /* Set registers so that rollover occurs soon to test this. */
2217 wr32(E1000_SYSTIML, 0x00000000);
2218 wr32(E1000_SYSTIMH, 0xFF800000);
2221 timecounter_init(&adapter->clock,
2223 ktime_to_ns(ktime_get_real()));
2225 * Synchronize our NIC clock against system wall clock. NIC
2226 * time stamp reading requires ~3us per sample, each sample
2227 * was pretty stable even under load => only require 10
2228 * samples for each offset comparison.
2230 memset(&adapter->compare, 0, sizeof(adapter->compare));
2231 adapter->compare.source = &adapter->clock;
2232 adapter->compare.target = ktime_get_real;
2233 adapter->compare.num_samples = 10;
2234 timecompare_update(&adapter->compare, 0);
2237 /* 82575 does not support timesync */
2245 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2246 * @adapter: board private structure to initialize
2248 * igb_sw_init initializes the Adapter private data structure.
2249 * Fields are initialized based on PCI device information and
2250 * OS network device settings (MTU size).
2252 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2254 struct e1000_hw *hw = &adapter->hw;
2255 struct net_device *netdev = adapter->netdev;
2256 struct pci_dev *pdev = adapter->pdev;
2258 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2260 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2261 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2262 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2263 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2265 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2266 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2268 #ifdef CONFIG_PCI_IOV
2269 if (hw->mac.type == e1000_82576)
2270 adapter->vfs_allocated_count = max_vfs;
2272 #endif /* CONFIG_PCI_IOV */
2273 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2276 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2277 * then we should combine the queues into a queue pair in order to
2278 * conserve interrupts due to limited supply
2280 if ((adapter->rss_queues > 4) ||
2281 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2282 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2284 /* This call may decrease the number of queues */
2285 if (igb_init_interrupt_scheme(adapter)) {
2286 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2290 igb_init_hw_timer(adapter);
2291 igb_probe_vfs(adapter);
2293 /* Explicitly disable IRQ since the NIC can be in any state. */
2294 igb_irq_disable(adapter);
2296 set_bit(__IGB_DOWN, &adapter->state);
2301 * igb_open - Called when a network interface is made active
2302 * @netdev: network interface device structure
2304 * Returns 0 on success, negative value on failure
2306 * The open entry point is called when a network interface is made
2307 * active by the system (IFF_UP). At this point all resources needed
2308 * for transmit and receive operations are allocated, the interrupt
2309 * handler is registered with the OS, the watchdog timer is started,
2310 * and the stack is notified that the interface is ready.
2312 static int igb_open(struct net_device *netdev)
2314 struct igb_adapter *adapter = netdev_priv(netdev);
2315 struct e1000_hw *hw = &adapter->hw;
2319 /* disallow open during test */
2320 if (test_bit(__IGB_TESTING, &adapter->state))
2323 netif_carrier_off(netdev);
2325 /* allocate transmit descriptors */
2326 err = igb_setup_all_tx_resources(adapter);
2330 /* allocate receive descriptors */
2331 err = igb_setup_all_rx_resources(adapter);
2335 igb_power_up_link(adapter);
2337 /* before we allocate an interrupt, we must be ready to handle it.
2338 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2339 * as soon as we call pci_request_irq, so we have to setup our
2340 * clean_rx handler before we do so. */
2341 igb_configure(adapter);
2343 err = igb_request_irq(adapter);
2347 /* From here on the code is the same as igb_up() */
2348 clear_bit(__IGB_DOWN, &adapter->state);
2350 for (i = 0; i < adapter->num_q_vectors; i++) {
2351 struct igb_q_vector *q_vector = adapter->q_vector[i];
2352 napi_enable(&q_vector->napi);
2355 /* Clear any pending interrupts. */
2358 igb_irq_enable(adapter);
2360 /* notify VFs that reset has been completed */
2361 if (adapter->vfs_allocated_count) {
2362 u32 reg_data = rd32(E1000_CTRL_EXT);
2363 reg_data |= E1000_CTRL_EXT_PFRSTD;
2364 wr32(E1000_CTRL_EXT, reg_data);
2367 netif_tx_start_all_queues(netdev);
2369 /* start the watchdog. */
2370 hw->mac.get_link_status = 1;
2371 schedule_work(&adapter->watchdog_task);
2376 igb_release_hw_control(adapter);
2377 igb_power_down_link(adapter);
2378 igb_free_all_rx_resources(adapter);
2380 igb_free_all_tx_resources(adapter);
2388 * igb_close - Disables a network interface
2389 * @netdev: network interface device structure
2391 * Returns 0, this is not allowed to fail
2393 * The close entry point is called when an interface is de-activated
2394 * by the OS. The hardware is still under the driver's control, but
2395 * needs to be disabled. A global MAC reset is issued to stop the
2396 * hardware, and all transmit and receive resources are freed.
2398 static int igb_close(struct net_device *netdev)
2400 struct igb_adapter *adapter = netdev_priv(netdev);
2402 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2405 igb_free_irq(adapter);
2407 igb_free_all_tx_resources(adapter);
2408 igb_free_all_rx_resources(adapter);
2414 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2415 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2417 * Return 0 on success, negative on failure
2419 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2421 struct device *dev = tx_ring->dev;
2424 size = sizeof(struct igb_buffer) * tx_ring->count;
2425 tx_ring->buffer_info = vmalloc(size);
2426 if (!tx_ring->buffer_info)
2428 memset(tx_ring->buffer_info, 0, size);
2430 /* round up to nearest 4K */
2431 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2432 tx_ring->size = ALIGN(tx_ring->size, 4096);
2434 tx_ring->desc = dma_alloc_coherent(dev,
2442 tx_ring->next_to_use = 0;
2443 tx_ring->next_to_clean = 0;
2447 vfree(tx_ring->buffer_info);
2449 "Unable to allocate memory for the transmit descriptor ring\n");
2454 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2455 * (Descriptors) for all queues
2456 * @adapter: board private structure
2458 * Return 0 on success, negative on failure
2460 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2462 struct pci_dev *pdev = adapter->pdev;
2465 for (i = 0; i < adapter->num_tx_queues; i++) {
2466 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2469 "Allocation for Tx Queue %u failed\n", i);
2470 for (i--; i >= 0; i--)
2471 igb_free_tx_resources(adapter->tx_ring[i]);
2476 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2477 int r_idx = i % adapter->num_tx_queues;
2478 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2484 * igb_setup_tctl - configure the transmit control registers
2485 * @adapter: Board private structure
2487 void igb_setup_tctl(struct igb_adapter *adapter)
2489 struct e1000_hw *hw = &adapter->hw;
2492 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2493 wr32(E1000_TXDCTL(0), 0);
2495 /* Program the Transmit Control Register */
2496 tctl = rd32(E1000_TCTL);
2497 tctl &= ~E1000_TCTL_CT;
2498 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2499 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2501 igb_config_collision_dist(hw);
2503 /* Enable transmits */
2504 tctl |= E1000_TCTL_EN;
2506 wr32(E1000_TCTL, tctl);
2510 * igb_configure_tx_ring - Configure transmit ring after Reset
2511 * @adapter: board private structure
2512 * @ring: tx ring to configure
2514 * Configure a transmit ring after a reset.
2516 void igb_configure_tx_ring(struct igb_adapter *adapter,
2517 struct igb_ring *ring)
2519 struct e1000_hw *hw = &adapter->hw;
2521 u64 tdba = ring->dma;
2522 int reg_idx = ring->reg_idx;
2524 /* disable the queue */
2525 txdctl = rd32(E1000_TXDCTL(reg_idx));
2526 wr32(E1000_TXDCTL(reg_idx),
2527 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2531 wr32(E1000_TDLEN(reg_idx),
2532 ring->count * sizeof(union e1000_adv_tx_desc));
2533 wr32(E1000_TDBAL(reg_idx),
2534 tdba & 0x00000000ffffffffULL);
2535 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2537 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2538 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2539 writel(0, ring->head);
2540 writel(0, ring->tail);
2542 txdctl |= IGB_TX_PTHRESH;
2543 txdctl |= IGB_TX_HTHRESH << 8;
2544 txdctl |= IGB_TX_WTHRESH << 16;
2546 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2547 wr32(E1000_TXDCTL(reg_idx), txdctl);
2551 * igb_configure_tx - Configure transmit Unit after Reset
2552 * @adapter: board private structure
2554 * Configure the Tx unit of the MAC after a reset.
2556 static void igb_configure_tx(struct igb_adapter *adapter)
2560 for (i = 0; i < adapter->num_tx_queues; i++)
2561 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2565 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2566 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2568 * Returns 0 on success, negative on failure
2570 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2572 struct device *dev = rx_ring->dev;
2575 size = sizeof(struct igb_buffer) * rx_ring->count;
2576 rx_ring->buffer_info = vmalloc(size);
2577 if (!rx_ring->buffer_info)
2579 memset(rx_ring->buffer_info, 0, size);
2581 desc_len = sizeof(union e1000_adv_rx_desc);
2583 /* Round up to nearest 4K */
2584 rx_ring->size = rx_ring->count * desc_len;
2585 rx_ring->size = ALIGN(rx_ring->size, 4096);
2587 rx_ring->desc = dma_alloc_coherent(dev,
2595 rx_ring->next_to_clean = 0;
2596 rx_ring->next_to_use = 0;
2601 vfree(rx_ring->buffer_info);
2602 rx_ring->buffer_info = NULL;
2603 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2609 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2610 * (Descriptors) for all queues
2611 * @adapter: board private structure
2613 * Return 0 on success, negative on failure
2615 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2617 struct pci_dev *pdev = adapter->pdev;
2620 for (i = 0; i < adapter->num_rx_queues; i++) {
2621 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2624 "Allocation for Rx Queue %u failed\n", i);
2625 for (i--; i >= 0; i--)
2626 igb_free_rx_resources(adapter->rx_ring[i]);
2635 * igb_setup_mrqc - configure the multiple receive queue control registers
2636 * @adapter: Board private structure
2638 static void igb_setup_mrqc(struct igb_adapter *adapter)
2640 struct e1000_hw *hw = &adapter->hw;
2642 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2647 static const u8 rsshash[40] = {
2648 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2649 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2650 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2651 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2653 /* Fill out hash function seeds */
2654 for (j = 0; j < 10; j++) {
2655 u32 rsskey = rsshash[(j * 4)];
2656 rsskey |= rsshash[(j * 4) + 1] << 8;
2657 rsskey |= rsshash[(j * 4) + 2] << 16;
2658 rsskey |= rsshash[(j * 4) + 3] << 24;
2659 array_wr32(E1000_RSSRK(0), j, rsskey);
2662 num_rx_queues = adapter->rss_queues;
2664 if (adapter->vfs_allocated_count) {
2665 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2666 switch (hw->mac.type) {
2683 if (hw->mac.type == e1000_82575)
2687 for (j = 0; j < (32 * 4); j++) {
2688 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2690 reta.bytes[j & 3] |= num_rx_queues << shift2;
2692 wr32(E1000_RETA(j >> 2), reta.dword);
2696 * Disable raw packet checksumming so that RSS hash is placed in
2697 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2698 * offloads as they are enabled by default
2700 rxcsum = rd32(E1000_RXCSUM);
2701 rxcsum |= E1000_RXCSUM_PCSD;
2703 if (adapter->hw.mac.type >= e1000_82576)
2704 /* Enable Receive Checksum Offload for SCTP */
2705 rxcsum |= E1000_RXCSUM_CRCOFL;
2707 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2708 wr32(E1000_RXCSUM, rxcsum);
2710 /* If VMDq is enabled then we set the appropriate mode for that, else
2711 * we default to RSS so that an RSS hash is calculated per packet even
2712 * if we are only using one queue */
2713 if (adapter->vfs_allocated_count) {
2714 if (hw->mac.type > e1000_82575) {
2715 /* Set the default pool for the PF's first queue */
2716 u32 vtctl = rd32(E1000_VT_CTL);
2717 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2718 E1000_VT_CTL_DISABLE_DEF_POOL);
2719 vtctl |= adapter->vfs_allocated_count <<
2720 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2721 wr32(E1000_VT_CTL, vtctl);
2723 if (adapter->rss_queues > 1)
2724 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2726 mrqc = E1000_MRQC_ENABLE_VMDQ;
2728 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2730 igb_vmm_control(adapter);
2732 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2733 E1000_MRQC_RSS_FIELD_IPV4_TCP);
2734 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2735 E1000_MRQC_RSS_FIELD_IPV6_TCP);
2736 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2737 E1000_MRQC_RSS_FIELD_IPV6_UDP);
2738 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2739 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2741 wr32(E1000_MRQC, mrqc);
2745 * igb_setup_rctl - configure the receive control registers
2746 * @adapter: Board private structure
2748 void igb_setup_rctl(struct igb_adapter *adapter)
2750 struct e1000_hw *hw = &adapter->hw;
2753 rctl = rd32(E1000_RCTL);
2755 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2756 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2758 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2759 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2762 * enable stripping of CRC. It's unlikely this will break BMC
2763 * redirection as it did with e1000. Newer features require
2764 * that the HW strips the CRC.
2766 rctl |= E1000_RCTL_SECRC;
2768 /* disable store bad packets and clear size bits. */
2769 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2771 /* enable LPE to prevent packets larger than max_frame_size */
2772 rctl |= E1000_RCTL_LPE;
2774 /* disable queue 0 to prevent tail write w/o re-config */
2775 wr32(E1000_RXDCTL(0), 0);
2777 /* Attention!!! For SR-IOV PF driver operations you must enable
2778 * queue drop for all VF and PF queues to prevent head of line blocking
2779 * if an un-trusted VF does not provide descriptors to hardware.
2781 if (adapter->vfs_allocated_count) {
2782 /* set all queue drop enable bits */
2783 wr32(E1000_QDE, ALL_QUEUES);
2786 wr32(E1000_RCTL, rctl);
2789 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2792 struct e1000_hw *hw = &adapter->hw;
2795 /* if it isn't the PF check to see if VFs are enabled and
2796 * increase the size to support vlan tags */
2797 if (vfn < adapter->vfs_allocated_count &&
2798 adapter->vf_data[vfn].vlans_enabled)
2799 size += VLAN_TAG_SIZE;
2801 vmolr = rd32(E1000_VMOLR(vfn));
2802 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2803 vmolr |= size | E1000_VMOLR_LPE;
2804 wr32(E1000_VMOLR(vfn), vmolr);
2810 * igb_rlpml_set - set maximum receive packet size
2811 * @adapter: board private structure
2813 * Configure maximum receivable packet size.
2815 static void igb_rlpml_set(struct igb_adapter *adapter)
2817 u32 max_frame_size = adapter->max_frame_size;
2818 struct e1000_hw *hw = &adapter->hw;
2819 u16 pf_id = adapter->vfs_allocated_count;
2822 max_frame_size += VLAN_TAG_SIZE;
2824 /* if vfs are enabled we set RLPML to the largest possible request
2825 * size and set the VMOLR RLPML to the size we need */
2827 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2828 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2831 wr32(E1000_RLPML, max_frame_size);
2834 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2837 struct e1000_hw *hw = &adapter->hw;
2841 * This register exists only on 82576 and newer so if we are older then
2842 * we should exit and do nothing
2844 if (hw->mac.type < e1000_82576)
2847 vmolr = rd32(E1000_VMOLR(vfn));
2848 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2850 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2852 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2854 /* clear all bits that might not be set */
2855 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2857 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2858 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2860 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2863 if (vfn <= adapter->vfs_allocated_count)
2864 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2866 wr32(E1000_VMOLR(vfn), vmolr);
2870 * igb_configure_rx_ring - Configure a receive ring after Reset
2871 * @adapter: board private structure
2872 * @ring: receive ring to be configured
2874 * Configure the Rx unit of the MAC after a reset.
2876 void igb_configure_rx_ring(struct igb_adapter *adapter,
2877 struct igb_ring *ring)
2879 struct e1000_hw *hw = &adapter->hw;
2880 u64 rdba = ring->dma;
2881 int reg_idx = ring->reg_idx;
2884 /* disable the queue */
2885 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2886 wr32(E1000_RXDCTL(reg_idx),
2887 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2889 /* Set DMA base address registers */
2890 wr32(E1000_RDBAL(reg_idx),
2891 rdba & 0x00000000ffffffffULL);
2892 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2893 wr32(E1000_RDLEN(reg_idx),
2894 ring->count * sizeof(union e1000_adv_rx_desc));
2896 /* initialize head and tail */
2897 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2898 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2899 writel(0, ring->head);
2900 writel(0, ring->tail);
2902 /* set descriptor configuration */
2903 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2904 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2905 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2906 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2907 srrctl |= IGB_RXBUFFER_16384 >>
2908 E1000_SRRCTL_BSIZEPKT_SHIFT;
2910 srrctl |= (PAGE_SIZE / 2) >>
2911 E1000_SRRCTL_BSIZEPKT_SHIFT;
2913 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2915 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2916 E1000_SRRCTL_BSIZEPKT_SHIFT;
2917 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2919 if (hw->mac.type == e1000_82580)
2920 srrctl |= E1000_SRRCTL_TIMESTAMP;
2921 /* Only set Drop Enable if we are supporting multiple queues */
2922 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2923 srrctl |= E1000_SRRCTL_DROP_EN;
2925 wr32(E1000_SRRCTL(reg_idx), srrctl);
2927 /* set filtering for VMDQ pools */
2928 igb_set_vmolr(adapter, reg_idx & 0x7, true);
2930 /* enable receive descriptor fetching */
2931 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2932 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2933 rxdctl &= 0xFFF00000;
2934 rxdctl |= IGB_RX_PTHRESH;
2935 rxdctl |= IGB_RX_HTHRESH << 8;
2936 rxdctl |= IGB_RX_WTHRESH << 16;
2937 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2941 * igb_configure_rx - Configure receive Unit after Reset
2942 * @adapter: board private structure
2944 * Configure the Rx unit of the MAC after a reset.
2946 static void igb_configure_rx(struct igb_adapter *adapter)
2950 /* set UTA to appropriate mode */
2951 igb_set_uta(adapter);
2953 /* set the correct pool for the PF default MAC address in entry 0 */
2954 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2955 adapter->vfs_allocated_count);
2957 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2958 * the Base and Length of the Rx Descriptor Ring */
2959 for (i = 0; i < adapter->num_rx_queues; i++)
2960 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2964 * igb_free_tx_resources - Free Tx Resources per Queue
2965 * @tx_ring: Tx descriptor ring for a specific queue
2967 * Free all transmit software resources
2969 void igb_free_tx_resources(struct igb_ring *tx_ring)
2971 igb_clean_tx_ring(tx_ring);
2973 vfree(tx_ring->buffer_info);
2974 tx_ring->buffer_info = NULL;
2976 /* if not set, then don't free */
2980 dma_free_coherent(tx_ring->dev, tx_ring->size,
2981 tx_ring->desc, tx_ring->dma);
2983 tx_ring->desc = NULL;
2987 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2988 * @adapter: board private structure
2990 * Free all transmit software resources
2992 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2996 for (i = 0; i < adapter->num_tx_queues; i++)
2997 igb_free_tx_resources(adapter->tx_ring[i]);
3000 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3001 struct igb_buffer *buffer_info)
3003 if (buffer_info->dma) {
3004 if (buffer_info->mapped_as_page)
3005 dma_unmap_page(tx_ring->dev,
3007 buffer_info->length,
3010 dma_unmap_single(tx_ring->dev,
3012 buffer_info->length,
3014 buffer_info->dma = 0;
3016 if (buffer_info->skb) {
3017 dev_kfree_skb_any(buffer_info->skb);
3018 buffer_info->skb = NULL;
3020 buffer_info->time_stamp = 0;
3021 buffer_info->length = 0;
3022 buffer_info->next_to_watch = 0;
3023 buffer_info->mapped_as_page = false;
3027 * igb_clean_tx_ring - Free Tx Buffers
3028 * @tx_ring: ring to be cleaned
3030 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3032 struct igb_buffer *buffer_info;
3036 if (!tx_ring->buffer_info)
3038 /* Free all the Tx ring sk_buffs */
3040 for (i = 0; i < tx_ring->count; i++) {
3041 buffer_info = &tx_ring->buffer_info[i];
3042 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3045 size = sizeof(struct igb_buffer) * tx_ring->count;
3046 memset(tx_ring->buffer_info, 0, size);
3048 /* Zero out the descriptor ring */
3049 memset(tx_ring->desc, 0, tx_ring->size);
3051 tx_ring->next_to_use = 0;
3052 tx_ring->next_to_clean = 0;
3056 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3057 * @adapter: board private structure
3059 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3063 for (i = 0; i < adapter->num_tx_queues; i++)
3064 igb_clean_tx_ring(adapter->tx_ring[i]);
3068 * igb_free_rx_resources - Free Rx Resources
3069 * @rx_ring: ring to clean the resources from
3071 * Free all receive software resources
3073 void igb_free_rx_resources(struct igb_ring *rx_ring)
3075 igb_clean_rx_ring(rx_ring);
3077 vfree(rx_ring->buffer_info);
3078 rx_ring->buffer_info = NULL;
3080 /* if not set, then don't free */
3084 dma_free_coherent(rx_ring->dev, rx_ring->size,
3085 rx_ring->desc, rx_ring->dma);
3087 rx_ring->desc = NULL;
3091 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3092 * @adapter: board private structure
3094 * Free all receive software resources
3096 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3100 for (i = 0; i < adapter->num_rx_queues; i++)
3101 igb_free_rx_resources(adapter->rx_ring[i]);
3105 * igb_clean_rx_ring - Free Rx Buffers per Queue
3106 * @rx_ring: ring to free buffers from
3108 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3110 struct igb_buffer *buffer_info;
3114 if (!rx_ring->buffer_info)
3117 /* Free all the Rx ring sk_buffs */
3118 for (i = 0; i < rx_ring->count; i++) {
3119 buffer_info = &rx_ring->buffer_info[i];
3120 if (buffer_info->dma) {
3121 dma_unmap_single(rx_ring->dev,
3123 rx_ring->rx_buffer_len,
3125 buffer_info->dma = 0;
3128 if (buffer_info->skb) {
3129 dev_kfree_skb(buffer_info->skb);
3130 buffer_info->skb = NULL;
3132 if (buffer_info->page_dma) {
3133 dma_unmap_page(rx_ring->dev,
3134 buffer_info->page_dma,
3137 buffer_info->page_dma = 0;
3139 if (buffer_info->page) {
3140 put_page(buffer_info->page);
3141 buffer_info->page = NULL;
3142 buffer_info->page_offset = 0;
3146 size = sizeof(struct igb_buffer) * rx_ring->count;
3147 memset(rx_ring->buffer_info, 0, size);
3149 /* Zero out the descriptor ring */
3150 memset(rx_ring->desc, 0, rx_ring->size);
3152 rx_ring->next_to_clean = 0;
3153 rx_ring->next_to_use = 0;
3157 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3158 * @adapter: board private structure
3160 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3164 for (i = 0; i < adapter->num_rx_queues; i++)
3165 igb_clean_rx_ring(adapter->rx_ring[i]);
3169 * igb_set_mac - Change the Ethernet Address of the NIC
3170 * @netdev: network interface device structure
3171 * @p: pointer to an address structure
3173 * Returns 0 on success, negative on failure
3175 static int igb_set_mac(struct net_device *netdev, void *p)
3177 struct igb_adapter *adapter = netdev_priv(netdev);
3178 struct e1000_hw *hw = &adapter->hw;
3179 struct sockaddr *addr = p;
3181 if (!is_valid_ether_addr(addr->sa_data))
3182 return -EADDRNOTAVAIL;
3184 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3185 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3187 /* set the correct pool for the new PF MAC address in entry 0 */
3188 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3189 adapter->vfs_allocated_count);
3195 * igb_write_mc_addr_list - write multicast addresses to MTA
3196 * @netdev: network interface device structure
3198 * Writes multicast address list to the MTA hash table.
3199 * Returns: -ENOMEM on failure
3200 * 0 on no addresses written
3201 * X on writing X addresses to MTA
3203 static int igb_write_mc_addr_list(struct net_device *netdev)
3205 struct igb_adapter *adapter = netdev_priv(netdev);
3206 struct e1000_hw *hw = &adapter->hw;
3207 struct netdev_hw_addr *ha;
3211 if (netdev_mc_empty(netdev)) {
3212 /* nothing to program, so clear mc list */
3213 igb_update_mc_addr_list(hw, NULL, 0);
3214 igb_restore_vf_multicasts(adapter);
3218 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3222 /* The shared function expects a packed array of only addresses. */
3224 netdev_for_each_mc_addr(ha, netdev)
3225 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3227 igb_update_mc_addr_list(hw, mta_list, i);
3230 return netdev_mc_count(netdev);
3234 * igb_write_uc_addr_list - write unicast addresses to RAR table
3235 * @netdev: network interface device structure
3237 * Writes unicast address list to the RAR table.
3238 * Returns: -ENOMEM on failure/insufficient address space
3239 * 0 on no addresses written
3240 * X on writing X addresses to the RAR table
3242 static int igb_write_uc_addr_list(struct net_device *netdev)
3244 struct igb_adapter *adapter = netdev_priv(netdev);
3245 struct e1000_hw *hw = &adapter->hw;
3246 unsigned int vfn = adapter->vfs_allocated_count;
3247 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3250 /* return ENOMEM indicating insufficient memory for addresses */
3251 if (netdev_uc_count(netdev) > rar_entries)
3254 if (!netdev_uc_empty(netdev) && rar_entries) {
3255 struct netdev_hw_addr *ha;
3257 netdev_for_each_uc_addr(ha, netdev) {
3260 igb_rar_set_qsel(adapter, ha->addr,
3266 /* write the addresses in reverse order to avoid write combining */
3267 for (; rar_entries > 0 ; rar_entries--) {
3268 wr32(E1000_RAH(rar_entries), 0);
3269 wr32(E1000_RAL(rar_entries), 0);
3277 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3278 * @netdev: network interface device structure
3280 * The set_rx_mode entry point is called whenever the unicast or multicast
3281 * address lists or the network interface flags are updated. This routine is
3282 * responsible for configuring the hardware for proper unicast, multicast,
3283 * promiscuous mode, and all-multi behavior.
3285 static void igb_set_rx_mode(struct net_device *netdev)
3287 struct igb_adapter *adapter = netdev_priv(netdev);
3288 struct e1000_hw *hw = &adapter->hw;
3289 unsigned int vfn = adapter->vfs_allocated_count;
3290 u32 rctl, vmolr = 0;
3293 /* Check for Promiscuous and All Multicast modes */
3294 rctl = rd32(E1000_RCTL);
3296 /* clear the effected bits */
3297 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3299 if (netdev->flags & IFF_PROMISC) {
3300 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3301 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3303 if (netdev->flags & IFF_ALLMULTI) {
3304 rctl |= E1000_RCTL_MPE;
3305 vmolr |= E1000_VMOLR_MPME;
3308 * Write addresses to the MTA, if the attempt fails
3309 * then we should just turn on promiscous mode so
3310 * that we can at least receive multicast traffic
3312 count = igb_write_mc_addr_list(netdev);
3314 rctl |= E1000_RCTL_MPE;
3315 vmolr |= E1000_VMOLR_MPME;
3317 vmolr |= E1000_VMOLR_ROMPE;
3321 * Write addresses to available RAR registers, if there is not
3322 * sufficient space to store all the addresses then enable
3323 * unicast promiscous mode
3325 count = igb_write_uc_addr_list(netdev);
3327 rctl |= E1000_RCTL_UPE;
3328 vmolr |= E1000_VMOLR_ROPE;
3330 rctl |= E1000_RCTL_VFE;
3332 wr32(E1000_RCTL, rctl);
3335 * In order to support SR-IOV and eventually VMDq it is necessary to set
3336 * the VMOLR to enable the appropriate modes. Without this workaround
3337 * we will have issues with VLAN tag stripping not being done for frames
3338 * that are only arriving because we are the default pool
3340 if (hw->mac.type < e1000_82576)
3343 vmolr |= rd32(E1000_VMOLR(vfn)) &
3344 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3345 wr32(E1000_VMOLR(vfn), vmolr);
3346 igb_restore_vf_multicasts(adapter);
3349 /* Need to wait a few seconds after link up to get diagnostic information from
3351 static void igb_update_phy_info(unsigned long data)
3353 struct igb_adapter *adapter = (struct igb_adapter *) data;
3354 igb_get_phy_info(&adapter->hw);
3358 * igb_has_link - check shared code for link and determine up/down
3359 * @adapter: pointer to driver private info
3361 bool igb_has_link(struct igb_adapter *adapter)
3363 struct e1000_hw *hw = &adapter->hw;
3364 bool link_active = false;
3367 /* get_link_status is set on LSC (link status) interrupt or
3368 * rx sequence error interrupt. get_link_status will stay
3369 * false until the e1000_check_for_link establishes link
3370 * for copper adapters ONLY
3372 switch (hw->phy.media_type) {
3373 case e1000_media_type_copper:
3374 if (hw->mac.get_link_status) {
3375 ret_val = hw->mac.ops.check_for_link(hw);
3376 link_active = !hw->mac.get_link_status;
3381 case e1000_media_type_internal_serdes:
3382 ret_val = hw->mac.ops.check_for_link(hw);
3383 link_active = hw->mac.serdes_has_link;
3386 case e1000_media_type_unknown:
3394 * igb_watchdog - Timer Call-back
3395 * @data: pointer to adapter cast into an unsigned long
3397 static void igb_watchdog(unsigned long data)
3399 struct igb_adapter *adapter = (struct igb_adapter *)data;
3400 /* Do the rest outside of interrupt context */
3401 schedule_work(&adapter->watchdog_task);
3404 static void igb_watchdog_task(struct work_struct *work)
3406 struct igb_adapter *adapter = container_of(work,
3409 struct e1000_hw *hw = &adapter->hw;
3410 struct net_device *netdev = adapter->netdev;
3414 link = igb_has_link(adapter);
3416 if (!netif_carrier_ok(netdev)) {
3418 hw->mac.ops.get_speed_and_duplex(hw,
3419 &adapter->link_speed,
3420 &adapter->link_duplex);
3422 ctrl = rd32(E1000_CTRL);
3423 /* Links status message must follow this format */
3424 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3425 "Flow Control: %s\n",
3427 adapter->link_speed,
3428 adapter->link_duplex == FULL_DUPLEX ?
3429 "Full Duplex" : "Half Duplex",
3430 ((ctrl & E1000_CTRL_TFCE) &&
3431 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3432 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3433 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3435 /* adjust timeout factor according to speed/duplex */
3436 adapter->tx_timeout_factor = 1;
3437 switch (adapter->link_speed) {
3439 adapter->tx_timeout_factor = 14;
3442 /* maybe add some timeout factor ? */
3446 netif_carrier_on(netdev);
3448 igb_ping_all_vfs(adapter);
3450 /* link state has changed, schedule phy info update */
3451 if (!test_bit(__IGB_DOWN, &adapter->state))
3452 mod_timer(&adapter->phy_info_timer,
3453 round_jiffies(jiffies + 2 * HZ));
3456 if (netif_carrier_ok(netdev)) {
3457 adapter->link_speed = 0;
3458 adapter->link_duplex = 0;
3459 /* Links status message must follow this format */
3460 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3462 netif_carrier_off(netdev);
3464 igb_ping_all_vfs(adapter);
3466 /* link state has changed, schedule phy info update */
3467 if (!test_bit(__IGB_DOWN, &adapter->state))
3468 mod_timer(&adapter->phy_info_timer,
3469 round_jiffies(jiffies + 2 * HZ));
3473 igb_update_stats(adapter);
3475 for (i = 0; i < adapter->num_tx_queues; i++) {
3476 struct igb_ring *tx_ring = adapter->tx_ring[i];
3477 if (!netif_carrier_ok(netdev)) {
3478 /* We've lost link, so the controller stops DMA,
3479 * but we've got queued Tx work that's never going
3480 * to get done, so reset controller to flush Tx.
3481 * (Do the reset outside of interrupt context). */
3482 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3483 adapter->tx_timeout_count++;
3484 schedule_work(&adapter->reset_task);
3485 /* return immediately since reset is imminent */
3490 /* Force detection of hung controller every watchdog period */
3491 tx_ring->detect_tx_hung = true;
3494 /* Cause software interrupt to ensure rx ring is cleaned */
3495 if (adapter->msix_entries) {
3497 for (i = 0; i < adapter->num_q_vectors; i++) {
3498 struct igb_q_vector *q_vector = adapter->q_vector[i];
3499 eics |= q_vector->eims_value;
3501 wr32(E1000_EICS, eics);
3503 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3506 /* Reset the timer */
3507 if (!test_bit(__IGB_DOWN, &adapter->state))
3508 mod_timer(&adapter->watchdog_timer,
3509 round_jiffies(jiffies + 2 * HZ));
3512 enum latency_range {
3516 latency_invalid = 255
3520 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3522 * Stores a new ITR value based on strictly on packet size. This
3523 * algorithm is less sophisticated than that used in igb_update_itr,
3524 * due to the difficulty of synchronizing statistics across multiple
3525 * receive rings. The divisors and thresholds used by this fuction
3526 * were determined based on theoretical maximum wire speed and testing
3527 * data, in order to minimize response time while increasing bulk
3529 * This functionality is controlled by the InterruptThrottleRate module
3530 * parameter (see igb_param.c)
3531 * NOTE: This function is called only when operating in a multiqueue
3532 * receive environment.
3533 * @q_vector: pointer to q_vector
3535 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3537 int new_val = q_vector->itr_val;
3538 int avg_wire_size = 0;
3539 struct igb_adapter *adapter = q_vector->adapter;
3541 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3542 * ints/sec - ITR timer value of 120 ticks.
3544 if (adapter->link_speed != SPEED_1000) {
3549 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3550 struct igb_ring *ring = q_vector->rx_ring;
3551 avg_wire_size = ring->total_bytes / ring->total_packets;
3554 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3555 struct igb_ring *ring = q_vector->tx_ring;
3556 avg_wire_size = max_t(u32, avg_wire_size,
3557 (ring->total_bytes /
3558 ring->total_packets));
3561 /* if avg_wire_size isn't set no work was done */
3565 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3566 avg_wire_size += 24;
3568 /* Don't starve jumbo frames */
3569 avg_wire_size = min(avg_wire_size, 3000);
3571 /* Give a little boost to mid-size frames */
3572 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3573 new_val = avg_wire_size / 3;
3575 new_val = avg_wire_size / 2;
3577 /* when in itr mode 3 do not exceed 20K ints/sec */
3578 if (adapter->rx_itr_setting == 3 && new_val < 196)
3582 if (new_val != q_vector->itr_val) {
3583 q_vector->itr_val = new_val;
3584 q_vector->set_itr = 1;
3587 if (q_vector->rx_ring) {
3588 q_vector->rx_ring->total_bytes = 0;
3589 q_vector->rx_ring->total_packets = 0;
3591 if (q_vector->tx_ring) {
3592 q_vector->tx_ring->total_bytes = 0;
3593 q_vector->tx_ring->total_packets = 0;
3598 * igb_update_itr - update the dynamic ITR value based on statistics
3599 * Stores a new ITR value based on packets and byte
3600 * counts during the last interrupt. The advantage of per interrupt
3601 * computation is faster updates and more accurate ITR for the current
3602 * traffic pattern. Constants in this function were computed
3603 * based on theoretical maximum wire speed and thresholds were set based
3604 * on testing data as well as attempting to minimize response time
3605 * while increasing bulk throughput.
3606 * this functionality is controlled by the InterruptThrottleRate module
3607 * parameter (see igb_param.c)
3608 * NOTE: These calculations are only valid when operating in a single-
3609 * queue environment.
3610 * @adapter: pointer to adapter
3611 * @itr_setting: current q_vector->itr_val
3612 * @packets: the number of packets during this measurement interval
3613 * @bytes: the number of bytes during this measurement interval
3615 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3616 int packets, int bytes)
3618 unsigned int retval = itr_setting;
3621 goto update_itr_done;
3623 switch (itr_setting) {
3624 case lowest_latency:
3625 /* handle TSO and jumbo frames */
3626 if (bytes/packets > 8000)
3627 retval = bulk_latency;
3628 else if ((packets < 5) && (bytes > 512))
3629 retval = low_latency;
3631 case low_latency: /* 50 usec aka 20000 ints/s */
3632 if (bytes > 10000) {
3633 /* this if handles the TSO accounting */
3634 if (bytes/packets > 8000) {
3635 retval = bulk_latency;
3636 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3637 retval = bulk_latency;
3638 } else if ((packets > 35)) {
3639 retval = lowest_latency;
3641 } else if (bytes/packets > 2000) {
3642 retval = bulk_latency;
3643 } else if (packets <= 2 && bytes < 512) {
3644 retval = lowest_latency;
3647 case bulk_latency: /* 250 usec aka 4000 ints/s */
3648 if (bytes > 25000) {
3650 retval = low_latency;
3651 } else if (bytes < 1500) {
3652 retval = low_latency;
3661 static void igb_set_itr(struct igb_adapter *adapter)
3663 struct igb_q_vector *q_vector = adapter->q_vector[0];
3665 u32 new_itr = q_vector->itr_val;
3667 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3668 if (adapter->link_speed != SPEED_1000) {
3674 adapter->rx_itr = igb_update_itr(adapter,
3676 q_vector->rx_ring->total_packets,
3677 q_vector->rx_ring->total_bytes);
3679 adapter->tx_itr = igb_update_itr(adapter,
3681 q_vector->tx_ring->total_packets,
3682 q_vector->tx_ring->total_bytes);
3683 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3685 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3686 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3687 current_itr = low_latency;
3689 switch (current_itr) {
3690 /* counts and packets in update_itr are dependent on these numbers */
3691 case lowest_latency:
3692 new_itr = 56; /* aka 70,000 ints/sec */
3695 new_itr = 196; /* aka 20,000 ints/sec */
3698 new_itr = 980; /* aka 4,000 ints/sec */
3705 q_vector->rx_ring->total_bytes = 0;
3706 q_vector->rx_ring->total_packets = 0;
3707 q_vector->tx_ring->total_bytes = 0;
3708 q_vector->tx_ring->total_packets = 0;
3710 if (new_itr != q_vector->itr_val) {
3711 /* this attempts to bias the interrupt rate towards Bulk
3712 * by adding intermediate steps when interrupt rate is
3714 new_itr = new_itr > q_vector->itr_val ?
3715 max((new_itr * q_vector->itr_val) /
3716 (new_itr + (q_vector->itr_val >> 2)),
3719 /* Don't write the value here; it resets the adapter's
3720 * internal timer, and causes us to delay far longer than
3721 * we should between interrupts. Instead, we write the ITR
3722 * value at the beginning of the next interrupt so the timing
3723 * ends up being correct.
3725 q_vector->itr_val = new_itr;
3726 q_vector->set_itr = 1;
3730 #define IGB_TX_FLAGS_CSUM 0x00000001
3731 #define IGB_TX_FLAGS_VLAN 0x00000002
3732 #define IGB_TX_FLAGS_TSO 0x00000004
3733 #define IGB_TX_FLAGS_IPV4 0x00000008
3734 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3735 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3736 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3738 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3739 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3741 struct e1000_adv_tx_context_desc *context_desc;
3744 struct igb_buffer *buffer_info;
3745 u32 info = 0, tu_cmd = 0;
3749 if (skb_header_cloned(skb)) {
3750 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3755 l4len = tcp_hdrlen(skb);
3758 if (skb->protocol == htons(ETH_P_IP)) {
3759 struct iphdr *iph = ip_hdr(skb);
3762 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3766 } else if (skb_is_gso_v6(skb)) {
3767 ipv6_hdr(skb)->payload_len = 0;
3768 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3769 &ipv6_hdr(skb)->daddr,
3773 i = tx_ring->next_to_use;
3775 buffer_info = &tx_ring->buffer_info[i];
3776 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3777 /* VLAN MACLEN IPLEN */
3778 if (tx_flags & IGB_TX_FLAGS_VLAN)
3779 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3780 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3781 *hdr_len += skb_network_offset(skb);
3782 info |= skb_network_header_len(skb);
3783 *hdr_len += skb_network_header_len(skb);
3784 context_desc->vlan_macip_lens = cpu_to_le32(info);
3786 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3787 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3789 if (skb->protocol == htons(ETH_P_IP))
3790 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3791 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3793 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3796 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3797 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3799 /* For 82575, context index must be unique per ring. */
3800 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3801 mss_l4len_idx |= tx_ring->reg_idx << 4;
3803 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3804 context_desc->seqnum_seed = 0;
3806 buffer_info->time_stamp = jiffies;
3807 buffer_info->next_to_watch = i;
3808 buffer_info->dma = 0;
3810 if (i == tx_ring->count)
3813 tx_ring->next_to_use = i;
3818 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3819 struct sk_buff *skb, u32 tx_flags)
3821 struct e1000_adv_tx_context_desc *context_desc;
3822 struct device *dev = tx_ring->dev;
3823 struct igb_buffer *buffer_info;
3824 u32 info = 0, tu_cmd = 0;
3827 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3828 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3829 i = tx_ring->next_to_use;
3830 buffer_info = &tx_ring->buffer_info[i];
3831 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3833 if (tx_flags & IGB_TX_FLAGS_VLAN)
3834 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3836 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3837 if (skb->ip_summed == CHECKSUM_PARTIAL)
3838 info |= skb_network_header_len(skb);
3840 context_desc->vlan_macip_lens = cpu_to_le32(info);
3842 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3844 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3847 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3848 const struct vlan_ethhdr *vhdr =
3849 (const struct vlan_ethhdr*)skb->data;
3851 protocol = vhdr->h_vlan_encapsulated_proto;
3853 protocol = skb->protocol;
3857 case cpu_to_be16(ETH_P_IP):
3858 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3859 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3860 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3861 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3862 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3864 case cpu_to_be16(ETH_P_IPV6):
3865 /* XXX what about other V6 headers?? */
3866 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3867 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3868 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3869 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3872 if (unlikely(net_ratelimit()))
3874 "partial checksum but proto=%x!\n",
3880 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3881 context_desc->seqnum_seed = 0;
3882 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3883 context_desc->mss_l4len_idx =
3884 cpu_to_le32(tx_ring->reg_idx << 4);
3886 buffer_info->time_stamp = jiffies;
3887 buffer_info->next_to_watch = i;
3888 buffer_info->dma = 0;
3891 if (i == tx_ring->count)
3893 tx_ring->next_to_use = i;
3900 #define IGB_MAX_TXD_PWR 16
3901 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3903 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3906 struct igb_buffer *buffer_info;
3907 struct device *dev = tx_ring->dev;
3908 unsigned int hlen = skb_headlen(skb);
3909 unsigned int count = 0, i;
3911 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3913 i = tx_ring->next_to_use;
3915 buffer_info = &tx_ring->buffer_info[i];
3916 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3917 buffer_info->length = hlen;
3918 /* set time_stamp *before* dma to help avoid a possible race */
3919 buffer_info->time_stamp = jiffies;
3920 buffer_info->next_to_watch = i;
3921 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3923 if (dma_mapping_error(dev, buffer_info->dma))
3926 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3927 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3928 unsigned int len = frag->size;
3932 if (i == tx_ring->count)
3935 buffer_info = &tx_ring->buffer_info[i];
3936 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3937 buffer_info->length = len;
3938 buffer_info->time_stamp = jiffies;
3939 buffer_info->next_to_watch = i;
3940 buffer_info->mapped_as_page = true;
3941 buffer_info->dma = dma_map_page(dev,
3946 if (dma_mapping_error(dev, buffer_info->dma))
3951 tx_ring->buffer_info[i].skb = skb;
3952 tx_ring->buffer_info[i].shtx = skb_shinfo(skb)->tx_flags;
3953 /* multiply data chunks by size of headers */
3954 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3955 tx_ring->buffer_info[i].gso_segs = gso_segs;
3956 tx_ring->buffer_info[first].next_to_watch = i;
3961 dev_err(dev, "TX DMA map failed\n");
3963 /* clear timestamp and dma mappings for failed buffer_info mapping */
3964 buffer_info->dma = 0;
3965 buffer_info->time_stamp = 0;
3966 buffer_info->length = 0;
3967 buffer_info->next_to_watch = 0;
3968 buffer_info->mapped_as_page = false;
3970 /* clear timestamp and dma mappings for remaining portion of packet */
3975 buffer_info = &tx_ring->buffer_info[i];
3976 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3982 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3983 u32 tx_flags, int count, u32 paylen,
3986 union e1000_adv_tx_desc *tx_desc;
3987 struct igb_buffer *buffer_info;
3988 u32 olinfo_status = 0, cmd_type_len;
3989 unsigned int i = tx_ring->next_to_use;
3991 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3992 E1000_ADVTXD_DCMD_DEXT);
3994 if (tx_flags & IGB_TX_FLAGS_VLAN)
3995 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3997 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3998 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4000 if (tx_flags & IGB_TX_FLAGS_TSO) {
4001 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4003 /* insert tcp checksum */
4004 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4006 /* insert ip checksum */
4007 if (tx_flags & IGB_TX_FLAGS_IPV4)
4008 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4010 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4011 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4014 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4015 (tx_flags & (IGB_TX_FLAGS_CSUM |
4017 IGB_TX_FLAGS_VLAN)))
4018 olinfo_status |= tx_ring->reg_idx << 4;
4020 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4023 buffer_info = &tx_ring->buffer_info[i];
4024 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4025 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4026 tx_desc->read.cmd_type_len =
4027 cpu_to_le32(cmd_type_len | buffer_info->length);
4028 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4031 if (i == tx_ring->count)
4033 } while (count > 0);
4035 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4036 /* Force memory writes to complete before letting h/w
4037 * know there are new descriptors to fetch. (Only
4038 * applicable for weak-ordered memory model archs,
4039 * such as IA-64). */
4042 tx_ring->next_to_use = i;
4043 writel(i, tx_ring->tail);
4044 /* we need this if more than one processor can write to our tail
4045 * at a time, it syncronizes IO on IA64/Altix systems */
4049 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4051 struct net_device *netdev = tx_ring->netdev;
4053 netif_stop_subqueue(netdev, tx_ring->queue_index);
4055 /* Herbert's original patch had:
4056 * smp_mb__after_netif_stop_queue();
4057 * but since that doesn't exist yet, just open code it. */
4060 /* We need to check again in a case another CPU has just
4061 * made room available. */
4062 if (igb_desc_unused(tx_ring) < size)
4066 netif_wake_subqueue(netdev, tx_ring->queue_index);
4067 tx_ring->tx_stats.restart_queue++;
4071 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4073 if (igb_desc_unused(tx_ring) >= size)
4075 return __igb_maybe_stop_tx(tx_ring, size);
4078 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4079 struct igb_ring *tx_ring)
4081 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4086 union skb_shared_tx *shtx = skb_tx(skb);
4088 /* need: 1 descriptor per page,
4089 * + 2 desc gap to keep tail from touching head,
4090 * + 1 desc for skb->data,
4091 * + 1 desc for context descriptor,
4092 * otherwise try next time */
4093 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4094 /* this is a hard error */
4095 return NETDEV_TX_BUSY;
4098 if (unlikely(shtx->hardware)) {
4099 shtx->in_progress = 1;
4100 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4103 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
4104 tx_flags |= IGB_TX_FLAGS_VLAN;
4105 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4108 if (skb->protocol == htons(ETH_P_IP))
4109 tx_flags |= IGB_TX_FLAGS_IPV4;
4111 first = tx_ring->next_to_use;
4112 if (skb_is_gso(skb)) {
4113 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4116 dev_kfree_skb_any(skb);
4117 return NETDEV_TX_OK;
4122 tx_flags |= IGB_TX_FLAGS_TSO;
4123 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4124 (skb->ip_summed == CHECKSUM_PARTIAL))
4125 tx_flags |= IGB_TX_FLAGS_CSUM;
4128 * count reflects descriptors mapped, if 0 or less then mapping error
4129 * has occured and we need to rewind the descriptor queue
4131 count = igb_tx_map_adv(tx_ring, skb, first);
4133 dev_kfree_skb_any(skb);
4134 tx_ring->buffer_info[first].time_stamp = 0;
4135 tx_ring->next_to_use = first;
4136 return NETDEV_TX_OK;
4139 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4141 /* Make sure there is space in the ring for the next send. */
4142 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4144 return NETDEV_TX_OK;
4147 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4148 struct net_device *netdev)
4150 struct igb_adapter *adapter = netdev_priv(netdev);
4151 struct igb_ring *tx_ring;
4154 if (test_bit(__IGB_DOWN, &adapter->state)) {
4155 dev_kfree_skb_any(skb);
4156 return NETDEV_TX_OK;
4159 if (skb->len <= 0) {
4160 dev_kfree_skb_any(skb);
4161 return NETDEV_TX_OK;
4164 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4165 tx_ring = adapter->multi_tx_table[r_idx];
4167 /* This goes back to the question of how to logically map a tx queue
4168 * to a flow. Right now, performance is impacted slightly negatively
4169 * if using multiple tx queues. If the stack breaks away from a
4170 * single qdisc implementation, we can look at this again. */
4171 return igb_xmit_frame_ring_adv(skb, tx_ring);
4175 * igb_tx_timeout - Respond to a Tx Hang
4176 * @netdev: network interface device structure
4178 static void igb_tx_timeout(struct net_device *netdev)
4180 struct igb_adapter *adapter = netdev_priv(netdev);
4181 struct e1000_hw *hw = &adapter->hw;
4183 /* Do the reset outside of interrupt context */
4184 adapter->tx_timeout_count++;
4186 if (hw->mac.type == e1000_82580)
4187 hw->dev_spec._82575.global_device_reset = true;
4189 schedule_work(&adapter->reset_task);
4191 (adapter->eims_enable_mask & ~adapter->eims_other));
4194 static void igb_reset_task(struct work_struct *work)
4196 struct igb_adapter *adapter;
4197 adapter = container_of(work, struct igb_adapter, reset_task);
4200 netdev_err(adapter->netdev, "Reset adapter\n");
4201 igb_reinit_locked(adapter);
4205 * igb_get_stats - Get System Network Statistics
4206 * @netdev: network interface device structure
4208 * Returns the address of the device statistics structure.
4209 * The statistics are actually updated from the timer callback.
4211 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4213 /* only return the current stats */
4214 return &netdev->stats;
4218 * igb_change_mtu - Change the Maximum Transfer Unit
4219 * @netdev: network interface device structure
4220 * @new_mtu: new value for maximum frame size
4222 * Returns 0 on success, negative on failure
4224 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4226 struct igb_adapter *adapter = netdev_priv(netdev);
4227 struct pci_dev *pdev = adapter->pdev;
4228 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4229 u32 rx_buffer_len, i;
4231 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4232 dev_err(&pdev->dev, "Invalid MTU setting\n");
4236 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4237 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4241 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4244 /* igb_down has a dependency on max_frame_size */
4245 adapter->max_frame_size = max_frame;
4247 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4248 * means we reserve 2 more, this pushes us to allocate from the next
4250 * i.e. RXBUFFER_2048 --> size-4096 slab
4253 if (adapter->hw.mac.type == e1000_82580)
4254 max_frame += IGB_TS_HDR_LEN;
4256 if (max_frame <= IGB_RXBUFFER_1024)
4257 rx_buffer_len = IGB_RXBUFFER_1024;
4258 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4259 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4261 rx_buffer_len = IGB_RXBUFFER_128;
4263 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4264 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4265 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4267 if ((adapter->hw.mac.type == e1000_82580) &&
4268 (rx_buffer_len == IGB_RXBUFFER_128))
4269 rx_buffer_len += IGB_RXBUFFER_64;
4271 if (netif_running(netdev))
4274 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4275 netdev->mtu, new_mtu);
4276 netdev->mtu = new_mtu;
4278 for (i = 0; i < adapter->num_rx_queues; i++)
4279 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4281 if (netif_running(netdev))
4286 clear_bit(__IGB_RESETTING, &adapter->state);
4292 * igb_update_stats - Update the board statistics counters
4293 * @adapter: board private structure
4296 void igb_update_stats(struct igb_adapter *adapter)
4298 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
4299 struct e1000_hw *hw = &adapter->hw;
4300 struct pci_dev *pdev = adapter->pdev;
4306 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4309 * Prevent stats update while adapter is being reset, or if the pci
4310 * connection is down.
4312 if (adapter->link_speed == 0)
4314 if (pci_channel_offline(pdev))
4319 for (i = 0; i < adapter->num_rx_queues; i++) {
4320 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4321 struct igb_ring *ring = adapter->rx_ring[i];
4322 ring->rx_stats.drops += rqdpc_tmp;
4323 net_stats->rx_fifo_errors += rqdpc_tmp;
4324 bytes += ring->rx_stats.bytes;
4325 packets += ring->rx_stats.packets;
4328 net_stats->rx_bytes = bytes;
4329 net_stats->rx_packets = packets;
4333 for (i = 0; i < adapter->num_tx_queues; i++) {
4334 struct igb_ring *ring = adapter->tx_ring[i];
4335 bytes += ring->tx_stats.bytes;
4336 packets += ring->tx_stats.packets;
4338 net_stats->tx_bytes = bytes;
4339 net_stats->tx_packets = packets;
4341 /* read stats registers */
4342 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4343 adapter->stats.gprc += rd32(E1000_GPRC);
4344 adapter->stats.gorc += rd32(E1000_GORCL);
4345 rd32(E1000_GORCH); /* clear GORCL */
4346 adapter->stats.bprc += rd32(E1000_BPRC);
4347 adapter->stats.mprc += rd32(E1000_MPRC);
4348 adapter->stats.roc += rd32(E1000_ROC);
4350 adapter->stats.prc64 += rd32(E1000_PRC64);
4351 adapter->stats.prc127 += rd32(E1000_PRC127);
4352 adapter->stats.prc255 += rd32(E1000_PRC255);
4353 adapter->stats.prc511 += rd32(E1000_PRC511);
4354 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4355 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4356 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4357 adapter->stats.sec += rd32(E1000_SEC);
4359 mpc = rd32(E1000_MPC);
4360 adapter->stats.mpc += mpc;
4361 net_stats->rx_fifo_errors += mpc;
4362 adapter->stats.scc += rd32(E1000_SCC);
4363 adapter->stats.ecol += rd32(E1000_ECOL);
4364 adapter->stats.mcc += rd32(E1000_MCC);
4365 adapter->stats.latecol += rd32(E1000_LATECOL);
4366 adapter->stats.dc += rd32(E1000_DC);
4367 adapter->stats.rlec += rd32(E1000_RLEC);
4368 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4369 adapter->stats.xontxc += rd32(E1000_XONTXC);
4370 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4371 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4372 adapter->stats.fcruc += rd32(E1000_FCRUC);
4373 adapter->stats.gptc += rd32(E1000_GPTC);
4374 adapter->stats.gotc += rd32(E1000_GOTCL);
4375 rd32(E1000_GOTCH); /* clear GOTCL */
4376 adapter->stats.rnbc += rd32(E1000_RNBC);
4377 adapter->stats.ruc += rd32(E1000_RUC);
4378 adapter->stats.rfc += rd32(E1000_RFC);
4379 adapter->stats.rjc += rd32(E1000_RJC);
4380 adapter->stats.tor += rd32(E1000_TORH);
4381 adapter->stats.tot += rd32(E1000_TOTH);
4382 adapter->stats.tpr += rd32(E1000_TPR);
4384 adapter->stats.ptc64 += rd32(E1000_PTC64);
4385 adapter->stats.ptc127 += rd32(E1000_PTC127);
4386 adapter->stats.ptc255 += rd32(E1000_PTC255);
4387 adapter->stats.ptc511 += rd32(E1000_PTC511);
4388 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4389 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4391 adapter->stats.mptc += rd32(E1000_MPTC);
4392 adapter->stats.bptc += rd32(E1000_BPTC);
4394 adapter->stats.tpt += rd32(E1000_TPT);
4395 adapter->stats.colc += rd32(E1000_COLC);
4397 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4398 /* read internal phy specific stats */
4399 reg = rd32(E1000_CTRL_EXT);
4400 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4401 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4402 adapter->stats.tncrs += rd32(E1000_TNCRS);
4405 adapter->stats.tsctc += rd32(E1000_TSCTC);
4406 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4408 adapter->stats.iac += rd32(E1000_IAC);
4409 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4410 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4411 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4412 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4413 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4414 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4415 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4416 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4418 /* Fill out the OS statistics structure */
4419 net_stats->multicast = adapter->stats.mprc;
4420 net_stats->collisions = adapter->stats.colc;
4424 /* RLEC on some newer hardware can be incorrect so build
4425 * our own version based on RUC and ROC */
4426 net_stats->rx_errors = adapter->stats.rxerrc +
4427 adapter->stats.crcerrs + adapter->stats.algnerrc +
4428 adapter->stats.ruc + adapter->stats.roc +
4429 adapter->stats.cexterr;
4430 net_stats->rx_length_errors = adapter->stats.ruc +
4432 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4433 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4434 net_stats->rx_missed_errors = adapter->stats.mpc;
4437 net_stats->tx_errors = adapter->stats.ecol +
4438 adapter->stats.latecol;
4439 net_stats->tx_aborted_errors = adapter->stats.ecol;
4440 net_stats->tx_window_errors = adapter->stats.latecol;
4441 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4443 /* Tx Dropped needs to be maintained elsewhere */
4446 if (hw->phy.media_type == e1000_media_type_copper) {
4447 if ((adapter->link_speed == SPEED_1000) &&
4448 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4449 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4450 adapter->phy_stats.idle_errors += phy_tmp;
4454 /* Management Stats */
4455 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4456 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4457 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4460 static irqreturn_t igb_msix_other(int irq, void *data)
4462 struct igb_adapter *adapter = data;
4463 struct e1000_hw *hw = &adapter->hw;
4464 u32 icr = rd32(E1000_ICR);
4465 /* reading ICR causes bit 31 of EICR to be cleared */
4467 if (icr & E1000_ICR_DRSTA)
4468 schedule_work(&adapter->reset_task);
4470 if (icr & E1000_ICR_DOUTSYNC) {
4471 /* HW is reporting DMA is out of sync */
4472 adapter->stats.doosync++;
4475 /* Check for a mailbox event */
4476 if (icr & E1000_ICR_VMMB)
4477 igb_msg_task(adapter);
4479 if (icr & E1000_ICR_LSC) {
4480 hw->mac.get_link_status = 1;
4481 /* guard against interrupt when we're going down */
4482 if (!test_bit(__IGB_DOWN, &adapter->state))
4483 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4486 if (adapter->vfs_allocated_count)
4487 wr32(E1000_IMS, E1000_IMS_LSC |
4489 E1000_IMS_DOUTSYNC);
4491 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4492 wr32(E1000_EIMS, adapter->eims_other);
4497 static void igb_write_itr(struct igb_q_vector *q_vector)
4499 struct igb_adapter *adapter = q_vector->adapter;
4500 u32 itr_val = q_vector->itr_val & 0x7FFC;
4502 if (!q_vector->set_itr)
4508 if (adapter->hw.mac.type == e1000_82575)
4509 itr_val |= itr_val << 16;
4511 itr_val |= 0x8000000;
4513 writel(itr_val, q_vector->itr_register);
4514 q_vector->set_itr = 0;
4517 static irqreturn_t igb_msix_ring(int irq, void *data)
4519 struct igb_q_vector *q_vector = data;
4521 /* Write the ITR value calculated from the previous interrupt. */
4522 igb_write_itr(q_vector);
4524 napi_schedule(&q_vector->napi);
4529 #ifdef CONFIG_IGB_DCA
4530 static void igb_update_dca(struct igb_q_vector *q_vector)
4532 struct igb_adapter *adapter = q_vector->adapter;
4533 struct e1000_hw *hw = &adapter->hw;
4534 int cpu = get_cpu();
4536 if (q_vector->cpu == cpu)
4539 if (q_vector->tx_ring) {
4540 int q = q_vector->tx_ring->reg_idx;
4541 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4542 if (hw->mac.type == e1000_82575) {
4543 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4544 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4546 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4547 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4548 E1000_DCA_TXCTRL_CPUID_SHIFT;
4550 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4551 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4553 if (q_vector->rx_ring) {
4554 int q = q_vector->rx_ring->reg_idx;
4555 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4556 if (hw->mac.type == e1000_82575) {
4557 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4558 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4560 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4561 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4562 E1000_DCA_RXCTRL_CPUID_SHIFT;
4564 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4565 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4566 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4567 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4569 q_vector->cpu = cpu;
4574 static void igb_setup_dca(struct igb_adapter *adapter)
4576 struct e1000_hw *hw = &adapter->hw;
4579 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4582 /* Always use CB2 mode, difference is masked in the CB driver. */
4583 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4585 for (i = 0; i < adapter->num_q_vectors; i++) {
4586 adapter->q_vector[i]->cpu = -1;
4587 igb_update_dca(adapter->q_vector[i]);
4591 static int __igb_notify_dca(struct device *dev, void *data)
4593 struct net_device *netdev = dev_get_drvdata(dev);
4594 struct igb_adapter *adapter = netdev_priv(netdev);
4595 struct pci_dev *pdev = adapter->pdev;
4596 struct e1000_hw *hw = &adapter->hw;
4597 unsigned long event = *(unsigned long *)data;
4600 case DCA_PROVIDER_ADD:
4601 /* if already enabled, don't do it again */
4602 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4604 if (dca_add_requester(dev) == 0) {
4605 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4606 dev_info(&pdev->dev, "DCA enabled\n");
4607 igb_setup_dca(adapter);
4610 /* Fall Through since DCA is disabled. */
4611 case DCA_PROVIDER_REMOVE:
4612 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4613 /* without this a class_device is left
4614 * hanging around in the sysfs model */
4615 dca_remove_requester(dev);
4616 dev_info(&pdev->dev, "DCA disabled\n");
4617 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4618 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4626 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4631 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4634 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4636 #endif /* CONFIG_IGB_DCA */
4638 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4640 struct e1000_hw *hw = &adapter->hw;
4644 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4645 ping = E1000_PF_CONTROL_MSG;
4646 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4647 ping |= E1000_VT_MSGTYPE_CTS;
4648 igb_write_mbx(hw, &ping, 1, i);
4652 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4654 struct e1000_hw *hw = &adapter->hw;
4655 u32 vmolr = rd32(E1000_VMOLR(vf));
4656 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4658 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4659 IGB_VF_FLAG_MULTI_PROMISC);
4660 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4662 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4663 vmolr |= E1000_VMOLR_MPME;
4664 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4667 * if we have hashes and we are clearing a multicast promisc
4668 * flag we need to write the hashes to the MTA as this step
4669 * was previously skipped
4671 if (vf_data->num_vf_mc_hashes > 30) {
4672 vmolr |= E1000_VMOLR_MPME;
4673 } else if (vf_data->num_vf_mc_hashes) {
4675 vmolr |= E1000_VMOLR_ROMPE;
4676 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4677 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4681 wr32(E1000_VMOLR(vf), vmolr);
4683 /* there are flags left unprocessed, likely not supported */
4684 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4691 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4692 u32 *msgbuf, u32 vf)
4694 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4695 u16 *hash_list = (u16 *)&msgbuf[1];
4696 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4699 /* salt away the number of multicast addresses assigned
4700 * to this VF for later use to restore when the PF multi cast
4703 vf_data->num_vf_mc_hashes = n;
4705 /* only up to 30 hash values supported */
4709 /* store the hashes for later use */
4710 for (i = 0; i < n; i++)
4711 vf_data->vf_mc_hashes[i] = hash_list[i];
4713 /* Flush and reset the mta with the new values */
4714 igb_set_rx_mode(adapter->netdev);
4719 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4721 struct e1000_hw *hw = &adapter->hw;
4722 struct vf_data_storage *vf_data;
4725 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4726 u32 vmolr = rd32(E1000_VMOLR(i));
4727 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4729 vf_data = &adapter->vf_data[i];
4731 if ((vf_data->num_vf_mc_hashes > 30) ||
4732 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4733 vmolr |= E1000_VMOLR_MPME;
4734 } else if (vf_data->num_vf_mc_hashes) {
4735 vmolr |= E1000_VMOLR_ROMPE;
4736 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4737 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4739 wr32(E1000_VMOLR(i), vmolr);
4743 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4745 struct e1000_hw *hw = &adapter->hw;
4746 u32 pool_mask, reg, vid;
4749 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4751 /* Find the vlan filter for this id */
4752 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4753 reg = rd32(E1000_VLVF(i));
4755 /* remove the vf from the pool */
4758 /* if pool is empty then remove entry from vfta */
4759 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4760 (reg & E1000_VLVF_VLANID_ENABLE)) {
4762 vid = reg & E1000_VLVF_VLANID_MASK;
4763 igb_vfta_set(hw, vid, false);
4766 wr32(E1000_VLVF(i), reg);
4769 adapter->vf_data[vf].vlans_enabled = 0;
4772 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4774 struct e1000_hw *hw = &adapter->hw;
4777 /* The vlvf table only exists on 82576 hardware and newer */
4778 if (hw->mac.type < e1000_82576)
4781 /* we only need to do this if VMDq is enabled */
4782 if (!adapter->vfs_allocated_count)
4785 /* Find the vlan filter for this id */
4786 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4787 reg = rd32(E1000_VLVF(i));
4788 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4789 vid == (reg & E1000_VLVF_VLANID_MASK))
4794 if (i == E1000_VLVF_ARRAY_SIZE) {
4795 /* Did not find a matching VLAN ID entry that was
4796 * enabled. Search for a free filter entry, i.e.
4797 * one without the enable bit set
4799 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4800 reg = rd32(E1000_VLVF(i));
4801 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4805 if (i < E1000_VLVF_ARRAY_SIZE) {
4806 /* Found an enabled/available entry */
4807 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4809 /* if !enabled we need to set this up in vfta */
4810 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4811 /* add VID to filter table */
4812 igb_vfta_set(hw, vid, true);
4813 reg |= E1000_VLVF_VLANID_ENABLE;
4815 reg &= ~E1000_VLVF_VLANID_MASK;
4817 wr32(E1000_VLVF(i), reg);
4819 /* do not modify RLPML for PF devices */
4820 if (vf >= adapter->vfs_allocated_count)
4823 if (!adapter->vf_data[vf].vlans_enabled) {
4825 reg = rd32(E1000_VMOLR(vf));
4826 size = reg & E1000_VMOLR_RLPML_MASK;
4828 reg &= ~E1000_VMOLR_RLPML_MASK;
4830 wr32(E1000_VMOLR(vf), reg);
4833 adapter->vf_data[vf].vlans_enabled++;
4837 if (i < E1000_VLVF_ARRAY_SIZE) {
4838 /* remove vf from the pool */
4839 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4840 /* if pool is empty then remove entry from vfta */
4841 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4843 igb_vfta_set(hw, vid, false);
4845 wr32(E1000_VLVF(i), reg);
4847 /* do not modify RLPML for PF devices */
4848 if (vf >= adapter->vfs_allocated_count)
4851 adapter->vf_data[vf].vlans_enabled--;
4852 if (!adapter->vf_data[vf].vlans_enabled) {
4854 reg = rd32(E1000_VMOLR(vf));
4855 size = reg & E1000_VMOLR_RLPML_MASK;
4857 reg &= ~E1000_VMOLR_RLPML_MASK;
4859 wr32(E1000_VMOLR(vf), reg);
4866 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4868 struct e1000_hw *hw = &adapter->hw;
4871 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4873 wr32(E1000_VMVIR(vf), 0);
4876 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4877 int vf, u16 vlan, u8 qos)
4880 struct igb_adapter *adapter = netdev_priv(netdev);
4882 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4885 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4888 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4889 igb_set_vmolr(adapter, vf, !vlan);
4890 adapter->vf_data[vf].pf_vlan = vlan;
4891 adapter->vf_data[vf].pf_qos = qos;
4892 dev_info(&adapter->pdev->dev,
4893 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4894 if (test_bit(__IGB_DOWN, &adapter->state)) {
4895 dev_warn(&adapter->pdev->dev,
4896 "The VF VLAN has been set,"
4897 " but the PF device is not up.\n");
4898 dev_warn(&adapter->pdev->dev,
4899 "Bring the PF device up before"
4900 " attempting to use the VF device.\n");
4903 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4905 igb_set_vmvir(adapter, vlan, vf);
4906 igb_set_vmolr(adapter, vf, true);
4907 adapter->vf_data[vf].pf_vlan = 0;
4908 adapter->vf_data[vf].pf_qos = 0;
4914 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4916 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4917 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4919 return igb_vlvf_set(adapter, vid, add, vf);
4922 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4925 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4926 adapter->vf_data[vf].last_nack = jiffies;
4928 /* reset offloads to defaults */
4929 igb_set_vmolr(adapter, vf, true);
4931 /* reset vlans for device */
4932 igb_clear_vf_vfta(adapter, vf);
4933 if (adapter->vf_data[vf].pf_vlan)
4934 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4935 adapter->vf_data[vf].pf_vlan,
4936 adapter->vf_data[vf].pf_qos);
4938 igb_clear_vf_vfta(adapter, vf);
4940 /* reset multicast table array for vf */
4941 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4943 /* Flush and reset the mta with the new values */
4944 igb_set_rx_mode(adapter->netdev);
4947 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4949 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4951 /* generate a new mac address as we were hotplug removed/added */
4952 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4953 random_ether_addr(vf_mac);
4955 /* process remaining reset events */
4956 igb_vf_reset(adapter, vf);
4959 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4961 struct e1000_hw *hw = &adapter->hw;
4962 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4963 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4965 u8 *addr = (u8 *)(&msgbuf[1]);
4967 /* process all the same items cleared in a function level reset */
4968 igb_vf_reset(adapter, vf);
4970 /* set vf mac address */
4971 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4973 /* enable transmit and receive for vf */
4974 reg = rd32(E1000_VFTE);
4975 wr32(E1000_VFTE, reg | (1 << vf));
4976 reg = rd32(E1000_VFRE);
4977 wr32(E1000_VFRE, reg | (1 << vf));
4979 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4981 /* reply to reset with ack and vf mac address */
4982 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4983 memcpy(addr, vf_mac, 6);
4984 igb_write_mbx(hw, msgbuf, 3, vf);
4987 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4989 unsigned char *addr = (char *)&msg[1];
4992 if (is_valid_ether_addr(addr))
4993 err = igb_set_vf_mac(adapter, vf, addr);
4998 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5000 struct e1000_hw *hw = &adapter->hw;
5001 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5002 u32 msg = E1000_VT_MSGTYPE_NACK;
5004 /* if device isn't clear to send it shouldn't be reading either */
5005 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5006 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5007 igb_write_mbx(hw, &msg, 1, vf);
5008 vf_data->last_nack = jiffies;
5012 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5014 struct pci_dev *pdev = adapter->pdev;
5015 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5016 struct e1000_hw *hw = &adapter->hw;
5017 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5020 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5023 /* if receive failed revoke VF CTS stats and restart init */
5024 dev_err(&pdev->dev, "Error receiving message from VF\n");
5025 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5026 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5031 /* this is a message we already processed, do nothing */
5032 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5036 * until the vf completes a reset it should not be
5037 * allowed to start any configuration.
5040 if (msgbuf[0] == E1000_VF_RESET) {
5041 igb_vf_reset_msg(adapter, vf);
5045 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5046 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5052 switch ((msgbuf[0] & 0xFFFF)) {
5053 case E1000_VF_SET_MAC_ADDR:
5054 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5056 case E1000_VF_SET_PROMISC:
5057 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5059 case E1000_VF_SET_MULTICAST:
5060 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5062 case E1000_VF_SET_LPE:
5063 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5065 case E1000_VF_SET_VLAN:
5066 if (adapter->vf_data[vf].pf_vlan)
5069 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5072 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5077 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5079 /* notify the VF of the results of what it sent us */
5081 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5083 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5085 igb_write_mbx(hw, msgbuf, 1, vf);
5088 static void igb_msg_task(struct igb_adapter *adapter)
5090 struct e1000_hw *hw = &adapter->hw;
5093 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5094 /* process any reset requests */
5095 if (!igb_check_for_rst(hw, vf))
5096 igb_vf_reset_event(adapter, vf);
5098 /* process any messages pending */
5099 if (!igb_check_for_msg(hw, vf))
5100 igb_rcv_msg_from_vf(adapter, vf);
5102 /* process any acks */
5103 if (!igb_check_for_ack(hw, vf))
5104 igb_rcv_ack_from_vf(adapter, vf);
5109 * igb_set_uta - Set unicast filter table address
5110 * @adapter: board private structure
5112 * The unicast table address is a register array of 32-bit registers.
5113 * The table is meant to be used in a way similar to how the MTA is used
5114 * however due to certain limitations in the hardware it is necessary to
5115 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5116 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5118 static void igb_set_uta(struct igb_adapter *adapter)
5120 struct e1000_hw *hw = &adapter->hw;
5123 /* The UTA table only exists on 82576 hardware and newer */
5124 if (hw->mac.type < e1000_82576)
5127 /* we only need to do this if VMDq is enabled */
5128 if (!adapter->vfs_allocated_count)
5131 for (i = 0; i < hw->mac.uta_reg_count; i++)
5132 array_wr32(E1000_UTA, i, ~0);
5136 * igb_intr_msi - Interrupt Handler
5137 * @irq: interrupt number
5138 * @data: pointer to a network interface device structure
5140 static irqreturn_t igb_intr_msi(int irq, void *data)
5142 struct igb_adapter *adapter = data;
5143 struct igb_q_vector *q_vector = adapter->q_vector[0];
5144 struct e1000_hw *hw = &adapter->hw;
5145 /* read ICR disables interrupts using IAM */
5146 u32 icr = rd32(E1000_ICR);
5148 igb_write_itr(q_vector);
5150 if (icr & E1000_ICR_DRSTA)
5151 schedule_work(&adapter->reset_task);
5153 if (icr & E1000_ICR_DOUTSYNC) {
5154 /* HW is reporting DMA is out of sync */
5155 adapter->stats.doosync++;
5158 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5159 hw->mac.get_link_status = 1;
5160 if (!test_bit(__IGB_DOWN, &adapter->state))
5161 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5164 napi_schedule(&q_vector->napi);
5170 * igb_intr - Legacy Interrupt Handler
5171 * @irq: interrupt number
5172 * @data: pointer to a network interface device structure
5174 static irqreturn_t igb_intr(int irq, void *data)
5176 struct igb_adapter *adapter = data;
5177 struct igb_q_vector *q_vector = adapter->q_vector[0];
5178 struct e1000_hw *hw = &adapter->hw;
5179 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5180 * need for the IMC write */
5181 u32 icr = rd32(E1000_ICR);
5183 return IRQ_NONE; /* Not our interrupt */
5185 igb_write_itr(q_vector);
5187 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5188 * not set, then the adapter didn't send an interrupt */
5189 if (!(icr & E1000_ICR_INT_ASSERTED))
5192 if (icr & E1000_ICR_DRSTA)
5193 schedule_work(&adapter->reset_task);
5195 if (icr & E1000_ICR_DOUTSYNC) {
5196 /* HW is reporting DMA is out of sync */
5197 adapter->stats.doosync++;
5200 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5201 hw->mac.get_link_status = 1;
5202 /* guard against interrupt when we're going down */
5203 if (!test_bit(__IGB_DOWN, &adapter->state))
5204 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5207 napi_schedule(&q_vector->napi);
5212 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5214 struct igb_adapter *adapter = q_vector->adapter;
5215 struct e1000_hw *hw = &adapter->hw;
5217 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5218 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5219 if (!adapter->msix_entries)
5220 igb_set_itr(adapter);
5222 igb_update_ring_itr(q_vector);
5225 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5226 if (adapter->msix_entries)
5227 wr32(E1000_EIMS, q_vector->eims_value);
5229 igb_irq_enable(adapter);
5234 * igb_poll - NAPI Rx polling callback
5235 * @napi: napi polling structure
5236 * @budget: count of how many packets we should handle
5238 static int igb_poll(struct napi_struct *napi, int budget)
5240 struct igb_q_vector *q_vector = container_of(napi,
5241 struct igb_q_vector,
5243 int tx_clean_complete = 1, work_done = 0;
5245 #ifdef CONFIG_IGB_DCA
5246 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5247 igb_update_dca(q_vector);
5249 if (q_vector->tx_ring)
5250 tx_clean_complete = igb_clean_tx_irq(q_vector);
5252 if (q_vector->rx_ring)
5253 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5255 if (!tx_clean_complete)
5258 /* If not enough Rx work done, exit the polling mode */
5259 if (work_done < budget) {
5260 napi_complete(napi);
5261 igb_ring_irq_enable(q_vector);
5268 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5269 * @adapter: board private structure
5270 * @shhwtstamps: timestamp structure to update
5271 * @regval: unsigned 64bit system time value.
5273 * We need to convert the system time value stored in the RX/TXSTMP registers
5274 * into a hwtstamp which can be used by the upper level timestamping functions
5276 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5277 struct skb_shared_hwtstamps *shhwtstamps,
5283 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5284 * 24 to match clock shift we setup earlier.
5286 if (adapter->hw.mac.type == e1000_82580)
5287 regval <<= IGB_82580_TSYNC_SHIFT;
5289 ns = timecounter_cyc2time(&adapter->clock, regval);
5290 timecompare_update(&adapter->compare, ns);
5291 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5292 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5293 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5297 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5298 * @q_vector: pointer to q_vector containing needed info
5299 * @buffer: pointer to igb_buffer structure
5301 * If we were asked to do hardware stamping and such a time stamp is
5302 * available, then it must have been for this skb here because we only
5303 * allow only one such packet into the queue.
5305 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5307 struct igb_adapter *adapter = q_vector->adapter;
5308 struct e1000_hw *hw = &adapter->hw;
5309 struct skb_shared_hwtstamps shhwtstamps;
5312 /* if skb does not support hw timestamp or TX stamp not valid exit */
5313 if (likely(!buffer_info->shtx.hardware) ||
5314 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5317 regval = rd32(E1000_TXSTMPL);
5318 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5320 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5321 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5325 * igb_clean_tx_irq - Reclaim resources after transmit completes
5326 * @q_vector: pointer to q_vector containing needed info
5327 * returns true if ring is completely cleaned
5329 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5331 struct igb_adapter *adapter = q_vector->adapter;
5332 struct igb_ring *tx_ring = q_vector->tx_ring;
5333 struct net_device *netdev = tx_ring->netdev;
5334 struct e1000_hw *hw = &adapter->hw;
5335 struct igb_buffer *buffer_info;
5336 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5337 unsigned int total_bytes = 0, total_packets = 0;
5338 unsigned int i, eop, count = 0;
5339 bool cleaned = false;
5341 i = tx_ring->next_to_clean;
5342 eop = tx_ring->buffer_info[i].next_to_watch;
5343 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5345 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5346 (count < tx_ring->count)) {
5347 for (cleaned = false; !cleaned; count++) {
5348 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5349 buffer_info = &tx_ring->buffer_info[i];
5350 cleaned = (i == eop);
5352 if (buffer_info->skb) {
5353 total_bytes += buffer_info->bytecount;
5354 /* gso_segs is currently only valid for tcp */
5355 total_packets += buffer_info->gso_segs;
5356 igb_tx_hwtstamp(q_vector, buffer_info);
5359 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5360 tx_desc->wb.status = 0;
5363 if (i == tx_ring->count)
5366 eop = tx_ring->buffer_info[i].next_to_watch;
5367 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5370 tx_ring->next_to_clean = i;
5372 if (unlikely(count &&
5373 netif_carrier_ok(netdev) &&
5374 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5375 /* Make sure that anybody stopping the queue after this
5376 * sees the new next_to_clean.
5379 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5380 !(test_bit(__IGB_DOWN, &adapter->state))) {
5381 netif_wake_subqueue(netdev, tx_ring->queue_index);
5382 tx_ring->tx_stats.restart_queue++;
5386 if (tx_ring->detect_tx_hung) {
5387 /* Detect a transmit hang in hardware, this serializes the
5388 * check with the clearing of time_stamp and movement of i */
5389 tx_ring->detect_tx_hung = false;
5390 if (tx_ring->buffer_info[i].time_stamp &&
5391 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5392 (adapter->tx_timeout_factor * HZ)) &&
5393 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5395 /* detected Tx unit hang */
5396 dev_err(tx_ring->dev,
5397 "Detected Tx Unit Hang\n"
5401 " next_to_use <%x>\n"
5402 " next_to_clean <%x>\n"
5403 "buffer_info[next_to_clean]\n"
5404 " time_stamp <%lx>\n"
5405 " next_to_watch <%x>\n"
5407 " desc.status <%x>\n",
5408 tx_ring->queue_index,
5409 readl(tx_ring->head),
5410 readl(tx_ring->tail),
5411 tx_ring->next_to_use,
5412 tx_ring->next_to_clean,
5413 tx_ring->buffer_info[eop].time_stamp,
5416 eop_desc->wb.status);
5417 netif_stop_subqueue(netdev, tx_ring->queue_index);
5420 tx_ring->total_bytes += total_bytes;
5421 tx_ring->total_packets += total_packets;
5422 tx_ring->tx_stats.bytes += total_bytes;
5423 tx_ring->tx_stats.packets += total_packets;
5424 return (count < tx_ring->count);
5428 * igb_receive_skb - helper function to handle rx indications
5429 * @q_vector: structure containing interrupt and ring information
5430 * @skb: packet to send up
5431 * @vlan_tag: vlan tag for packet
5433 static void igb_receive_skb(struct igb_q_vector *q_vector,
5434 struct sk_buff *skb,
5437 struct igb_adapter *adapter = q_vector->adapter;
5439 if (vlan_tag && adapter->vlgrp)
5440 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5443 napi_gro_receive(&q_vector->napi, skb);
5446 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5447 u32 status_err, struct sk_buff *skb)
5449 skb->ip_summed = CHECKSUM_NONE;
5451 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5452 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5453 (status_err & E1000_RXD_STAT_IXSM))
5456 /* TCP/UDP checksum error bit is set */
5458 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5460 * work around errata with sctp packets where the TCPE aka
5461 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5462 * packets, (aka let the stack check the crc32c)
5464 if ((skb->len == 60) &&
5465 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5466 ring->rx_stats.csum_err++;
5468 /* let the stack verify checksum errors */
5471 /* It must be a TCP or UDP packet with a valid checksum */
5472 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5473 skb->ip_summed = CHECKSUM_UNNECESSARY;
5475 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5478 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5479 struct sk_buff *skb)
5481 struct igb_adapter *adapter = q_vector->adapter;
5482 struct e1000_hw *hw = &adapter->hw;
5486 * If this bit is set, then the RX registers contain the time stamp. No
5487 * other packet will be time stamped until we read these registers, so
5488 * read the registers to make them available again. Because only one
5489 * packet can be time stamped at a time, we know that the register
5490 * values must belong to this one here and therefore we don't need to
5491 * compare any of the additional attributes stored for it.
5493 * If nothing went wrong, then it should have a skb_shared_tx that we
5494 * can turn into a skb_shared_hwtstamps.
5496 if (staterr & E1000_RXDADV_STAT_TSIP) {
5497 u32 *stamp = (u32 *)skb->data;
5498 regval = le32_to_cpu(*(stamp + 2));
5499 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5500 skb_pull(skb, IGB_TS_HDR_LEN);
5502 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5505 regval = rd32(E1000_RXSTMPL);
5506 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5509 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5511 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5512 union e1000_adv_rx_desc *rx_desc)
5514 /* HW will not DMA in data larger than the given buffer, even if it
5515 * parses the (NFS, of course) header to be larger. In that case, it
5516 * fills the header buffer and spills the rest into the page.
5518 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5519 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5520 if (hlen > rx_ring->rx_buffer_len)
5521 hlen = rx_ring->rx_buffer_len;
5525 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5526 int *work_done, int budget)
5528 struct igb_ring *rx_ring = q_vector->rx_ring;
5529 struct net_device *netdev = rx_ring->netdev;
5530 struct device *dev = rx_ring->dev;
5531 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5532 struct igb_buffer *buffer_info , *next_buffer;
5533 struct sk_buff *skb;
5534 bool cleaned = false;
5535 int cleaned_count = 0;
5536 int current_node = numa_node_id();
5537 unsigned int total_bytes = 0, total_packets = 0;
5543 i = rx_ring->next_to_clean;
5544 buffer_info = &rx_ring->buffer_info[i];
5545 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5546 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5548 while (staterr & E1000_RXD_STAT_DD) {
5549 if (*work_done >= budget)
5553 skb = buffer_info->skb;
5554 prefetch(skb->data - NET_IP_ALIGN);
5555 buffer_info->skb = NULL;
5558 if (i == rx_ring->count)
5561 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5563 next_buffer = &rx_ring->buffer_info[i];
5565 length = le16_to_cpu(rx_desc->wb.upper.length);
5569 if (buffer_info->dma) {
5570 dma_unmap_single(dev, buffer_info->dma,
5571 rx_ring->rx_buffer_len,
5573 buffer_info->dma = 0;
5574 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5575 skb_put(skb, length);
5578 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5582 dma_unmap_page(dev, buffer_info->page_dma,
5583 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5584 buffer_info->page_dma = 0;
5586 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5588 buffer_info->page_offset,
5591 if ((page_count(buffer_info->page) != 1) ||
5592 (page_to_nid(buffer_info->page) != current_node))
5593 buffer_info->page = NULL;
5595 get_page(buffer_info->page);
5598 skb->data_len += length;
5599 skb->truesize += length;
5602 if (!(staterr & E1000_RXD_STAT_EOP)) {
5603 buffer_info->skb = next_buffer->skb;
5604 buffer_info->dma = next_buffer->dma;
5605 next_buffer->skb = skb;
5606 next_buffer->dma = 0;
5610 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5611 dev_kfree_skb_irq(skb);
5615 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5616 igb_rx_hwtstamp(q_vector, staterr, skb);
5617 total_bytes += skb->len;
5620 igb_rx_checksum_adv(rx_ring, staterr, skb);
5622 skb->protocol = eth_type_trans(skb, netdev);
5623 skb_record_rx_queue(skb, rx_ring->queue_index);
5625 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5626 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5628 igb_receive_skb(q_vector, skb, vlan_tag);
5631 rx_desc->wb.upper.status_error = 0;
5633 /* return some buffers to hardware, one at a time is too slow */
5634 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5635 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5639 /* use prefetched values */
5641 buffer_info = next_buffer;
5642 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5645 rx_ring->next_to_clean = i;
5646 cleaned_count = igb_desc_unused(rx_ring);
5649 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5651 rx_ring->total_packets += total_packets;
5652 rx_ring->total_bytes += total_bytes;
5653 rx_ring->rx_stats.packets += total_packets;
5654 rx_ring->rx_stats.bytes += total_bytes;
5659 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5660 * @adapter: address of board private structure
5662 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5664 struct net_device *netdev = rx_ring->netdev;
5665 union e1000_adv_rx_desc *rx_desc;
5666 struct igb_buffer *buffer_info;
5667 struct sk_buff *skb;
5671 i = rx_ring->next_to_use;
5672 buffer_info = &rx_ring->buffer_info[i];
5674 bufsz = rx_ring->rx_buffer_len;
5676 while (cleaned_count--) {
5677 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5679 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5680 if (!buffer_info->page) {
5681 buffer_info->page = netdev_alloc_page(netdev);
5682 if (!buffer_info->page) {
5683 rx_ring->rx_stats.alloc_failed++;
5686 buffer_info->page_offset = 0;
5688 buffer_info->page_offset ^= PAGE_SIZE / 2;
5690 buffer_info->page_dma =
5691 dma_map_page(rx_ring->dev, buffer_info->page,
5692 buffer_info->page_offset,
5695 if (dma_mapping_error(rx_ring->dev,
5696 buffer_info->page_dma)) {
5697 buffer_info->page_dma = 0;
5698 rx_ring->rx_stats.alloc_failed++;
5703 skb = buffer_info->skb;
5705 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5707 rx_ring->rx_stats.alloc_failed++;
5711 buffer_info->skb = skb;
5713 if (!buffer_info->dma) {
5714 buffer_info->dma = dma_map_single(rx_ring->dev,
5718 if (dma_mapping_error(rx_ring->dev,
5719 buffer_info->dma)) {
5720 buffer_info->dma = 0;
5721 rx_ring->rx_stats.alloc_failed++;
5725 /* Refresh the desc even if buffer_addrs didn't change because
5726 * each write-back erases this info. */
5727 if (bufsz < IGB_RXBUFFER_1024) {
5728 rx_desc->read.pkt_addr =
5729 cpu_to_le64(buffer_info->page_dma);
5730 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5732 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5733 rx_desc->read.hdr_addr = 0;
5737 if (i == rx_ring->count)
5739 buffer_info = &rx_ring->buffer_info[i];
5743 if (rx_ring->next_to_use != i) {
5744 rx_ring->next_to_use = i;
5746 i = (rx_ring->count - 1);
5750 /* Force memory writes to complete before letting h/w
5751 * know there are new descriptors to fetch. (Only
5752 * applicable for weak-ordered memory model archs,
5753 * such as IA-64). */
5755 writel(i, rx_ring->tail);
5765 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5767 struct igb_adapter *adapter = netdev_priv(netdev);
5768 struct mii_ioctl_data *data = if_mii(ifr);
5770 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5775 data->phy_id = adapter->hw.phy.addr;
5778 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5790 * igb_hwtstamp_ioctl - control hardware time stamping
5795 * Outgoing time stamping can be enabled and disabled. Play nice and
5796 * disable it when requested, although it shouldn't case any overhead
5797 * when no packet needs it. At most one packet in the queue may be
5798 * marked for time stamping, otherwise it would be impossible to tell
5799 * for sure to which packet the hardware time stamp belongs.
5801 * Incoming time stamping has to be configured via the hardware
5802 * filters. Not all combinations are supported, in particular event
5803 * type has to be specified. Matching the kind of event packet is
5804 * not supported, with the exception of "all V2 events regardless of
5808 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5809 struct ifreq *ifr, int cmd)
5811 struct igb_adapter *adapter = netdev_priv(netdev);
5812 struct e1000_hw *hw = &adapter->hw;
5813 struct hwtstamp_config config;
5814 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5815 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5816 u32 tsync_rx_cfg = 0;
5821 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5824 /* reserved for future extensions */
5828 switch (config.tx_type) {
5829 case HWTSTAMP_TX_OFF:
5831 case HWTSTAMP_TX_ON:
5837 switch (config.rx_filter) {
5838 case HWTSTAMP_FILTER_NONE:
5841 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5842 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5843 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5844 case HWTSTAMP_FILTER_ALL:
5846 * register TSYNCRXCFG must be set, therefore it is not
5847 * possible to time stamp both Sync and Delay_Req messages
5848 * => fall back to time stamping all packets
5850 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5851 config.rx_filter = HWTSTAMP_FILTER_ALL;
5853 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5854 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5855 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5858 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5859 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5860 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5863 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5864 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5865 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5866 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5869 config.rx_filter = HWTSTAMP_FILTER_SOME;
5871 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5872 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5873 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5874 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5877 config.rx_filter = HWTSTAMP_FILTER_SOME;
5879 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5880 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5881 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5882 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5883 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5890 if (hw->mac.type == e1000_82575) {
5891 if (tsync_rx_ctl | tsync_tx_ctl)
5897 * Per-packet timestamping only works if all packets are
5898 * timestamped, so enable timestamping in all packets as
5899 * long as one rx filter was configured.
5901 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5902 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5903 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5906 /* enable/disable TX */
5907 regval = rd32(E1000_TSYNCTXCTL);
5908 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5909 regval |= tsync_tx_ctl;
5910 wr32(E1000_TSYNCTXCTL, regval);
5912 /* enable/disable RX */
5913 regval = rd32(E1000_TSYNCRXCTL);
5914 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5915 regval |= tsync_rx_ctl;
5916 wr32(E1000_TSYNCRXCTL, regval);
5918 /* define which PTP packets are time stamped */
5919 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5921 /* define ethertype filter for timestamped packets */
5924 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5925 E1000_ETQF_1588 | /* enable timestamping */
5926 ETH_P_1588)); /* 1588 eth protocol type */
5928 wr32(E1000_ETQF(3), 0);
5930 #define PTP_PORT 319
5931 /* L4 Queue Filter[3]: filter by destination port and protocol */
5933 u32 ftqf = (IPPROTO_UDP /* UDP */
5934 | E1000_FTQF_VF_BP /* VF not compared */
5935 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5936 | E1000_FTQF_MASK); /* mask all inputs */
5937 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5939 wr32(E1000_IMIR(3), htons(PTP_PORT));
5940 wr32(E1000_IMIREXT(3),
5941 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5942 if (hw->mac.type == e1000_82576) {
5943 /* enable source port check */
5944 wr32(E1000_SPQF(3), htons(PTP_PORT));
5945 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5947 wr32(E1000_FTQF(3), ftqf);
5949 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5953 adapter->hwtstamp_config = config;
5955 /* clear TX/RX time stamp registers, just to be sure */
5956 regval = rd32(E1000_TXSTMPH);
5957 regval = rd32(E1000_RXSTMPH);
5959 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5969 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5975 return igb_mii_ioctl(netdev, ifr, cmd);
5977 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5983 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5985 struct igb_adapter *adapter = hw->back;
5988 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5990 return -E1000_ERR_CONFIG;
5992 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5997 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5999 struct igb_adapter *adapter = hw->back;
6002 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6004 return -E1000_ERR_CONFIG;
6006 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6011 static void igb_vlan_rx_register(struct net_device *netdev,
6012 struct vlan_group *grp)
6014 struct igb_adapter *adapter = netdev_priv(netdev);
6015 struct e1000_hw *hw = &adapter->hw;
6018 igb_irq_disable(adapter);
6019 adapter->vlgrp = grp;
6022 /* enable VLAN tag insert/strip */
6023 ctrl = rd32(E1000_CTRL);
6024 ctrl |= E1000_CTRL_VME;
6025 wr32(E1000_CTRL, ctrl);
6027 /* Disable CFI check */
6028 rctl = rd32(E1000_RCTL);
6029 rctl &= ~E1000_RCTL_CFIEN;
6030 wr32(E1000_RCTL, rctl);
6032 /* disable VLAN tag insert/strip */
6033 ctrl = rd32(E1000_CTRL);
6034 ctrl &= ~E1000_CTRL_VME;
6035 wr32(E1000_CTRL, ctrl);
6038 igb_rlpml_set(adapter);
6040 if (!test_bit(__IGB_DOWN, &adapter->state))
6041 igb_irq_enable(adapter);
6044 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6046 struct igb_adapter *adapter = netdev_priv(netdev);
6047 struct e1000_hw *hw = &adapter->hw;
6048 int pf_id = adapter->vfs_allocated_count;
6050 /* attempt to add filter to vlvf array */
6051 igb_vlvf_set(adapter, vid, true, pf_id);
6053 /* add the filter since PF can receive vlans w/o entry in vlvf */
6054 igb_vfta_set(hw, vid, true);
6057 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6059 struct igb_adapter *adapter = netdev_priv(netdev);
6060 struct e1000_hw *hw = &adapter->hw;
6061 int pf_id = adapter->vfs_allocated_count;
6064 igb_irq_disable(adapter);
6065 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6067 if (!test_bit(__IGB_DOWN, &adapter->state))
6068 igb_irq_enable(adapter);
6070 /* remove vlan from VLVF table array */
6071 err = igb_vlvf_set(adapter, vid, false, pf_id);
6073 /* if vid was not present in VLVF just remove it from table */
6075 igb_vfta_set(hw, vid, false);
6078 static void igb_restore_vlan(struct igb_adapter *adapter)
6080 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6082 if (adapter->vlgrp) {
6084 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6085 if (!vlan_group_get_device(adapter->vlgrp, vid))
6087 igb_vlan_rx_add_vid(adapter->netdev, vid);
6092 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6094 struct pci_dev *pdev = adapter->pdev;
6095 struct e1000_mac_info *mac = &adapter->hw.mac;
6100 case SPEED_10 + DUPLEX_HALF:
6101 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6103 case SPEED_10 + DUPLEX_FULL:
6104 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6106 case SPEED_100 + DUPLEX_HALF:
6107 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6109 case SPEED_100 + DUPLEX_FULL:
6110 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6112 case SPEED_1000 + DUPLEX_FULL:
6114 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6116 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6118 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6124 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6126 struct net_device *netdev = pci_get_drvdata(pdev);
6127 struct igb_adapter *adapter = netdev_priv(netdev);
6128 struct e1000_hw *hw = &adapter->hw;
6129 u32 ctrl, rctl, status;
6130 u32 wufc = adapter->wol;
6135 netif_device_detach(netdev);
6137 if (netif_running(netdev))
6140 igb_clear_interrupt_scheme(adapter);
6143 retval = pci_save_state(pdev);
6148 status = rd32(E1000_STATUS);
6149 if (status & E1000_STATUS_LU)
6150 wufc &= ~E1000_WUFC_LNKC;
6153 igb_setup_rctl(adapter);
6154 igb_set_rx_mode(netdev);
6156 /* turn on all-multi mode if wake on multicast is enabled */
6157 if (wufc & E1000_WUFC_MC) {
6158 rctl = rd32(E1000_RCTL);
6159 rctl |= E1000_RCTL_MPE;
6160 wr32(E1000_RCTL, rctl);
6163 ctrl = rd32(E1000_CTRL);
6164 /* advertise wake from D3Cold */
6165 #define E1000_CTRL_ADVD3WUC 0x00100000
6166 /* phy power management enable */
6167 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6168 ctrl |= E1000_CTRL_ADVD3WUC;
6169 wr32(E1000_CTRL, ctrl);
6171 /* Allow time for pending master requests to run */
6172 igb_disable_pcie_master(hw);
6174 wr32(E1000_WUC, E1000_WUC_PME_EN);
6175 wr32(E1000_WUFC, wufc);
6178 wr32(E1000_WUFC, 0);
6181 *enable_wake = wufc || adapter->en_mng_pt;
6183 igb_power_down_link(adapter);
6185 igb_power_up_link(adapter);
6187 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6188 * would have already happened in close and is redundant. */
6189 igb_release_hw_control(adapter);
6191 pci_disable_device(pdev);
6197 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6202 retval = __igb_shutdown(pdev, &wake);
6207 pci_prepare_to_sleep(pdev);
6209 pci_wake_from_d3(pdev, false);
6210 pci_set_power_state(pdev, PCI_D3hot);
6216 static int igb_resume(struct pci_dev *pdev)
6218 struct net_device *netdev = pci_get_drvdata(pdev);
6219 struct igb_adapter *adapter = netdev_priv(netdev);
6220 struct e1000_hw *hw = &adapter->hw;
6223 pci_set_power_state(pdev, PCI_D0);
6224 pci_restore_state(pdev);
6225 pci_save_state(pdev);
6227 err = pci_enable_device_mem(pdev);
6230 "igb: Cannot enable PCI device from suspend\n");
6233 pci_set_master(pdev);
6235 pci_enable_wake(pdev, PCI_D3hot, 0);
6236 pci_enable_wake(pdev, PCI_D3cold, 0);
6238 if (igb_init_interrupt_scheme(adapter)) {
6239 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6245 /* let the f/w know that the h/w is now under the control of the
6247 igb_get_hw_control(adapter);
6249 wr32(E1000_WUS, ~0);
6251 if (netif_running(netdev)) {
6252 err = igb_open(netdev);
6257 netif_device_attach(netdev);
6263 static void igb_shutdown(struct pci_dev *pdev)
6267 __igb_shutdown(pdev, &wake);
6269 if (system_state == SYSTEM_POWER_OFF) {
6270 pci_wake_from_d3(pdev, wake);
6271 pci_set_power_state(pdev, PCI_D3hot);
6275 #ifdef CONFIG_NET_POLL_CONTROLLER
6277 * Polling 'interrupt' - used by things like netconsole to send skbs
6278 * without having to re-enable interrupts. It's not called while
6279 * the interrupt routine is executing.
6281 static void igb_netpoll(struct net_device *netdev)
6283 struct igb_adapter *adapter = netdev_priv(netdev);
6284 struct e1000_hw *hw = &adapter->hw;
6287 if (!adapter->msix_entries) {
6288 struct igb_q_vector *q_vector = adapter->q_vector[0];
6289 igb_irq_disable(adapter);
6290 napi_schedule(&q_vector->napi);
6294 for (i = 0; i < adapter->num_q_vectors; i++) {
6295 struct igb_q_vector *q_vector = adapter->q_vector[i];
6296 wr32(E1000_EIMC, q_vector->eims_value);
6297 napi_schedule(&q_vector->napi);
6300 #endif /* CONFIG_NET_POLL_CONTROLLER */
6303 * igb_io_error_detected - called when PCI error is detected
6304 * @pdev: Pointer to PCI device
6305 * @state: The current pci connection state
6307 * This function is called after a PCI bus error affecting
6308 * this device has been detected.
6310 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6311 pci_channel_state_t state)
6313 struct net_device *netdev = pci_get_drvdata(pdev);
6314 struct igb_adapter *adapter = netdev_priv(netdev);
6316 netif_device_detach(netdev);
6318 if (state == pci_channel_io_perm_failure)
6319 return PCI_ERS_RESULT_DISCONNECT;
6321 if (netif_running(netdev))
6323 pci_disable_device(pdev);
6325 /* Request a slot slot reset. */
6326 return PCI_ERS_RESULT_NEED_RESET;
6330 * igb_io_slot_reset - called after the pci bus has been reset.
6331 * @pdev: Pointer to PCI device
6333 * Restart the card from scratch, as if from a cold-boot. Implementation
6334 * resembles the first-half of the igb_resume routine.
6336 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6338 struct net_device *netdev = pci_get_drvdata(pdev);
6339 struct igb_adapter *adapter = netdev_priv(netdev);
6340 struct e1000_hw *hw = &adapter->hw;
6341 pci_ers_result_t result;
6344 if (pci_enable_device_mem(pdev)) {
6346 "Cannot re-enable PCI device after reset.\n");
6347 result = PCI_ERS_RESULT_DISCONNECT;
6349 pci_set_master(pdev);
6350 pci_restore_state(pdev);
6351 pci_save_state(pdev);
6353 pci_enable_wake(pdev, PCI_D3hot, 0);
6354 pci_enable_wake(pdev, PCI_D3cold, 0);
6357 wr32(E1000_WUS, ~0);
6358 result = PCI_ERS_RESULT_RECOVERED;
6361 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6363 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6364 "failed 0x%0x\n", err);
6365 /* non-fatal, continue */
6372 * igb_io_resume - called when traffic can start flowing again.
6373 * @pdev: Pointer to PCI device
6375 * This callback is called when the error recovery driver tells us that
6376 * its OK to resume normal operation. Implementation resembles the
6377 * second-half of the igb_resume routine.
6379 static void igb_io_resume(struct pci_dev *pdev)
6381 struct net_device *netdev = pci_get_drvdata(pdev);
6382 struct igb_adapter *adapter = netdev_priv(netdev);
6384 if (netif_running(netdev)) {
6385 if (igb_up(adapter)) {
6386 dev_err(&pdev->dev, "igb_up failed after reset\n");
6391 netif_device_attach(netdev);
6393 /* let the f/w know that the h/w is now under the control of the
6395 igb_get_hw_control(adapter);
6398 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6401 u32 rar_low, rar_high;
6402 struct e1000_hw *hw = &adapter->hw;
6404 /* HW expects these in little endian so we reverse the byte order
6405 * from network order (big endian) to little endian
6407 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6408 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6409 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6411 /* Indicate to hardware the Address is Valid. */
6412 rar_high |= E1000_RAH_AV;
6414 if (hw->mac.type == e1000_82575)
6415 rar_high |= E1000_RAH_POOL_1 * qsel;
6417 rar_high |= E1000_RAH_POOL_1 << qsel;
6419 wr32(E1000_RAL(index), rar_low);
6421 wr32(E1000_RAH(index), rar_high);
6425 static int igb_set_vf_mac(struct igb_adapter *adapter,
6426 int vf, unsigned char *mac_addr)
6428 struct e1000_hw *hw = &adapter->hw;
6429 /* VF MAC addresses start at end of receive addresses and moves
6430 * torwards the first, as a result a collision should not be possible */
6431 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6433 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6435 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6440 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6442 struct igb_adapter *adapter = netdev_priv(netdev);
6443 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6445 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6446 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6447 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6448 " change effective.");
6449 if (test_bit(__IGB_DOWN, &adapter->state)) {
6450 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6451 " but the PF device is not up.\n");
6452 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6453 " attempting to use the VF device.\n");
6455 return igb_set_vf_mac(adapter, vf, mac);
6458 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6463 static int igb_ndo_get_vf_config(struct net_device *netdev,
6464 int vf, struct ifla_vf_info *ivi)
6466 struct igb_adapter *adapter = netdev_priv(netdev);
6467 if (vf >= adapter->vfs_allocated_count)
6470 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6472 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6473 ivi->qos = adapter->vf_data[vf].pf_qos;
6477 static void igb_vmm_control(struct igb_adapter *adapter)
6479 struct e1000_hw *hw = &adapter->hw;
6482 switch (hw->mac.type) {
6485 /* replication is not supported for 82575 */
6488 /* notify HW that the MAC is adding vlan tags */
6489 reg = rd32(E1000_DTXCTL);
6490 reg |= E1000_DTXCTL_VLAN_ADDED;
6491 wr32(E1000_DTXCTL, reg);
6493 /* enable replication vlan tag stripping */
6494 reg = rd32(E1000_RPLOLR);
6495 reg |= E1000_RPLOLR_STRVLAN;
6496 wr32(E1000_RPLOLR, reg);
6498 /* none of the above registers are supported by i350 */
6502 if (adapter->vfs_allocated_count) {
6503 igb_vmdq_set_loopback_pf(hw, true);
6504 igb_vmdq_set_replication_pf(hw, true);
6506 igb_vmdq_set_loopback_pf(hw, false);
6507 igb_vmdq_set_replication_pf(hw, false);